diff --git a/.github/workflows/operatorx-sweep.yml b/.github/workflows/operatorx-sweep.yml index 593ed58a86..7f3a5b8527 100644 --- a/.github/workflows/operatorx-sweep.yml +++ b/.github/workflows/operatorx-sweep.yml @@ -3,21 +3,35 @@ name: OperatorX Sweep on: workflow_dispatch: inputs: - pool: - description: GPU runner pool (one physical node per shard) + runner: + description: Self-hosted runner label (one physical node per shard) type: choice - default: h100-dgxc - options: [h100-dgxc, h200-dgxc, b200-nscale, b300, gb200, gb300, mi300x, mi325x, mi355x] + default: cluster:h100-dsxe + options: + - cluster:h100-dsxe + - cluster:h200-dgxc + - cluster:b200-nscale + - cluster:b300-dsxe + - cluster:gb200-nv + - cluster:gb300-nv + - cluster:mi300x-amd + - cluster:mi325x-amds + - cluster:mi355x-amds backends: - description: Comma-separated backends (torch GEMM/attention, AMD aiter attention, vllm routed MoE) + description: Comma-separated backends (vllm GEMM; AMD also torch GEMM) type: string - default: torch + default: vllm testlists: description: Comma-separated testlist stems type: string default: gemm + mode: + description: timing (latency, telemetry, profiler replay) or counters (per-kernel ncu / rocprofv3 counters; latencies perturbed) + type: choice + default: timing + options: [timing, counters] world_sizes: - description: Comma-separated world sizes (1,2,4; also 8 on eight-GPU pools) + description: Comma-separated world sizes (1,2,4; also 8 on eight-GPU nodes) type: string default: '1' chunk_size: @@ -25,9 +39,13 @@ on: type: number default: 500 recovery_run_id: - description: Optional recent OperatorX run on this pool whose failed cleanup should be retried + description: Optional recent OperatorX run on this runner whose failed cleanup should be retried type: string default: '' + ingest: + description: Ingest the results into the OperatorX database (false for test runs) + type: boolean + default: true pull_request: paths: - operatorx/** @@ -60,22 +78,27 @@ jobs: - name: Plan requested coverage id: plan env: - OPERATORX_POOL: ${{ inputs.pool || 'h100-dgxc' }} - OPERATORX_BACKENDS: ${{ inputs.backends || 'torch' }} + OPERATORX_RUNNER: ${{ inputs.runner || 'cluster:h100-dsxe' }} + OPERATORX_BACKENDS: ${{ inputs.backends || 'vllm' }} OPERATORX_TESTLISTS: ${{ inputs.testlists || 'gemm' }} OPERATORX_WORLD_SIZES: ${{ inputs.world_sizes || '1' }} + OPERATORX_MODE: ${{ inputs.mode || 'timing' }} OPERATORX_RECOVERY_RUN: ${{ inputs.recovery_run_id }} OPERATORX_CHUNK_SIZE: ${{ github.event_name == 'workflow_dispatch' && format('{0}', inputs.chunk_size) || '500' }} PYTHONPATH: .:inferencex-e2e run: | set -eo pipefail source inferencex-e2e/benchmarks/check_env.sh - check_env_vars OPERATORX_POOL OPERATORX_BACKENDS OPERATORX_TESTLISTS OPERATORX_WORLD_SIZES OPERATORX_CHUNK_SIZE GITHUB_RUN_ID GITHUB_RUN_ATTEMPT GITHUB_SHA GITHUB_OUTPUT - python3 -m operatorx.ci plan \ + check_env_vars OPERATORX_RUNNER OPERATORX_BACKENDS OPERATORX_TESTLISTS OPERATORX_WORLD_SIZES OPERATORX_MODE OPERATORX_CHUNK_SIZE GITHUB_RUN_ID GITHUB_RUN_ATTEMPT GITHUB_SHA GITHUB_OUTPUT + # Cases take InferenceX's srt-slurm recipes, expanded by InferenceX's own code (srtctl). + git submodule update --init inferencex-e2e/utils/srt-slurm + uv run --no-project --exclude-newer PT12H --python 3.12 --with pyyaml --with ruamel.yaml \ + --with marshmallow --with marshmallow-dataclass --with pydantic --with jinja2 --with requests --with rich \ + python -m operatorx.ci plan \ --platform-config operatorx/platforms.json \ - --pool "$OPERATORX_POOL" --backends "$OPERATORX_BACKENDS" \ + --runner "$OPERATORX_RUNNER" --backends "$OPERATORX_BACKENDS" \ --testlists "$OPERATORX_TESTLISTS" --world-sizes "$OPERATORX_WORLD_SIZES" \ - --chunk-size "$OPERATORX_CHUNK_SIZE" --run-id "$GITHUB_RUN_ID" \ + --chunk-size "$OPERATORX_CHUNK_SIZE" --mode "$OPERATORX_MODE" --run-id "$GITHUB_RUN_ID" \ --attempt "$GITHUB_RUN_ATTEMPT" --source-sha "$GITHUB_SHA" \ --out operatorx-manifest.json > operatorx-matrix.json if [ -n "$OPERATORX_RECOVERY_RUN" ]; then @@ -106,10 +129,10 @@ jobs: runs-on: >- # zizmor: ignore[self-hosted-runner] GPU Slurm runners by design; this job runs only on workflow_dispatch ${{ fromJSON(vars.PRIORITY_SCHEDULER_ENABLED == 'true' && format('["self-hosted",{0},"nodes:1",{1},{2}]', - toJSON(matrix.pool), + toJSON(matrix.runner), toJSON(format('ci-job-{0}-{1}', needs.setup.outputs.priority, matrix.queue-token)), toJSON(format('ci-attempt-{0}', github.run_attempt))) || - format('["self-hosted",{0}]', toJSON(matrix.pool))) }} + format('["self-hosted",{0}]', toJSON(matrix.runner))) }} name: p${{ needs.setup.outputs.priority }} | OperatorX ${{ matrix.id }} env: OPERATORX_SHARD: ${{ matrix.id }} @@ -149,13 +172,13 @@ jobs: if: inputs.recovery_run_id != '' env: OPERATORX_RECOVERY_RUN: ${{ inputs.recovery_run_id }} - OPERATORX_POOL: ${{ matrix.pool }} + OPERATORX_RUNNER: ${{ matrix.runner }} run: | set -eo pipefail source inferencex-e2e/benchmarks/check_env.sh - check_env_vars RUNNER_TEMP OPERATORX_RECOVERY_RUN OPERATORX_POOL OPERATORX_CLEANUP_SECONDS + check_env_vars RUNNER_TEMP OPERATORX_RECOVERY_RUN OPERATORX_RUNNER OPERATORX_CLEANUP_SECONDS python3 -m operatorx.ci recover --artifacts operatorx-recovery \ - --run-id "$OPERATORX_RECOVERY_RUN" --pool "$OPERATORX_POOL" \ + --run-id "$OPERATORX_RECOVERY_RUN" --runner "$OPERATORX_RUNNER" \ --cleanup-seconds "$OPERATORX_CLEANUP_SECONDS" \ --platform-config operatorx/platforms.json - name: Run one Slurm allocation @@ -233,3 +256,27 @@ jobs: path: operatorx-summary.json if-no-files-found: error retention-days: 14 + + trigger-ingest: + name: Trigger OperatorX database ingest + # A run with failed shards is still stored: its unrun cases read as missing. + if: >- + always() && github.event_name == 'workflow_dispatch' && inputs.ingest && + (needs.summarize.result == 'success' || needs.summarize.result == 'failure') + needs: [summarize] + runs-on: ubuntu-latest + steps: + - name: Dispatch ingest to InferenceX-app + uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 + env: + RUN_ID: ${{ github.run_id }} + with: + # Cross-repository dispatch credential, as run-sweep.yml's database ingest uses. + github-token: ${{ secrets.FRONTEND_PAT }} # zizmor: ignore[secrets-outside-env] + script: | + await github.rest.repos.createDispatchEvent({ + owner: "SemiAnalysisAI", + repo: "InferenceX-app", + event_type: "ingest-operatorx", + client_payload: { "run-id": process.env.RUN_ID } + }); diff --git a/operatorx/CI.md b/operatorx/CI.md index 8968da160c..b6ef0012f6 100644 --- a/operatorx/CI.md +++ b/operatorx/CI.md @@ -1,190 +1,43 @@ # OperatorX GitHub Actions -[OperatorX Sweep](../.github/workflows/operatorx-sweep.yml) runs manually on -`h100-dgxc` (default), `h200-dgxc`, `b200-nscale`, `b300`, `gb200`, `gb300`, -`mi300x`, `mi325x`, or `mi355x`. -Pull requests only run the hosted planner; GPU work requires `workflow_dispatch`. +[OperatorX Sweep](../.github/workflows/operatorx-sweep.yml) runs on `workflow_dispatch` only (pull requests run just the hosted planner). -| GPU | Pool | GPUs per physical node | Image platform | Result cluster | +| GPU | Runner label | GPUs / node | Image platform | Result cluster | | --- | --- | ---: | --- | --- | -| H100 | `h100-dgxc` | 8 | `linux/amd64` | `h100_dgxc_8x` | -| H200 | `h200-dgxc` | 8 | `linux/amd64` | `h200_dgxc_8x` | -| B200 | `b200-nscale` | 8 | `linux/amd64` | `b200_nscale_8x` | -| B300 | `b300` | 8 | `linux/amd64` | `b300_dsxe_8x` | -| GB200 | `gb200` | 4 | `linux/arm64` | `gb200_nvl72_4x` | -| GB300 | `gb300` | 4 | `linux/arm64` | `gb300_nvl72_4x` | -| MI300X | `mi300x` | 8 | `linux/amd64` | `mi300x_amds_8x` | -| MI325X | `mi325x` | 8 | `linux/amd64` | `mi325x_amds_8x` | -| MI355X | `mi355x` | 8 | `linux/amd64` | `mi355x_8x` | - -GB200/GB300 runs use one four-GPU tray, not the full NVL72 rack. Dense GEMM uses -`world_sizes=1` on every pool; reported TFLOPS remains per GPU. Hardware facts come -from CollectiveX's platform registry, and both planning and execution validate them. +| H100 | `cluster:h100-dsxe` (default) | 8 | `linux/amd64` | `h100_dgxc_8x` | +| H200 | `cluster:h200-dgxc` | 8 | `linux/amd64` | `h200_dgxc_8x` | +| B200 | `cluster:b200-nscale` | 8 | `linux/amd64` | `b200_nscale_8x` | +| B300 | `cluster:b300-dsxe` | 8 | `linux/amd64` | `b300_dsxe_8x` | +| GB200 | `cluster:gb200-nv` | 4 | `linux/arm64` | `gb200_nvl72_4x` | +| GB300 | `cluster:gb300-nv` | 4 | `linux/arm64` | `gb300_nvl72_4x` | +| MI300X | `cluster:mi300x-amd` | 8 | `linux/amd64` | `mi300x_amds_8x` | +| MI325X | `cluster:mi325x-amds` | 8 | `linux/amd64` | `mi325x_amds_8x` | +| MI355X | `cluster:mi355x-amds` | 8 | `linux/amd64` | `mi355x_8x` | ## Dispatch -Select **OperatorX Sweep → Run workflow**, -choose the source branch, and keep the initial defaults: `pool=h100-dgxc`, -`backends=torch`, `testlists=gemm`, `world_sizes=1`, `chunk_size=500`. -This schedules the complete checked-in GEMM catalog in bounded shards (currently -7,212 cases in 15 shards). The catalog includes formats unsupported by a selected -backend and shapes that can exceed device memory. Unsupported rows remain visible; -actual kernel and allocation errors fail CI. A full catalog run is not a promise -that every case fits or is supported on H100. - ```bash gh workflow run operatorx-sweep.yml --repo SemiAnalysisAI/InferenceX \ - --ref -f pool=h100-dgxc -f backends=torch \ - -f testlists=gemm -f world_sizes=1 -f chunk_size=500 + --ref -f runner=cluster:h100-dsxe -f backends=vllm \ + -f testlists=gemm -f world_sizes=1 -f chunk_size=500 -f ingest=false ``` -For a quick infrastructure smoke check, explicitly select `testlists=gemm_perf` -and `chunk_size=50` (11 BF16 cases). Other NVIDIA backends and testlists are explicit -selections, not validated Hopper coverage. Unsupported operations remain visible in results. Backend -import errors, benchmark errors, and zero successful rows fail the shard. -Start with BF16 GEMM, then bounded collectives and compatible MoE combinations. -Do not infer that Blackwell-specific FP4 kernels work on Hopper. +- Inputs: `runner`, `backends` (`vllm`; AMD also `torch`), `testlists`, `mode` (`timing` | `counters`), `world_sizes` (1, 2, 4, 8), `chunk_size` (1-500), `recovery_run_id`, `ingest` (default true; `false` for test runs). +- `mode=counters`: each op once under Nsight Compute (NVIDIA) or rocprofv3 (AMD, 12 passes: use a smaller `chunk_size`); raw files in `results/counters/`; latencies perturbed. +- Smoke check: `testlists=gemm_perf`, `chunk_size=50`. Dispatch `attn_*` testlists apart from gemm / moe ones. ## Execution contract -- Hosted planning validates inputs, groups backends by container image, separates - world sizes and MoE parallelism triples, and splits shapes into bounded chunks. - At most 256 shards are accepted. World sizes are restricted to 1, 2, 4, and 8, - and must fit one physical node (GB200/GB300 reject 8). Shapes outside the - requested sizes are counted in `excluded_shapes`. -- Each Actions shard holds exactly one exclusive physical Slurm node (four or eight GPUs). The GPU - process count is the selected world size. Admission uses the existing priority - scorer and `ci-job-*`, `ci-attempt-*`, and exactly one `nodes:1` label. All shards remain eligible; the priority/node scheduler controls physical-node - admission. A second GitHub matrix cap can strand assigned labels on held jobs. - Both scheduler switches must remain enabled. -- Runner settings come from CollectiveX's tracked platform registry. Source is - checked out at the workflow SHA and copied into a private, compute-visible - directory below the configured shared squash parent or a writable configured - `storage_roots` entry (GB200). B300 uses the compute-visible account home from - the password database; an explicit `stage_dir` takes - precedence. Results never depend on - a submit-host `/tmp` mount being visible to compute nodes. -- The planner resolves each image digest. Imports are locked and cached by image - plus digest and CPU architecture, with a second digest check after import. A moved or unresolvable - tag fails rather than claiming the planned image was measured. Images must be - anonymously readable from the planning and import hosts. Imports verify the host - CPU architecture and run inside their allocation. B300 follows the inference - launcher's compute-node import because its submit host lacks extraction space - for this image. Enroot and GNU parallel use private temporary directories; - Enroot uses explicit registry URLs and any pool-configured cache path. Allocation - forwards account, QoS, and quarantined nodes; B300/GB pools retain their existing - remap-root and memory settings. B300 leaves QoS selection to its partition/account, - matching the inference launcher; the former `batch_1_qos` override is rejected - by the current cluster. Its former excluded node names also do not exist in - this pool and have been removed; Slurm still honors drained nodes. GB300 retains - its configured QoS and exclusions. -- The launcher remains active through allocation, import, and execution. The - allocation time limit is 45 minutes; Actions permits 70 minutes including - queueing and cleanup. Slurm job names match the Actions runner name. -- Signals and the workflow's `always()` recovery step cancel recorded allocations, - stop writers, recover partial results, and remove staged sources. The workflow - explicitly allows 180 seconds for Slurm epilog/node release, including on H200. A failed - cleanup retains staging for investigation. Slurm's time limit is the last - bound if the runner host disappears. -- Strict CI runs atomically checkpoint rank-zero rows after each operation, - outside kernel timing. The existing non-CI timing loop is unchanged. - -## Artifacts and reruns - -`operatorx-manifest-` records requested cases, image digests and source -SHA. It remains available to failed-job reruns. Each attempt uploads separate -`operatorx-shard---` artifacts containing execution -metadata, allocation/import/benchmark logs, status and any raw result JSONs. -Startup failures can have logs without results; cancellation checkpoints are -partial coverage. A successful shard requires successful measurements, not just -successful Slurm submission. Result environments record the workflow run, -attempt, shard, source SHA and image digest. - -Download artifacts through `gh run download`. Preserve raw files and provenance; -`scripts/consolidate_results.py` is not part of CI. Dashboard ingestion is separate. - -## Local validation +- Planning: shards per backend image, `parallel` split and InferenceX recipe (`recipes.py`: image, launch env, `vllm serve` args), chunked by `chunk_size`; at most 256 shards; a case with no recipe runs on the image InferenceX serves the backend with on that hardware (`containers.toml` only where it serves none). Digest-pinned images import by digest (InferenceX's `enroot_uri`). +- One exclusive Slurm node per shard (GB200/GB300: one 4-GPU tray, no world size 8); 45 min allocation, 70 min job. +- Image: digest resolved at planning, imported with enroot on the allocated node, cached by image + digest + arch; a tag that moved since planning fails the shard. +- Ranks: one process per GPU of the split's world size, env:// rendezvous; runner settings from CollectiveX's platform registry overlaid by `platforms.json`. +- Results: rows checkpointed after each op; unsupported cases stay visible as rows, errors or zero successful rows fail the shard; artifacts `operatorx-manifest-` and `operatorx-shard---`; ingested into the OperatorX database unless `ingest=false`. +- Cleanup: cancellation and the `always()` step cancel the allocation and keep partial results; after a failed cleanup, rerun one shard with `recovery_run_id`. -Planning requires Python 3.11 or newer. Compute-side control code uses the existing -Python 3.10+ Slurm-host environment. CPU tests run the real planner, benchmark -orchestration and launcher with external GPU/Slurm collaborators substituted. +## Local tests ```bash uv run --no-project --python 3.12 --with pytest --with pyyaml --with torch --with numpy \ python -m pytest operatorx/tests/ -q ``` - -Real acceptance additionally requires a smoke run with artifacts on each selected pool, a -failed-shard rerun, and cancellation with confirmed allocation release. CPU -checks alone do not establish GPU compatibility or cluster storage visibility. - -The final coverage job selects the newest artifact attempt for each requested -shard, preserves successful shards from previous attempts, and fails if any shard -is missing or failed. Its summary separates requested shapes from result rows -(one shape may run on multiple backends). - -If cleanup failed, a single-shard dispatch can set `recovery_run_id` to the recent -OperatorX run from the same pool. It downloads the execution artifacts and retries -allocation/staging cleanup before allocating a new node. Recovery checks the run, -pool, and private staging parent; do not select unrelated or old Slurm executions. - -`cleanup.log` records the active-job query used to confirm allocation release. -It queries the current user’s job list because querying a removed job ID directly -can return a Slurm error even after that allocation has terminated. - -## AMD execution - -`platforms.json` overlays the CollectiveX registry with the AMDS Slurm pools. -AMD accepts single-GPU `torch` GEMM and `torch,aiter` attention. ROCm PyTorch uses HIP events through -`torch.cuda`; FP8 selects FNUZ on gfx942 and OCP on gfx950. Unsupported formats -remain explicit. Staging lives outside `_work`, below the shared runner root -derived from `RUNNER_TEMP`. Containers never write to the checkout. MI300X/MI325X -forward `/dev/kfd` and `/dev/dri`; CPU requests follow each inference launcher. - -## Attention - -Select `testlists=attention_perf` for eight BF16/FP16 MHA/GQA and materialized MLA -prefill/decode cases, or `attention` for the full 2,315-case catalog. NVIDIA uses -`backends=torch`; AMD also supports `backends=torch,aiter`. Attention measures -latency in microseconds. Unsupported precision/layout combinations are recorded, -and allocation or kernel errors still fail CI. Strict CI retains unsupported -backend/operator pairs instead of dropping requested coverage. - -PyTorch uses bottom-right causal masking for rectangular decode inputs. Grouped -KV expansion happens before timing. Both MLA backends time attention on -materialized Q/K/V; compressed-cache projection and RoPE are excluded. AITER calls -`flash_attn_func` directly with native grouped KV heads and bottom-right causality. -It accepts uniform BF16/FP16, contiguous KV, and head dimensions divisible by eight -up to 256; other requests remain unsupported rather than fall back to torch. -Experimental operator changes are recorded in the adjacent `perf-changelog.yaml`, -separately from the `../inferencex-e2e/perf-changelog.yaml` config-key schema. - - -## Kimi K3 routed MoE benchmark profile - -`testlists=kimi_k3_moe_perf`, `backends=vllm`, `world_sizes=1` runs eight BF16 -routed-expert cases: 1, 16, 128 and 1024 local tokens, each with EP8 or TP8 shapes. -The generic profile follows [vLLM #50082](https://github.com/vllm-project/vllm/pull/50082): -896 experts, top-16, hidden 7168 and intermediate 3072. EP8 allocates 112 experts -with intermediate 3072; TP8 allocates 896 experts with intermediate 384. -Both execute on **one GPU**, with no live distributed groups or communication. - -This is explicitly **Kimi K3 (vLLM benchmark profile)**. It measures vLLM's generic -SiLU routed expert kernel with synthetic, uniform-random local routing prepared -before timing. The timer includes the fused expert implementation's token sorting, -gate/up GEMM, SiLU-and-multiply, down GEMM and weighted reduction; it excludes -router/top-k calculation, shared experts and communication. It does not measure -the released Kimi K3 layer's SITU activation, 3584-wide latent expert path, -latent projections or shared experts. Those need a separate native-layer profile. -No model weights or Hugging Face credentials are required. - -NVIDIA uses `vllm/vllm-openai:v0.19.0` (amd64/arm64); AMD uses the existing ROCm -image. Select `backends=vllm` explicitly. Unsupported precision, routing or shared -expert requests produce unsupported rows; import and kernel failures fail CI. - -Useful routed matmul TFLOPS per GPU is -`6*num_tokens*top_k*hidden*(intermediate/routed_tensor_parallel_size)/(latency_us*1e6)`. -The local top-k routes all target the local expert table, matching the generic -benchmark's EP emulation. Do not divide the measured work by EP again or multiply -by the number of GPUs in the allocation. These kernel measurements exclude -activation and routing FLOPs and are not full-model throughput. diff --git a/operatorx/CLUSTERS.md b/operatorx/CLUSTERS.md deleted file mode 100644 index 046c132cef..0000000000 --- a/operatorx/CLUSTERS.md +++ /dev/null @@ -1,186 +0,0 @@ -# Clusters - -Reference for the clusters operatorx targets: hardware, how the runner is -launched per platform, and the per-platform quirks. Access details -(hostnames, credentials, checkout paths) are deployment-specific and are not -included here. Fill them in for your own environment. - -Platform → cluster routing lives in `operatorx/clusters.py` (`CLUSTER_PLATFORMS`). -SLURM/env defaults live in `scripts/submit_run.py` (`DEFAULT_CLUSTER`). - -## Conventions - -- `submit_run.py` is **SLURM-only**. It submits `sbatch`/`srun` jobs and is - used on the NVIDIA and AMD clusters. TPU and Trainium hosts do not use - SLURM. Run `python -m operatorx` directly inside the appropriate venv/image. -- Point `OPERATORX_SQUASH_DIR` at your container squash directory, and set - `OPERATORX_PARTITION` / `OPERATORX_ACCOUNT` / `OPERATORX_QOS` to your - cluster's SLURM values (see the env-var table at the bottom). -- Set `OPERATORX_JOB_NAME` to a recognizable SLURM job name for your runs. - -## At a glance - -| Cluster id | Platform | Hardware | Scheduler | -|---------------|----------|---------------------------------|------------| -| `b200_dgx_8x` | nvidia | 8× B200 SXM (DGX-style) | SLURM | -| `b300_hgx_8x` | nvidia | 8× B300 (HGX-style) | SLURM | -| `b200_nvl72` | nvidia | B200 NVL72 (routing only) | SLURM | -| `mi355x_8x` | amd | 8× MI355 OAM per node | SLURM | -| `v6e_1x` | tpu | 1× TPU v6e (1×1) | run direct | -| `v6e_4x` | tpu | 4× TPU v6e (2×2) | run direct | -| `v6e_pod` | tpu | TPU v6e pod slice | run direct | -| `trn3_1x` | trainium | 1 LNC of a trn3 host | run direct | -| `trn3_8x` | trainium | trn3 subset | run direct | -| `trn3_16x` | trainium | full trn3 host (16 devices) | run direct | - -## NVIDIA - -### b200 — DGX-style, 8× B200 SXM (`b200_dgx_8x`) - -| Setting | Value | -|----------|----------------------------------------------------------| -| Hardware | 8× B200 SXM per node | -| GRES | e.g. `gpu:nvidia_b200:8` | -| Defaults | `submit_run.py` defaults target this cluster (`OPERATORX_CLUSTER=b200_dgx_8x`, partition/squash dir from the script/env) | - -```bash -OPERATORX_JOB_NAME= python3 scripts/submit_run.py nvidia -``` - -### b300 — HGX-style, 8× B300 (`b300_hgx_8x`) - -| Setting | Value | -|-------------------|--------------------------------------------------------------| -| Hardware | 8× B300 per node | -| OS note | If the login node runs Python 3.10, `submit_run.py` falls back to `tomli` for TOML parsing | -| Excluded backends | Exclude DeepEP because IBGDA is known to hang on this fabric | - -Override the SLURM knobs for your cluster and drop DeepEP: - -```bash -OPERATORX_CLUSTER=b300_hgx_8x \ -OPERATORX_PARTITION= \ -OPERATORX_ACCOUNT= \ -OPERATORX_QOS= \ -OPERATORX_SQUASH_DIR= \ -OPERATORX_BACKENDS=torch,deepgemm,flashinfer,sglang \ -OPERATORX_JOB_NAME= \ -python3 scripts/submit_run.py nvidia -``` - -### `b200_nvl72` - -Present in `CLUSTER_PLATFORMS` (routes to nvidia) as a placeholder. There is no -hardware-specific guidance yet. - -## AMD — `mi355x_8x` - -| Setting | Value | -|-----------|--------------------------------------------------------------| -| Hardware | 8× MI355 OAM per node | -| GRES | e.g. `gpu:amd_instinct_mi355_oam:8` | -| Backends | `containers.toml` `amd.torch` / `amd.aiter` → `vllm-openai-rocm` | - -```bash -OPERATORX_CLUSTER=mi355x_8x \ -OPERATORX_PARTITION= \ -OPERATORX_SQUASH_DIR= \ -OPERATORX_JOB_NAME= \ -python3 scripts/submit_run.py amd -``` - -## TPU - -No SLURM is available. Run `python -m operatorx` directly on the TPU VM. Access a VM with -`gcloud compute tpus tpu-vm ssh --zone=`. - -| Cluster id | Topology | Chips | ws range | Use for | -|------------|----------|-------|----------|-----------------------------------------------------| -| `v6e_1x` | 1×1 | 1 | 1 | Single-chip tests (gemm, moe_gemm). No collectives. | -| `v6e_4x` | 2×2 | 4 | 1–4 | Collectives + MoE-EP up to ws=4. | -| `v6e_pod` | pod | pod | auto | Multi-host v6e pod slice. Device count is auto-detected. | - -```bash -OPERATORX_CLUSTER=v6e_4x python -m operatorx -``` - -`moe_forward` on TPU emits `unsupported` unless Google's MaxText is installed -on the VM (the `jax` backend works without it): - -```bash -git clone https://github.com/AI-Hypercomputer/maxtext ~/maxtext -pip install -e ~/maxtext -``` - -## Trainium — `trn3_16x` - -No SLURM is available. Run `python -m operatorx` directly on the instance. - -| Setting | Value | -|---------------------|-------------------------------------------------------------------| -| Hardware | 16 Neuron devices × 4 cores (64 physical NCs, 8 LNC=2 logical units) | -| Per-device HBM | 144 GiB | -| Cluster ids | `trn3_16x` (full host). `trn3_8x` and `trn3_1x` are subset configs | -| Device topology cmd | `/opt/aws/neuron/bin/neuron-ls` (may not be on `PATH`) | - -Neuron venvs (standard AWS Neuron paths under `/opt/`): - -- `aws_neuronx_venv_pytorch_2_9` provides base PyTorch and NeuronX -- `aws_neuronx_venv_pytorch_2_9_nxd_inference` adds NeuronX Distributed Inference -- `aws_neuronx_venv_pytorch_2_9_nxd_training` adds NXD training -- `aws_neuronx_venv_pytorch_inference_vllm_0_16` provides the vLLM build -- `aws_neuronx_venv_jax_0_7` provides Jax (not used by operatorx) - -```bash -source /opt/aws_neuronx_venv_pytorch_2_9/bin/activate -OPERATORX_CLUSTER=trn3_16x python -m operatorx -``` - -The Trainium runner forces `NEURON_LOGICAL_NC_CONFIG=2` and appends -`--logical-nc-config=2` to `NEURON_CC_FLAGS` -(`operatorx/runners/trainium/runner.py`). `world_size=1` on this platform = -one LNC=2 unit (1 HBM bank + 2 paired NC-v4 cores). - -## Cluster id → platform (`operatorx/clusters.py`) - -```python -CLUSTER_PLATFORMS = { - "h100_dgxc_8x": "nvidia", - "h200_dgxc_8x": "nvidia", - "b200_dgx_8x": "nvidia", - "b200_nscale_8x": "nvidia", - "b300_dsxe_8x": "nvidia", - "gb200_nvl72_4x": "nvidia", - "gb300_nvl72_4x": "nvidia", - "b300_hgx_8x": "nvidia", - "b200_nvl72": "nvidia", - "mi355x_8x": "amd", - "mi300x_amds_8x": "amd", - "mi325x_amds_8x": "amd", - "v6e_1x": "tpu", - "v6e_4x": "tpu", - "v6e_pod": "tpu", - "v7x_4x": "tpu", - "trn3_1x": "trainium", - "trn3_8x": "trainium", - "trn3_16x": "trainium", -} -``` - -## `submit_run.py` env vars - -| Var | Default | Notes | -|------------------------|-------------------------------|---------------------------------------------------------------------| -| `OPERATORX_CLUSTER` | per-platform (see script) | Routes the runner via `CLUSTER_PLATFORMS` | -| `OPERATORX_PARTITION` | site default | SLURM partition | -| `OPERATORX_ACCOUNT` | (omitted) | SLURM `--account` | -| `OPERATORX_QOS` | (omitted) | SLURM `--qos` | -| `OPERATORX_SQUASH_DIR` | site default | Where `.sqsh` lives (used by `srun --container-image=`) | -| `OPERATORX_BACKENDS` | all backends for the platform | CSV allowlist | -| `OPERATORX_JOB_NAME` | `benchmark` | SLURM job name | -| `OPERATORX_TESTLISTS` | all under `testlists/` | CSV of testlist stems to run | -| `OPERATORX_TIME_MIN` | `30` | `--time` (minutes) | - -`WORLD_SIZES = [1, 2, 4, 8]` supports single-node runs only. Multi-node NCCL IB bring-up -currently hangs on the B200/B300 fabrics. `MASTER_ADDR` is derived by parsing -`SLURM_NODELIST`. diff --git a/operatorx/README.md b/operatorx/README.md index 0251028c46..c81ff849f4 100644 --- a/operatorx/README.md +++ b/operatorx/README.md @@ -1,90 +1,31 @@ -# operatorx - Experimental (Beta) +# operatorx -Multi-platform inference operator benchmark suite. Times one op at a time -(gemm, attention, moe, collectives, ...) on NVIDIA / AMD / TPU / Trainium and -emits one JSON per run under `results///`. +Inference operator benchmarks: times one op at a time on NVIDIA and AMD through the serving framework's own layers (vLLM). -See `CLUSTERS.md` for how to reach each cluster and the per-host quirks. +| Op type | Schema | What it is | +| --- | --- | --- | +| `gemm` | `ops/gemm.py` | dense GEMM, quantized operands (`a` activation, `b` weight) | +| `moe` | `ops/moe.py` | one MoE layer, router GEMM to combined output | +| `mla`, `mla_dsa`, `dsv4_attn`, `gqa`, `qsa`, `gdn`, `kda` | `ops/attention.py` | whole attention modules, one op type each | -## Running on a cluster +- Split over devices: each case's `parallel` arg (`{"tp", "dp", "ep", "dcp"}`), defined in `core/parallel.py`. +- Testlists: `testlists/*.json`; every entry lists its `sources` (`//`). Dispatch `attn_*` testlists apart from gemm / moe ones (each builds its own vLLM engine). -`scripts/submit_run.py ` fans out one sbatch job per -`(container_image, world_size)` pair to your local SLURM. Each job runs -`python -m operatorx` inside the container. +## Running -The examples assume the repository is checked out at `$HOME/inferencex`. - -### b200 (DGX-style, 8x B200 SXM) - -```bash -ssh tailscale-b200 -cd "$HOME/inferencex/operatorx" -python3 scripts/submit_run.py nvidia -``` - -Defaults that apply: `OPERATORX_CLUSTER=b200_dgx_8x`, -`OPERATORX_PARTITION=gpu-2`, `OPERATORX_SQUASH_DIR=/home/sa-shared/containers`. - -### b300 (HGX-style, 8x B300) - -The b300 cluster needs a non-default partition + account + qos, has its own -squash dir, and DeepEP has known IBGDA issues here so we exclude it. -This example uses container images stored in `$HOME/containers`. +Dispatch the **OperatorX Sweep** workflow ([CI.md](CI.md)). The same planner runs by hand on a login node: ```bash -ssh tailscale-b300 -cd "$HOME/inferencex/operatorx" -OPERATORX_CLUSTER=b300_hgx_8x \ -OPERATORX_PARTITION=batch_1 \ -OPERATORX_ACCOUNT=benchmark \ -OPERATORX_SQUASH_DIR="$HOME/containers" \ -OPERATORX_BACKENDS=torch,deepgemm,flashinfer,sglang \ -python3 scripts/submit_run.py nvidia +PYTHONPATH=.:inferencex-e2e python3 -m operatorx.ci plan --platform-config operatorx/platforms.json \ + --runner cluster:h200-dgxc --backends vllm --testlists gemm_parallel --world-sizes 1,2,4,8 \ + --chunk-size 500 --mode timing --run-id local --attempt 1 --source-sha "$(git rev-parse HEAD)" \ + --out manifest.json ``` -### TPU / Trainium - -These hosts have no SLURM, so `submit_run.py` (which submits `sbatch` jobs) -does not apply. Run the benchmark directly on the VM or instance: - -```bash -OPERATORX_CLUSTER=v6e_4x python -m operatorx # TPU (default tpu cluster) -OPERATORX_CLUSTER=trn3_16x python -m operatorx # Trainium (default trainium cluster) -``` - -The TPU `maxtext` backend depends on Google's MaxText library. Install it -once per TPU VM (the `jax` backend works without it): - -```bash -git clone https://github.com/AI-Hypercomputer/maxtext ~/maxtext -pip install -e ~/maxtext -``` - -If MaxText isn't installed, `moe_forward` on TPU emits `unsupported` rows -rather than running our old single-device dense fallback. - -## Env vars honored by `submit_run.py` - -| Var | Default | Notes | -|-----|---------|-------| -| `OPERATORX_CLUSTER` | per-platform (see script) | Cluster id used to route the runner (see `operatorx.clusters.CLUSTER_PLATFORMS`). | -| `OPERATORX_PARTITION` | `gpu-2` | SLURM partition. | -| `OPERATORX_ACCOUNT` | (omitted) | SLURM `--account`. | -| `OPERATORX_QOS` | (omitted) | SLURM `--qos`. | -| `OPERATORX_SQUASH_DIR` | `/home/sa-shared/containers` | Where `.sqsh` lives. | -| `OPERATORX_BACKENDS` | all backends for the platform | CSV allowlist. | -| `OPERATORX_JOB_NAME` | `benchmark` | SLURM job name. Set a recognizable name for your runs (see `CLUSTERS.md`). | - -`WORLD_SIZES` in the script is `[1, 2, 4, 8]` and supports single-node runs only. -Values above 8 are disabled because multi-node NCCL IB bring-up currently hangs on b200/b300. -`MASTER_ADDR` is derived by parsing `SLURM_NODELIST`. +Results: one JSON per run under `results///`; in CI, the `operatorx-shard-*` artifacts, then ingested into the OperatorX database. ## Adding a backend / op -- Backend impl: `operatorx/runners//backends/.py`, exports an - `IMPLS = [BackendImpl(...)]` list. -- Op spec: `operatorx/ops/.py`, calls `register(OpSpec(..., flops=, bytes=))`. -- Add the container image to `containers.toml`, then on each cluster run - `python3 scripts/pull_containers.py nvidia` to enroot-import it into - `$OPERATORX_SQUASH_DIR`. -- Add shapes to `testlists/.json`. +- Backend: `runners//backends/.py` exporting `IMPLS = [BackendImpl(...)]`; its image in `containers.toml`. +- Op: `ops/.py` calling `register(OpSpec(type=..., arg_schema=..., parallel_axes=...))`. +- Shapes: `testlists/.json`. diff --git a/operatorx/ci.py b/operatorx/ci.py index 30b42e3e89..ff044dde39 100644 --- a/operatorx/ci.py +++ b/operatorx/ci.py @@ -20,31 +20,135 @@ from pathlib import Path ROOT = Path(__file__).resolve().parent -POOLS = { - "h100-dgxc": "h100_dgxc_8x", - "h200-dgxc": "h200_dgxc_8x", - "b200-nscale": "b200_nscale_8x", - "b300": "b300_dsxe_8x", - "gb200": "gb200_nvl72_4x", - "gb300": "gb300_nvl72_4x", - "mi300x": "mi300x_amds_8x", - "mi325x": "mi325x_amds_8x", - "mi355x": "mi355x_8x", +# runner label (as InferenceX's workflows request) -> (Slurm cluster id, CollectiveX +# platform profile in configs/platform_config.json) +RUNNERS = { + "cluster:h100-dsxe": ("h100_dgxc_8x", "h100-dgxc"), + "cluster:h200-dgxc": ("h200_dgxc_8x", "h200-dgxc"), + "cluster:b200-nscale": ("b200_nscale_8x", "b200-nscale"), + "cluster:b300-dsxe": ("b300_dsxe_8x", "b300"), + "cluster:gb200-nv": ("gb200_nvl72_4x", "gb200"), + "cluster:gb300-nv": ("gb300_nvl72_4x", "gb300"), + "cluster:mi300x-amd": ("mi300x_amds_8x", "mi300x"), + "cluster:mi325x-amds": ("mi325x_amds_8x", "mi325x"), + "cluster:mi355x-amds": ("mi355x_8x", "mi355x"), +} +MODES = ("timing", "counters") +# per-kernel hardware counters; a counters run's latencies are perturbed by the profiler +NCU_METRICS = ",".join(( + "gpu__time_duration.sum", "sm__cycles_elapsed.avg.per_second", + "gpc__cycles_elapsed.avg.per_second", "dram__cycles_elapsed.avg.per_second", + "sm__throughput.avg.pct_of_peak_sustained_elapsed", + "dram__throughput.avg.pct_of_peak_sustained_elapsed", + "sm__pipe_tensor_cycles_active.avg.pct_of_peak_sustained_elapsed", + "sm__warps_active.avg.pct_of_peak_sustained_active", + "sm__cycles_active.sum", "sm__cycles_active.avg", "sm__cycles_active.min", + "sm__cycles_active.max", "dram__bytes_read.sum", "dram__bytes_read.avg", + "dram__bytes_read.min", "dram__bytes_read.max", "dram__bytes_write.sum", + "lts__t_sectors.sum", "lts__t_sectors.avg", "lts__t_sectors.min", "lts__t_sectors.max", + "lts__t_sector_hit_rate.pct", "lts__t_sectors_lookup_hit.sum", + "lts__t_sectors_lookup_miss.sum", "lts__t_sectors_op_read.sum", + "lts__t_sectors_op_write.sum", "lts__t_sectors_op_atom.sum", "lts__t_sectors_op_red.sum", + "lts__t_sectors_aperture_sysmem_op_read.sum", "lts__t_sectors_aperture_sysmem_op_write.sum", + "l1tex__t_sector_hit_rate.pct", "l1tex__t_sectors_pipe_lsu_mem_global_op_ld.sum", + "l1tex__t_sectors_pipe_lsu_mem_global_op_st.sum", + "l1tex__t_sectors_pipe_lsu_mem_local_op_ld.sum", + "l1tex__t_sectors_pipe_lsu_mem_local_op_st.sum", + "l1tex__data_pipe_lsu_wavefronts_mem_shared_op_ld.sum", + "l1tex__data_pipe_lsu_wavefronts_mem_shared_op_st.sum", +)) +# host Nsight Compute: real installs (/nsight-compute[-]/ncu), then PATH +NCU_SEARCH = ( + "ls -d /opt/nvidia/nsight-compute/*/ncu /usr/local/cuda*/nsight-compute*/ncu " + "/opt/nvidia/nsight-compute*/ncu $(command -v ncu) 2>/dev/null | xargs -r readlink -f | sort -u" +) + + +def pick_ncu(paths: list[str]) -> Path | None: + """Newest real Nsight Compute install; a PATH wrapper (e.g. cuda/bin/ncu) only as a fallback.""" + def version(p: Path) -> tuple: + name = p.parent.name.removeprefix("nsight-compute").lstrip("-") + return tuple(int(x) for x in re.findall(r"\d+", name)) + real = [Path(p) for p in paths if "nsight-compute" in Path(p).parent.as_posix()] + if real: + return max(real, key=version) + return Path(paths[-1]) if paths else None +# rocprofv3 fits only a few counters per hardware pass: a counters run repeats per pass +ROCPROF_PASSES = { + "fetch": ["FETCH_SIZE"], + "hm": ["TCC_HIT_sum", "TCC_MISS_sum"], + "rdhist": ["TCC_EA0_RDREQ_128B_sum", "TCC_EA0_RDREQ_32B_sum", "TCC_EA0_RDREQ_64B_sum", + "TCC_EA0_RDREQ_sum"], + "sqcomp": ["SQ_BUSY_CYCLES", "SQ_INSTS_MFMA", "SQ_INSTS_VALU", "SQ_VALU_MFMA_BUSY_CYCLES", + "SQ_WAVES", "SQ_WAVE_CYCLES"], + "sqmem": ["SQ_INSTS_LDS_LOAD", "SQ_INSTS_LDS_STORE", "SQ_INSTS_SMEM", "SQ_INSTS_VMEM_RD", + "SQ_INSTS_VMEM_WR", "SQ_LDS_BANK_CONFLICT"], + "stalls": ["TCC_BUBBLE_sum", "TCC_EA0_RDREQ_DRAM_CREDIT_STALL_sum", + "TCC_EA0_WRREQ_DRAM_CREDIT_STALL_sum"], + "tccops": ["TCC_ATOMIC_sum", "TCC_READ_sum", "TCC_REQ_sum", "TCC_WRITE_sum"], + "tcp": ["TCP_TAGRAM0_REQ_sum", "TCP_TOTAL_CACHE_ACCESSES_sum", "TCP_TOTAL_READ_sum", + "TCP_TOTAL_WRITE_sum"], + "util": ["LdsUtil", "MemUnitStalled", "SALUBusy", "VALUBusy", "VALUUtilization"], + "util2": ["GPU_UTIL", "SQC_DCACHE_HITS", "SQC_ICACHE_HITS", "TD_TD_BUSY_sum"], + "wom": ["GRBM_GUI_ACTIVE", "MfmaUtil", "OccupancyPercent", "WRITE_SIZE"], + "wrdst": ["TCC_EA0_RDREQ_DRAM_sum", "TCC_EA0_WRREQ_64B_sum", "TCC_EA0_WRREQ_DRAM_sum", + "TCC_EA0_WRREQ_sum"], } -AMD_POOLS = {"mi300x", "mi325x", "mi355x"} def load_platforms(path: Path) -> dict: + """Per runner label: its CollectiveX profile overlaid with this file's entry.""" document = json.loads(path.read_text()) - platforms = {} + base = {} if "base" in document: - base = path.parent / document["base"] - platforms.update(json.loads(base.read_text())["platforms"]) - for pool, hardware in document["platforms"].items(): - platforms[pool] = {**platforms.get(pool, {}), **hardware} + base = json.loads((path.parent / document["base"]).read_text())["platforms"] + platforms = {} + for runner, (_, profile) in RUNNERS.items(): + own = document["platforms"].get(runner, {}) + merged = {**base.get(profile, {}), **own} + if "operator" in own: # add to CollectiveX's operator settings, not replace them + merged["operator"] = {**base.get(profile, {}).get("operator", {}), **own["operator"]} + if merged: + platforms[runner] = merged return platforms +# CollectiveX's network profile scrub (runtime/common.sh collx_apply_network_profile). +_NETWORK_ENV = ("NCCL_NET", "NCCL_SOCKET_IFNAME", "NCCL_IB_", "NVSHMEM_", "MORI_RDMA_", "UCCL_") + + +def stage_model_config(checkpoint: str, root: Path) -> None: + """The checkpoint's config.json, without weights.""" + import urllib.request + + dest = root / checkpoint / "config.json" + dest.parent.mkdir(parents=True, exist_ok=True) + try: + with urllib.request.urlopen(f"https://huggingface.co/{checkpoint}/raw/main/config.json", + timeout=60) as r: + dest.write_bytes(r.read()) + except OSError as e: + print(f"[operatorx] no config.json for {checkpoint}: {e}", flush=True) + + +def _split_axes(op_type: str) -> tuple[str, ...]: + import operatorx.ops # noqa: F401 registers the op specs + from operatorx.core import op_registry, parallel + try: + return op_registry.get(op_type).parallel_axes or ("tp",) + except KeyError: + return parallel.AXES + + +def family(name: str) -> str: + """Hardware family of a runner label: 'cluster:mi355x-amds' -> 'mi355x'.""" + return name.removeprefix("cluster:").split("-")[0] + + +def is_amd(runner: str) -> bool: + return family(runner).startswith("mi") + + def write_json(path: Path, value: object) -> None: path.parent.mkdir(parents=True, exist_ok=True) temporary = path.with_suffix(path.suffix + ".tmp") @@ -53,91 +157,111 @@ def write_json(path: Path, value: object) -> None: def plan( - pool: str, + runner: str, backends: list[str], testlists: dict[str, list[dict]], images: dict[str, dict], world_sizes: list[int], chunk_size: int, platforms: dict[str, dict], + mode: str = "timing", + find_recipe=None, + default_image=None, ) -> dict: - if pool not in POOLS: - raise ValueError(f"unsupported pool: {pool}") - hardware = platforms[pool] + """Shards for a selection, one per (parallel split, InferenceX recipe). A case whose + source checkpoint InferenceX serves on this hardware runs under that recipe + (find_recipe = operatorx.recipes.find): its image, launch env and server arguments.""" + from operatorx.core import parallel + from operatorx.recipes import FRAMEWORKS, checkpoints + if runner not in RUNNERS: + raise ValueError(f"unsupported runner: {runner}") + if mode not in MODES: + raise ValueError(f"mode must be one of {', '.join(MODES)}") + hardware = platforms[runner] gpus = hardware["gpus_per_node"] image_platform = hardware["image_platform"] if type(gpus) is not int or gpus not in (4, 8): - raise ValueError("pool must supply four or eight GPUs per physical node") + raise ValueError("runner must supply four or eight GPUs per physical node") if image_platform not in ("linux/amd64", "linux/arm64"): raise ValueError("unsupported image platform") if not backends or set(backends) - images.keys(): raise ValueError("select at least one registered backend for this GPU platform") - if pool in AMD_POOLS and ( - set(backends) - {"torch", "aiter", "vllm"} - or world_sizes != [1] + if is_amd(runner) and ( + set(backends) - {"torch", "vllm"} or any( - shape["type"] not in {"gemm", "attention_mha", "attention_mla", "moe_gemm"} + shape["type"] not in {"gemm", "moe", "mla", "mla_dsa", "dsv4_attn", "gqa", "qsa", "gdn", "kda"} for shapes in testlists.values() for shape in shapes ) ): - raise ValueError( - "AMD CI supports single-GPU torch GEMM, torch/aiter attention and vllm MoE" - ) + raise ValueError("AMD CI supports the torch/vllm GEMM, MoE and attention ops") if not world_sizes or set(world_sizes) - {1, 2, 4, 8}: raise ValueError("world sizes must be selected from 1,2,4,8 (single node)") if any(ws > gpus for ws in world_sizes): - raise ValueError(f"world size exceeds the pool's {gpus}-GPU physical node") + raise ValueError(f"world size exceeds the runner's {gpus}-GPU physical node") + if "torch" in backends and world_sizes != [1]: + raise ValueError("the torch backend runs single-device GEMM only") if not 1 <= chunk_size <= 500: raise ValueError("chunk size must be between 1 and 500") groups = defaultdict(list) + recipes_seen: dict[str, dict] = {} excluded = 0 for name, shapes in sorted(testlists.items()): for shape in shapes: - args = shape["args"] - ws = args.get("world_size", 1) - if type(ws) is not int or ws < 1: - raise ValueError("world_size must be a positive integer") - if ws not in world_sizes: + split = parallel.normalize(shape["args"].get("parallel")) + if parallel.world_size(split) not in world_sizes: excluded += 1 continue - moe = ( - tuple( - args.get(key, 1) - for key in ( - "expert_parallel_size", - "routed_tensor_parallel_size", - "shared_tensor_parallel_size", - ) - ) - if shape["type"] == "moe_forward" - else () - ) - groups[(ws, moe)].append({"testlist": name, "shape": shape}) + recipe = next((r for b in backends if b in FRAMEWORKS and find_recipe + for r in [find_recipe(b, family(runner), checkpoints(shape.get("sources")), + split, _split_axes(shape["type"]))] if r), None) + if recipe: + recipes_seen[recipe["recipe"]] = recipe + # attention builds its own engine; never in a process with gemm/moe + state = "layer" if shape["type"] in ("gemm", "moe") else "engine" + groups[(json.dumps(split, sort_keys=True), recipe["recipe"] if recipe else "", state)].append( + {"testlist": name, "shape": shape}) image_groups = defaultdict(list) for backend in sorted(set(backends)): - image_groups[images[backend]["image"]].append(backend) + # the image InferenceX serves the backend with on this hardware, else operatorx's own + image = default_image and default_image(backend, family(runner)) + image_groups[image or images[backend]["image"]].append(backend) cells = [] - for image, selected in sorted(image_groups.items()): - for (ws, moe), cases in sorted(groups.items()): + shards = [] + for (split, recipe, _state), cases in sorted(groups.items()): + split = json.loads(split) + ws = parallel.world_size(split) + rest = image_groups + if recipe: + r = recipes_seen[recipe] + framework = next(b for b in backends if b in FRAMEWORKS) + shards.append((r["image"], [framework], ws, cases, + {"parallel": split, "recipe": recipe, "recipe_match": r["match"], + "checkpoint": r["checkpoint"], "env": r["env"], "engine_args": r["engine_args"]})) + # the other backends run the same cases on their own images + rest = {i: [b for b in bs if b != framework] for i, bs in image_groups.items()} + shards += [(image, selected, ws, cases, {"parallel": split}) + for image, selected in sorted(rest.items()) if selected] + for image, selected, ws, cases, extra in shards: for offset in range(0, len(cases), chunk_size): cell = { - "pool": pool, - "cluster": POOLS[pool], + "runner": runner, + "cluster": RUNNERS[runner][0], "nodes": 1, "gpus_per_node": gpus, "image_platform": image_platform, "world_size": ws, - "moe": moe, "image": image, + "mode": mode, "backends": selected, "offset": offset, "cases": cases[offset : offset + chunk_size], + **extra, } identity = hashlib.sha256( json.dumps(cell, sort_keys=True).encode() ).hexdigest()[:16] - cells.append({"id": f"{pool}-{identity}", **cell}) + cells.append({"id": f"{runner.removeprefix('cluster:')}-{identity}", **cell}) if not cells: raise ValueError("selection contains no runnable shapes") if len(cells) > 256: @@ -213,17 +337,22 @@ def cleanup(root: Path, timeout_seconds: int) -> None: ) +def _pinned_digest(image: str) -> str: + """The digest of a digest-pinned reference (image:tag@sha256:...), else empty.""" + return image.rpartition("@")[2] if "@sha256:" in image else "" + + def image_key(image: str, digest: str, image_platform: str) -> str: - # Preserve the already-qualified amd64 cache while isolating Arm imports. + # keep the existing amd64 cache keys; Arm imports get their own suffix = "" if image_platform == "linux/amd64" else f":{image_platform}" return hashlib.sha256((image + digest + suffix).encode()).hexdigest() -def shared_base(profile: dict, pool: str) -> Path: - """Resolve the pool's configured/shared account storage, never temporary HOME.""" +def shared_base(profile: dict, runner: str) -> Path: + """The runner's configured/shared account storage, never a temporary HOME.""" if profile.get("stage_dir"): roots = [Path(profile["stage_dir"])] - elif pool in AMD_POOLS: + elif is_amd(runner): runner_temp = Path(os.environ["RUNNER_TEMP"]) if ( runner_temp.parts[-2:] != ("_work", "_temp") @@ -231,9 +360,9 @@ def shared_base(profile: dict, pool: str) -> Path: ): raise ValueError("AMD staging requires the shared runner _work/_temp path") roots = [runner_temp.parent.parent] - elif pool == "b300": - # CollectiveX uses the compute-visible account home on this pool. - # The shared squash parent is not writable by the GHA service account. + elif family(runner) == "b300": + # compute-visible account home, as CollectiveX; the shared squash parent isn't + # writable by the GHA service account roots = [Path(pwd.getpwuid(os.getuid()).pw_dir)] elif profile.get("squash_dir"): roots = [Path(profile["squash_dir"]).parent] @@ -242,11 +371,11 @@ def shared_base(profile: dict, pool: str) -> Path: for root in roots: if root.is_dir() and os.access(root, os.W_OK | os.X_OK): return root / f".operatorx-{os.getuid()}" - raise ValueError("no writable shared storage root configured for this pool") + raise ValueError("no writable shared storage root configured for this runner") def import_image(args) -> None: - # Runs on the configured import host with a compute-visible cache and lock. + # runs on the import host, with a compute-visible cache and lock import fcntl image, digest = args.image, args.digest @@ -271,10 +400,12 @@ def import_image(args) -> None: temporary = squash.with_suffix(".partial") temporary.unlink(missing_ok=True) try: - host, repository, tag = probe_module().registry_reference(image) - uri = f"docker://{host}#{repository}:{tag}" - # B300 login/compute homes are node-local; every importer gets private - # temporary paths. Preserve an explicitly configured shared cache. + # the planner's InferenceX import URI (by digest for a pinned image), else by tag + uri = getattr(args, "uri", "") + if not uri: + host, repository, tag = probe_module().registry_reference(image) + uri = f"docker://{host}#{repository}:{tag}" + # B300 homes are node-local: private temp paths unless a shared cache is configured with tempfile.TemporaryDirectory(prefix="operatorx-enroot-") as scratch: env = dict(os.environ) env["TMPDIR"] = scratch @@ -293,9 +424,8 @@ def import_image(args) -> None: env=env, ) subprocess.run(["unsquashfs", "-s", str(temporary)], check=True) - # The importer uses the tag, just like CollectiveX. Refuse a tag that moved - # between planning and import rather than mislabelling the measurement. - if probe_module().resolve_image_digest(image) != digest: + # imported by tag: refuse a tag that moved since planning + if not _pinned_digest(image) and probe_module().resolve_image_digest(image) != digest: raise RuntimeError( "image tag moved or digest verification failed; dispatch again" ) @@ -328,18 +458,18 @@ def finalize(root: Path, cleanup_seconds: int) -> None: def recover( - artifacts: Path, run_id: str, pool: str, platform_config: Path, cleanup_seconds: int + artifacts: Path, run_id: str, runner: str, platform_config: Path, cleanup_seconds: int ) -> None: - profile = load_platforms(platform_config)[pool]["operator"] - base = shared_base(profile, pool).resolve() + profile = load_platforms(platform_config)[runner]["operator"] + base = shared_base(profile, runner).resolve() recovered = 0 for execution in artifacts.rglob("execution.json"): data = json.loads(execution.read_text()) - if data["run_id"] != run_id or data["cell"]["pool"] != pool: - raise ValueError("recovery artifact does not match requested run/pool") + if data["run_id"] != run_id or data["cell"]["runner"] != runner: + raise ValueError("recovery artifact does not match requested run/runner") stage = Path(data["stage"]) if stage.parent.resolve() != base: - raise ValueError("recovery stage does not belong to this pool/user") + raise ValueError("recovery stage does not belong to this runner/user") finalize(execution.parent, cleanup_seconds) recovered += 1 if not recovered: @@ -355,7 +485,7 @@ def execute(args) -> None: raise ValueError("manifest source/run mismatch") cells = manifest["include"] cell = next(c for c in cells if c["id"] == args.shard) - hardware = load_platforms(args.platform_config)[cell["pool"]] + hardware = load_platforms(args.platform_config)[cell["runner"]] profile = hardware["operator"] gpus = hardware["gpus_per_node"] image_platform = hardware["image_platform"] @@ -364,8 +494,8 @@ def execute(args) -> None: or cell["image_platform"] != image_platform or cell["world_size"] > gpus ): - raise ValueError("manifest hardware differs from the selected pool") - base = shared_base(profile, cell["pool"]) + raise ValueError("manifest hardware differs from the selected runner") + base = shared_base(profile, cell["runner"]) base.mkdir(mode=0o700, exist_ok=True) if ( base.is_symlink() @@ -399,10 +529,14 @@ def interrupted(signum, frame): stage / "source/operatorx", ignore=shutil.ignore_patterns("__pycache__", "results", ".venv", "tests"), ) - shutil.copytree( - ROOT.parent / "collectivex/runtime", - stage / "source/collectivex/runtime", - ) + for part in ("runtime", "bench"): # probes; the EP harness's cross-rank timing + shutil.copytree( + ROOT.parent / "collectivex" / part, + stage / "source/collectivex" / part, + ignore=shutil.ignore_patterns("__pycache__"), + ) + if cell.get("checkpoint"): + stage_model_config(cell["checkpoint"], stage / "models") for name in sorted({c["testlist"] for c in cell["cases"]}): write_json( stage / "testlists" / f"{name}.json", @@ -427,11 +561,12 @@ def interrupted(signum, frame): ): if profile.get(field): allocation.append(f"--{flag}={profile[field]}") - if cell["pool"] in ("b200-nscale", "b300", "h100-dgxc", "gb200", "gb300"): + hw = family(cell["runner"]) + if hw in ("b200", "b300", "h100", "gb200", "gb300"): allocation.append("--mem=0") - if cell["pool"] in ("gb200", "gb300"): + if hw in ("gb200", "gb300"): allocation.append("--cpus-per-task=35") - if cell["pool"] in AMD_POOLS: + if is_amd(cell["runner"]): allocation.append(f"--cpus-per-task={profile['cpus_per_node'] // gpus}") command(allocation, root / "allocation.log") jobs = allocation_ids(root) @@ -455,18 +590,18 @@ def interrupted(signum, frame): cell["image"], "--digest", cell["digest"], + "--uri", + cell.get("import_uri", ""), "--image-platform", image_platform, ] - # Import on the allocated architecture, including B300: its submit host - # lacks PyTorch extraction space, as the inference launcher notes. + # import on the allocated node (B300's submit host lacks extraction space) import_env = dict(os.environ) if profile.get("enroot_cache_path"): import_env["ENROOT_CACHE_PATH"] = profile["enroot_cache_path"] command(import_command, root / "import.log", env=import_env) key = image_key(cell["image"], cell["digest"], image_platform) env = dict(os.environ) - env.pop("OPERATORX_MOE_PARALLELISM", None) env.update( OPERATORX_CLUSTER=cell["cluster"], OPERATORX_CONTAINER_IMAGE=cell["image"], @@ -478,19 +613,41 @@ def interrupted(signum, frame): OPERATORX_GITHUB_RUN_ATTEMPT=args.attempt, OPERATORX_SHARD_ID=cell["id"], OPERATORX_BACKENDS=",".join(cell["backends"]), + OPERATORX_MODE=cell.get("mode", "timing"), + OPERATORX_RECIPE=cell.get("recipe", ""), + OPERATORX_ENGINE_ARGS=json.dumps(cell.get("engine_args") or {}), OPERATORX_TESTLISTS=",".join( sorted({c["testlist"] for c in cell["cases"]}) ), PYTHONPATH="/opx/source", PYTHONDONTWRITEBYTECODE="1", WORLD_SIZE=str(cell["world_size"]), + OPERATORX_PARALLEL=json.dumps(cell.get("parallel") or {}), MASTER_ADDR="127.0.0.1", - MASTER_PORT="29500", + MASTER_PORT=str(29500 + int(hashlib.sha256(f"{job}:{cell['id']}".encode()).hexdigest(), 16) % 2000), ) - if cell["moe"]: - env["OPERATORX_MOE_PARALLELISM"] = ":".join(map(str, cell["moe"])) + if (stage / "models" / cell.get("checkpoint", "-") / "config.json").is_file(): + env["OPERATORX_MODEL_CONFIG"] = f"/opx/models/{cell['checkpoint']}" + # --export=ALL forwards the runner's network profile; single node needs none + for key in [k for k in env if k.startswith(_NETWORK_ENV)]: + del env[key] + if not is_amd(cell["runner"]): + env["NCCL_CUMEM_ENABLE"] = "1" + env.update(cell.get("env", {})) # the InferenceX recipe's launch env mounts = f"{stage}:/opx" - if cell["pool"] in ("mi300x", "mi325x"): + # NVIDIA images carry no Nsight Compute; counters runs mount the node's newest. + if cell.get("mode") == "counters" and not is_amd(cell["runner"]): + found = subprocess.run( + ["srun", f"--jobid={job}", "--nodes=1", "--ntasks=1", "bash", "-c", NCU_SEARCH], + capture_output=True, text=True, timeout=120, + ).stdout.split() + (root / "ncu.log").write_text("\n".join(found) + "\n") + ncu = pick_ncu(found) + if ncu is not None: + # ncu finds its install relative to itself: mount the whole tree in place + mounts += f",{ncu.parent.parent}:{ncu.parent.parent}" + env["OPERATORX_NCU"] = str(ncu) + if hw in ("mi300x", "mi325x"): mounts += ",/dev/kfd:/dev/kfd,/dev/dri:/dev/dri" run = [ "srun", @@ -508,17 +665,19 @@ def interrupted(signum, frame): "--container-writable", "--export=ALL", ] - if cell["pool"] in {"h200-dgxc", "b300", "h100-dgxc", "gb200", "gb300"} | AMD_POOLS: + if hw in ("h200", "b300", "h100", "gb200", "gb300") or is_amd(cell["runner"]): run.append("--container-remap-root") - # The AWS dsxe pools (b300, h100-dgxc) share one Slurm and pyxis setup. - if cell["pool"] in ("b300", "h100-dgxc"): + # the AWS dsxe pools (b300, h100) share one Slurm and pyxis setup + if hw in ("b300", "h100"): run.append("--mpi=none") - # The Python entrypoint preserves the allocated GPU mask without a shell. - run += ["python3", "-m", "operatorx.ci", "rank"] + # by path: an image's own PYTHONPATH (ROCm images set one) replaces the host's + run += ["python3", "/opx/source/operatorx/ci.py", "rank"] + # bound a collective that hangs without failing + run = ["timeout", "-k", "30", str(max(300, args.time_minutes * 60 - 300)), *run] command(run, root / "benchmark.log", env=env) rc = 0 finally: - # Stop writers before collecting; failed cleanup retains the evidence. + # stop writers before collecting; a failed cleanup keeps the evidence for sig in (signal.SIGINT, signal.SIGTERM, signal.SIGHUP): signal.signal(sig, signal.SIG_IGN) finalize(root, args.cleanup_seconds) @@ -532,6 +691,9 @@ def rank() -> None: if not os.environ.get(key): raise ValueError(f"required rank input missing: {key}") os.environ["RANK"] = os.environ["SLURM_PROCID"] + source = "/opx/source" + paths = [x for x in os.environ.get("PYTHONPATH", "").split(":") if x and x != source] + os.environ["PYTHONPATH"] = ":".join([source, *paths]) os.environ["LOCAL_RANK"] = os.environ["SLURM_LOCALID"] cache = ( Path("/tmp") / f"operatorx-{os.environ['SLURM_JOB_ID']}-{os.environ['RANK']}" @@ -542,19 +704,51 @@ def rank() -> None: TRITON_CACHE_DIR=str(cache / "triton"), MPLCONFIGDIR=str(cache / "matplotlib"), ) - os.execv( + bench = [ sys.executable, - [ - sys.executable, - "-m", - "operatorx", - "--strict", - "--testlist-dir", - "/opx/testlists", - "--results-dir", - "/opx/results", - ], + "-m", + "operatorx", + "--strict", + "--testlist-dir", + "/opx/testlists", + "--results-dir", + "/opx/results", + ] + if os.environ.get("OPERATORX_MODE", "timing") != "counters": + os.execv(sys.executable, bench) + # one marked, profiled replay per op so counters join to ops; no timing warmups + os.environ.update( + OPERATORX_PROFILE="1", + OPERATORX_PROFILE_MARKERS="1", + OPERATORX_PROFILE_ITERS="1", + OPERATORX_WARMUP_MIN_S="0", + OPERATORX_FIRST_WARMUP_S="0", + OPERATORX_COOLDOWN_RATIO="0", + OPERATORX_THROTTLE_RETRIES="0", ) + out = Path("/opx/results/counters") + out.mkdir(parents=True, exist_ok=True) + rank_id = os.environ["RANK"] + rocprof = shutil.which("rocprofv3") + if rocprof: + for i, (tag, counters) in enumerate(ROCPROF_PASSES.items()): + argv = [rocprof, "--pmc", *counters, "--kernel-trace", "--marker-trace", + "--output-format", "csv", "-d", str(out / tag), "-o", f"rank{rank_id}", + "--", *bench] + if i: # results rows come from the first pass only + argv[-1] = str(cache / f"results-{tag}") + subprocess.run(argv, check=True) + return + ncu = os.environ.get("OPERATORX_NCU") or shutil.which("ncu") + if not ncu: + raise RuntimeError( + "counters mode needs ncu: none in the image, and the node has none under " + "/opt/nvidia/nsight-compute, /usr/local/cuda*/nsight-compute* or on PATH") + os.execv(ncu, [ + ncu, "--clock-control", "none", "--target-processes", "all", + "--nvtx", "--nvtx-include", "regex:opx[0-9]+/", "--metrics", NCU_METRICS, + "--csv", "--log-file", str(out / f"ncu-rank{rank_id}.csv"), *bench, + ]) def summarize(manifest: dict, artifacts: Path) -> dict: @@ -609,7 +803,7 @@ def main() -> None: sub = parser.add_subparsers(dest="command", required=True) p = sub.add_parser("plan") for name in ( - "pool", + "runner", "backends", "testlists", "world-sizes", @@ -619,6 +813,7 @@ def main() -> None: ): p.add_argument("--" + name, required=True) p.add_argument("--chunk-size", required=True, type=int) + p.add_argument("--mode", default="timing", choices=MODES) p.add_argument("--platform-config", required=True, type=Path) p.add_argument("--out", required=True, type=Path) p = sub.add_parser("execute") @@ -633,6 +828,7 @@ def main() -> None: p.add_argument("--cache", required=True, type=Path) p.add_argument("--image", required=True) p.add_argument("--digest", required=True) + p.add_argument("--uri", default="") p.add_argument( "--image-platform", required=True, choices=("linux/amd64", "linux/arm64") ) @@ -648,37 +844,63 @@ def main() -> None: p.add_argument("--cleanup-seconds", required=True, type=int) p.add_argument("--artifacts", required=True, type=Path) p.add_argument("--run-id", required=True) - p.add_argument("--pool", required=True, choices=tuple(POOLS)) + p.add_argument("--runner", required=True, choices=tuple(RUNNERS)) p.add_argument("--platform-config", required=True, type=Path) args = parser.parse_args() if args.command == "plan": import tomllib + from operatorx import recipes + names = args.testlists.split(",") if any(not re.fullmatch(r"[A-Za-z0-9_-]+", n) for n in names): raise ValueError("invalid testlist name") lists = { n: json.loads((ROOT / "testlists" / f"{n}.json").read_text()) for n in names } - vendor = "amd" if args.pool in AMD_POOLS else "nvidia" + vendor = "amd" if is_amd(args.runner) else "nvidia" images = tomllib.loads((ROOT / "containers.toml").read_text())[vendor] + # fail before any node is allocated for a backend with no module + missing = [ + b for b in args.backends.split(",") + if not (ROOT / "runners" / vendor / "backends" / f"{b}.py").is_file() + ] + if missing: + raise ValueError(f"no {vendor} backend module for: {', '.join(missing)}") result = plan( - args.pool, + args.runner, args.backends.split(","), lists, images, [int(w) for w in args.world_sizes.split(",")], args.chunk_size, load_platforms(args.platform_config), + args.mode, + find_recipe=recipes.find, + default_image=recipes.default_image, ) digests = {c["image"]: "" for c in result["include"]} for image in digests: - digest = probe_module().resolve_image_digest(image) - digests[image] = digest - if not digest: + digests[image] = _pinned_digest(image) or probe_module().resolve_image_digest(image) + for cell in result["include"]: + # an InferenceX image (a recipe's or the hardware's default) whose tag the registry + # has pruned: the backend's own image, with any recipe env; the manifest records it + fallback = images[cell["backends"][0]]["image"] + if not digests[cell["image"]] and cell["image"] != fallback: + cell["recipe_image_unavailable" if cell.get("recipe") else "image_unavailable"] = cell["image"] + cell["image"] = fallback + if not digests.get(cell["image"]): + digests[cell["image"]] = (_pinned_digest(cell["image"]) + or probe_module().resolve_image_digest(cell["image"])) + for image in {c["image"] for c in result["include"]}: + if not digests.get(image): raise RuntimeError(f"cannot resolve image digest: {image}") + from infx.launch.backends.slurm.squash import enroot_uri + for cell in result["include"]: cell["digest"] = digests[cell["image"]] + # InferenceX's import URI: a digest-pinned image imports by digest + cell["import_uri"] = enroot_uri(cell["image"]) cell["queue-token"] = hashlib.sha256( f"{args.run_id}:{args.attempt}:{cell['id']}".encode() ).hexdigest()[:32] @@ -688,7 +910,7 @@ def main() -> None: write_json(args.out, result) slim = { "include": [ - {k: c[k] for k in ("id", "pool", "nodes", "queue-token")} + {k: c[k] for k in ("id", "runner", "nodes", "queue-token")} for c in result["include"] ] } @@ -711,7 +933,7 @@ def main() -> None: recover( args.artifacts, args.run_id, - args.pool, + args.runner, args.platform_config, args.cleanup_seconds, ) diff --git a/operatorx/clusters.py b/operatorx/clusters.py index 9cb4880e98..17d87c2b86 100644 --- a/operatorx/clusters.py +++ b/operatorx/clusters.py @@ -1,9 +1,4 @@ -"""Minimal cluster routing table. - -Maps cluster id -> platform (for runner dispatch) and cluster id -> chip -(for legacy/grouped layouts). No peak-throughput or bandwidth info — that -lives in dashboard_build/hardware.py. -""" +"""Cluster id -> platform (runner dispatch) and cluster id -> chip.""" from __future__ import annotations @@ -17,16 +12,10 @@ "gb300_nvl72_4x": "nvidia", "b300_hgx_8x": "nvidia", "b200_nvl72": "nvidia", + "h200_hgx_8x": "nvidia", "mi355x_8x": "amd", "mi300x_amds_8x": "amd", "mi325x_amds_8x": "amd", - "v6e_1x": "tpu", - "v6e_4x": "tpu", - "v6e_pod": "tpu", - "v7x_4x": "tpu", - "trn3_1x": "trainium", - "trn3_8x": "trainium", - "trn3_16x": "trainium", } CLUSTER_CHIPS: dict[str, str] = { @@ -39,14 +28,8 @@ "gb300_nvl72_4x": "gb300", "b300_hgx_8x": "b300", "b200_nvl72": "b200", + "h200_hgx_8x": "h200", "mi355x_8x": "mi355x", "mi300x_amds_8x": "mi300x", "mi325x_amds_8x": "mi325x", - "v6e_1x": "v6e", - "v6e_4x": "v6e", - "v6e_pod": "v6e", - "v7x_4x": "v7x", - "trn3_1x": "trn3", - "trn3_8x": "trn3", - "trn3_16x": "trn3", } diff --git a/operatorx/containers.toml b/operatorx/containers.toml index d39245b493..c391949141 100644 --- a/operatorx/containers.toml +++ b/operatorx/containers.toml @@ -1,38 +1,10 @@ -[nvidia.torch] -image = "nvcr.io/nvidia/pytorch:26.03-py3" - +# A backend's image where InferenceX serves no configuration with that framework on the +# runner's hardware; otherwise cases take InferenceX's own image (operatorx.recipes). [nvidia.vllm] -image = "vllm/vllm-openai:v0.19.0" - -[nvidia.flashinfer] -image = "lmsysorg/sglang:latest" - -[nvidia.deepgemm] -image = "lmsysorg/sglang:latest" - -[nvidia.sglang] -image = "lmsysorg/sglang:latest" - -[nvidia.deepep] -image = "lmsysorg/sglang:latest" - -[nvidia.flashinfer_comm] -image = "lmsysorg/sglang:latest" - -[nvidia.sglang_comm] -image = "lmsysorg/sglang:latest" +image = "vllm/vllm-openai:nightly-39983a5e7a5a575a7829f5d5ada50a33d9337ec1@sha256:9c482f0f927b7064777c7baf0faafe5721804a9eb8b9bfea9d939cfbc3138348" [amd.torch] -image = "vllm/vllm-openai-rocm:v0.19.0" +image = "vllm/vllm-openai-rocm:v0.31.0@sha256:749f6f3f944f12af49966ac523c1f8573e4b229594b541954bd5f879b4496b1f" [amd.vllm] -image = "vllm/vllm-openai-rocm:v0.19.0" - -[amd.aiter] -image = "vllm/vllm-openai-rocm:v0.19.0" - -[tpu.jax] -image = "us-central1-docker.pkg.dev/jax-image-build/jax-tpu/jax-tpu:latest" - -[trainium.neuronx] -image = "public.ecr.aws/neuron/pytorch-training-neuronx:2.5.1-neuronx-py310-sdk2.21.0-ubuntu22.04" +image = "vllm/vllm-openai-rocm:v0.31.0@sha256:749f6f3f944f12af49966ac523c1f8573e4b229594b541954bd5f879b4496b1f" diff --git a/operatorx/core/backend.py b/operatorx/core/backend.py index 13a4265a7c..452b2b82d2 100644 --- a/operatorx/core/backend.py +++ b/operatorx/core/backend.py @@ -9,21 +9,19 @@ @dataclass(frozen=True) class BackendImpl: + """prepare(op) -> ctx; kernel(ctx) runs the op once. + + launcher(ctx), if set, returns (callable, cuda_graph): what the runner times instead of + kernel(ctx) (how the framework would execute the op) and whether it is a graph replay.""" op_type: str prepare: Callable[[Op], Any] kernel: Callable[[Any], None] + launcher: Callable[[Any], tuple[Callable[[], None], bool]] | None = None def lookup_versions(*pkg_names: str) -> dict[str, str]: - """Resolve installed versions for a set of package names. - - Tries ``importlib.metadata.version`` first (catches pip-installed - packages), then falls back to importing the module and reading - ``__version__`` (catches PYTHONPATH-installed checkouts like MaxText). - Missing packages are silently skipped. - - Used by each backend's ``versions()`` to declare what it depends on. - """ + """Installed versions: package metadata, else the module's __version__ (PYTHONPATH + checkouts); missing packages are skipped.""" import importlib as _importlib out: dict[str, str] = {} diff --git a/operatorx/core/errors.py b/operatorx/core/errors.py index 4aac89a7a6..cce52ce653 100644 --- a/operatorx/core/errors.py +++ b/operatorx/core/errors.py @@ -1,7 +1,2 @@ class UnsupportedOpError(Exception): - """Backend cannot execute this op (wrong dtype, wrong category, kernel missing, ...). - - Distinct from a runtime crash: the backend simply doesn't claim support for this - (op_type, args) combination. Smoke tests record this as ``status="unsupported"`` - rather than ``"error"``. - """ + """The backend doesn't support this (op_type, args); recorded as status="unsupported", not "error".""" diff --git a/operatorx/core/op.py b/operatorx/core/op.py index a4239b89fd..48f32f82ff 100644 --- a/operatorx/core/op.py +++ b/operatorx/core/op.py @@ -1,5 +1,6 @@ from __future__ import annotations +import json from dataclasses import dataclass from types import MappingProxyType from typing import Any, Mapping @@ -10,13 +11,13 @@ class Op: type: str args: Mapping[str, Any] backend: str - # Optional preset name for shapes that match a real model - # (e.g. "dsv3", "llama3-8b"). Equality/hash ignore this — same - # (type, args, backend) is the same op regardless of label. - name: str | None = None + # one "//" per layer that runs this op (e.g. + # "deepseek-ai/DeepSeek-V4-Pro/q_a_proj"); ignored by equality/hash + sources: tuple[str, ...] = () def __post_init__(self) -> None: object.__setattr__(self, "args", MappingProxyType(dict(self.args))) + object.__setattr__(self, "sources", tuple(self.sources)) def __eq__(self, other: object) -> bool: if not isinstance(other, Op): @@ -28,7 +29,8 @@ def __eq__(self, other: object) -> bool: ) def __hash__(self) -> int: - return hash((self.type, tuple(sorted(self.args.items())), self.backend)) + # args may nest (e.g. gemm operand descriptors); canonical JSON is hashable. + return hash((self.type, json.dumps(dict(self.args), sort_keys=True), self.backend)) @dataclass(frozen=True) @@ -36,3 +38,5 @@ class OpSpec: type: str arg_schema: type description: str = "" + # axes (core/parallel.py) the "parallel" arg may use; empty: one device only + parallel_axes: tuple[str, ...] = () diff --git a/operatorx/core/op_registry.py b/operatorx/core/op_registry.py index ccf21915bb..73751c4e42 100644 --- a/operatorx/core/op_registry.py +++ b/operatorx/core/op_registry.py @@ -2,6 +2,7 @@ from typing import Sequence +from operatorx.core import parallel from operatorx.core.op import Op, OpSpec _REGISTRY: dict[str, OpSpec] = {} @@ -24,9 +25,12 @@ def all_ops() -> Sequence[OpSpec]: def validate(op: Op) -> Op: + """Check args against the schema; "parallel" is checked against the op type's axes.""" spec = get(op.type) + args = dict(op.args) + parallel.check(args.pop("parallel", None), spec.parallel_axes) try: - spec.arg_schema(**dict(op.args)) + spec.arg_schema(**args) except TypeError as e: raise ValueError(f"op {op.type!r} args don't match schema: {e}") from e return op diff --git a/operatorx/core/parallel.py b/operatorx/core/parallel.py new file mode 100644 index 0000000000..7fe37ad787 --- /dev/null +++ b/operatorx/core/parallel.py @@ -0,0 +1,44 @@ +"""How an op is split over the devices of one node. + + "parallel": {"tp": T, "dp": D, "ep": E, "dcp": C} (each optional, default 1) + + tp: weights (attention: heads) split T ways; the reduction is part of the op. + dp: D groups each bring their own batch (MoE experts see D x tokens in total). + ep: whole experts partitioned E ways; E is 1 or T x D. + dcp: each request's KV cache split C ways within a tp group, outputs merged; C divides T. + +Runs on T x D devices. Args keep the full unsplit shape and each dp group's batch; how +the split is carried out is the backend's choice. No "parallel": one device. +""" +from __future__ import annotations + +from typing import Any + +AXES = ("tp", "dp", "ep", "dcp") +GEMM_AXES = ("tp",) +MOE_AXES = ("tp", "dp", "ep") +ATTENTION_AXES = ("tp", "dp", "dcp") + + +def check(p: Any, allowed: tuple[str, ...] = AXES) -> None: + if p is None: + return + if not isinstance(p, dict) or set(p) - set(allowed): + raise ValueError(f"parallel must be a dict with keys from {list(allowed)}, got {p!r}") + for k, v in p.items(): + if not isinstance(v, int) or isinstance(v, bool) or v < 1: + raise ValueError(f"parallel.{k} must be a positive int, got {v!r}") + if p.get("ep", 1) not in (1, world_size(p)): + raise ValueError("parallel.ep must be 1 or tp x dp") + if p.get("tp", 1) % p.get("dcp", 1): + raise ValueError("parallel.dcp must divide parallel.tp") + + +def world_size(p: dict | None) -> int: + p = p or {} + return p.get("tp", 1) * p.get("dp", 1) + + +def normalize(p: dict | None) -> dict: + """Every axis spelled out: the key a process's parallel state is built for.""" + return {k: (p or {}).get(k, 1) for k in AXES} diff --git a/operatorx/core/result.py b/operatorx/core/result.py index b5eb5f5208..160493132c 100644 --- a/operatorx/core/result.py +++ b/operatorx/core/result.py @@ -17,8 +17,8 @@ class Result: op: Op metrics: Mapping[str, float] = field(default_factory=dict) status: str = "ok" # "ok" | "unsupported" | "error" - message: str | None = None # only set when status != "ok" - testlist: str | None = None # which testlist file produced this op + message: str | None = None # only when status != "ok" + testlist: str | None = None # testlist file that produced this op def to_dict(r: Result) -> dict: @@ -27,8 +27,7 @@ def to_dict(r: Result) -> dict: "args": dict(r.op.args), "backend": r.op.backend, } - if r.op.name is not None: - op_dict["name"] = r.op.name + op_dict["sources"] = list(r.op.sources) out: dict = { "op": op_dict, "metrics": dict(r.metrics), @@ -46,7 +45,7 @@ def _result_from_dict(d: dict) -> Result: type=d["op"]["type"], args=d["op"]["args"], backend=d["op"]["backend"], - name=d["op"].get("name"), + sources=d["op"].get("sources", ()), ) return Result( op=op, @@ -58,11 +57,7 @@ def _result_from_dict(d: dict) -> Result: def write_run_result(path: Path | str, run: RunInfo, results: Iterable[Result]) -> None: - """Write one run's worth of results to ``path`` in the run-result wrapper shape. - - Body: ``{"schema_version", "run": {...RunInfo...}, "rows": [{op, metrics}, ...]}``. - Parent directories are created. Existing files are overwritten — one file = one run. - """ + """{"schema_version", "run", "rows"}; one file = one run, replaced atomically.""" path = Path(path) path.parent.mkdir(parents=True, exist_ok=True) body = { @@ -76,7 +71,6 @@ def write_run_result(path: Path | str, run: RunInfo, results: Iterable[Result]) def read_run_result(path: Path | str) -> tuple[RunInfo, list[Result]]: - """Inverse of :func:`write_run_result`. Returns ``(run, rows)``.""" path = Path(path) body = json.loads(path.read_text()) run = _run_from_dict(body["run"]) diff --git a/operatorx/main.py b/operatorx/main.py index e05763cdb8..0caaf632d9 100644 --- a/operatorx/main.py +++ b/operatorx/main.py @@ -1,20 +1,8 @@ -"""Run every testlist × every backend on the detected platform. +"""``python -m operatorx``: run every testlist x backend on this platform. -Invoke as ``python -m operatorx`` (or ``python -m operatorx.main``). - -Reads testlists from ``/testlists/*.json`` (one file per testlist; filename -without extension = testlist name). Default = all testlists; use ``--testlists a,b,c`` -to filter. - -Backend op-type support is derived from each backend module's ``IMPLS`` list. If a -backend doesn't claim an op_type, the (op, backend) combination is silently skipped -— no Result row is emitted. Args-level rejection (e.g. dtype mismatch) is captured -as ``status="unsupported"``. Other exceptions become ``status="error"``. - -Platform is detected from ``$OPERATORX_CLUSTER`` → ``CLUSTERS[id].platform``, or -overridden with ``--platform``. Backend filter via ``--backends`` (CSV) or env -``OPERATORX_BACKENDS``. World size from ``WORLD_SIZE`` env (set by torchrun, or -manually for TPU); auto-detected for TPU when unset. +Op types a backend's IMPLS doesn't claim are skipped (no row, unless --strict); +UnsupportedOpError -> status="unsupported", other exceptions -> "error". +Env: OPERATORX_CLUSTER, OPERATORX_BACKENDS, OPERATORX_PARALLEL (JSON), WORLD_SIZE/RANK. """ from __future__ import annotations @@ -23,16 +11,20 @@ import json import os import pkgutil +import re import sys from pathlib import Path import operatorx.ops # noqa: F401 populates op registry +from operatorx.core import op_registry, parallel from operatorx import Op, Result, UnsupportedOpError, write_run_result from operatorx.clusters import CLUSTER_PLATFORMS from operatorx.runtime import runtime_snapshot, utc_now_iso +# a testlist source: "//" +_SOURCE = re.compile(r"[^/\s]+/[^/\s]+/[^/\s]+") + -# Package directory contains the checked-in testlists and local results. _REPO_ROOT = Path(__file__).resolve().parent @@ -41,9 +33,6 @@ def _discover_backends(platform: str) -> list[str]: - """All backends for a platform = the python modules under - ``operatorx/runners//backends/`` - """ try: pkg = importlib.import_module(f"operatorx.runners.{platform}.backends") except ImportError: @@ -61,7 +50,7 @@ def _csv(s: str | None) -> list[str]: def _load_testlists(names: list[str] | None, directory: Path = TESTLIST_DIR) -> dict[str, list[dict]]: - """Returns {testlist_name: [shape_dict, ...]}. Default = all available.""" + """{testlist_name: [shape_dict, ...]}; all available by default.""" available = {p.stem: p for p in sorted(directory.glob("*.json"))} if names: wanted = {n: available[n] for n in names if n in available} @@ -70,11 +59,19 @@ def _load_testlists(names: list[str] | None, directory: Path = TESTLIST_DIR) -> raise SystemExit(f"unknown testlist(s): {missing}; available: {sorted(available)}") else: wanted = available - return {name: json.loads(path.read_text()) for name, path in wanted.items()} + lists = {name: json.loads(path.read_text()) for name, path in wanted.items()} + for name, entries in lists.items(): + for i, entry in enumerate(entries): + v = entry.get("sources") + if not isinstance(v, list) or not all(isinstance(s, str) and _SOURCE.fullmatch(s) for s in v): + raise SystemExit(f"{name}[{i}]: every testlist entry needs a 'sources' list of " + f"'//' strings (empty for a shape from no model)") + if "name" in entry: + raise SystemExit(f"{name}[{i}]: 'name' is gone; the role is the last part of each source") + return lists def _backend_supported_ops(platform: str, backends: list[str], strict: bool = False) -> dict[str, set[str]]: - """Discover per-backend supported op_types from each module's IMPLS list.""" out: dict[str, set[str]] = {} for b in backends: try: @@ -90,11 +87,7 @@ def _backend_supported_ops(platform: str, backends: list[str], strict: bool = Fa def _collect_backend_versions(platform: str, backends: list[str]) -> dict[str, str]: - """Ask each backend module to report its own library versions. - - Each backend exports ``versions() -> dict[str, str]``. Returned keys are - merged into ``RunInfo.software``. Missing or failing probes are skipped. - """ + """Each backend's versions(); failing probes are skipped.""" out: dict[str, str] = {} for b in backends: try: @@ -123,17 +116,13 @@ def _resolve_platform(args_platform: str | None, run_cluster: str | None) -> str ) -def _resolve_world_size(platform: str) -> int: - ws = os.environ.get("WORLD_SIZE") - if ws: - return int(ws) - if platform == "tpu": - try: - import jax - return jax.device_count() - except Exception: - pass - return 1 +def _resolve_split() -> dict: + """This process's split (OPERATORX_PARALLEL, JSON; default one device); must match WORLD_SIZE.""" + split = parallel.normalize(json.loads(os.environ.get("OPERATORX_PARALLEL") or "null")) + ws = int(os.environ.get("WORLD_SIZE", "1")) + if parallel.world_size(split) != ws: + raise SystemExit(f"OPERATORX_PARALLEL={split} needs {parallel.world_size(split)} ranks, launched {ws}") + return split def main() -> int: @@ -152,79 +141,41 @@ def main() -> int: run = runtime_snapshot() platform = _resolve_platform(args.platform, run.cluster) - # Backends requested = _csv(args.backends) or _csv(os.environ.get("OPERATORX_BACKENDS")) backends = requested if requested else _discover_backends(platform) if not backends: raise SystemExit(f"no backends configured for platform={platform!r}") backend_ops = _backend_supported_ops(platform, backends, strict=args.strict) - # Merge per-backend library versions into the run's software dict. run.software.update(_collect_backend_versions(platform, backends)) - ws = _resolve_world_size(platform) + split = _resolve_split() + ws = parallel.world_size(split) rank = int(os.environ.get("RANK", "0")) - # Testlists testlists = _load_testlists( _csv(args.testlists) or _csv(os.environ.get("OPERATORX_TESTLISTS")) or None, args.testlist_dir, ) - # submit_run.py fans out one job per (ep, routed_tp, shared_tp) MoE combo - # plus one job per ws for non-MoE ops, since sglang's group state is - # process-global. OPERATORX_MOE_PARALLELISM scopes a job to one combo; - # absence scopes it to non-MoE ops only. - moe_par_env = os.environ.get("OPERATORX_MOE_PARALLELISM") - moe_filter: tuple[int, int, int] | None = None - if moe_par_env: - parts = moe_par_env.split(":") - if len(parts) != 3: - raise SystemExit(f"OPERATORX_MOE_PARALLELISM must be ep:routed_tp:shared_tp; got {moe_par_env!r}") - moe_filter = (int(parts[0]), int(parts[1]), int(parts[2])) - - # Load runner runner_mod = importlib.import_module(f"operatorx.runners.{platform}.runner") - # Build entries: (op, testlist_name). entries: list[tuple[Op, str]] = [] for tl_name, shapes in testlists.items(): for shape in shapes: - # In a multi-rank job (ws>1) only run ops explicitly tagged with the - # matching world_size — running single-rank ops on every rank is - # both wasteful and meaningless (each rank would redo the same work - # in parallel). Per-rank ops belong in the ws=1 job. - shape_ws = shape["args"].get("world_size") - if shape_ws is None: - if ws != 1: - continue - elif int(shape_ws) != ws: - continue - # Combo job (moe_filter set): only moe_forward shapes matching the - # parallelism triple. Non-combo job: everything except moe_forward. - if moe_filter is not None: - if shape["type"] != "moe_forward": - continue - a = shape["args"] - triple = ( - a.get("expert_parallel_size", 1), - a.get("routed_tensor_parallel_size", 1), - a.get("shared_tensor_parallel_size", 1), - ) - if triple != moe_filter: - continue - elif shape["type"] == "moe_forward": + # one parallel state per process; other splits run in their own process + if parallel.normalize(shape["args"].get("parallel")) != split: continue for backend in backends: if not args.strict and shape["type"] not in backend_ops.get(backend, set()): - continue # Strict CI retains unsupported backend/operator pairs. + continue # strict CI keeps unsupported backend/op pairs entries.append(( Op(type=shape["type"], args=shape["args"], backend=backend, - name=shape.get("name")), + sources=shape["sources"]), tl_name, )) if rank == 0: - print(f"[run_smoke] platform={platform} cluster={run.cluster!r} ws={ws}") + print(f"[run_smoke] platform={platform} cluster={run.cluster!r} parallel={split}") print(f"[run_smoke] backends={backends}") print(f"[run_smoke] testlists={list(testlists)} -> {len(entries)} (op,backend) entries") @@ -237,6 +188,8 @@ def main() -> int: try: if op.type not in backend_ops.get(op.backend, set()): raise UnsupportedOpError(f"{platform}/{op.backend} has no implementation for {op.type}") + # a case the op's schema rejects is a bad testlist entry: an error, never measured + op_registry.validate(op) r = runner_mod.run(op) results.append(Result(op=op, metrics=r.metrics, status="ok", testlist=tl)) @@ -260,19 +213,17 @@ def main() -> int: latency_str = f" ERROR ({type(e).__name__})" wall_s = _time.perf_counter() - _t0 if rank == 0: - # Checkpoint outside the timed kernel so cancellation preserves completed rows. + # checkpoint after each op, outside timing, so cancellation keeps completed rows if args.strict: run.finished_at = utc_now_iso() write_run_result(out_path, run, results) shape_str = " ".join( - f"{k}={v}" for k, v in op.args.items() if not k.startswith("dtype") + f"{k}={json.dumps(v, separators=(',', ':')) if isinstance(v, (dict, list)) else v}" + for k, v in op.args.items() if not k.startswith("dtype") ) print(f"[ws={ws}] {tl:12} {status:11} {op.type:18} {op.backend:10} " f"{latency_str} wall={wall_s:6.1f}s {shape_str}", flush=True) - # Release per-op tensors back to the CUDA driver so the next op's - # allocator sees the full GPU. Without this, PyTorch's caching - # allocator hangs on to large output buffers and the next big shape - # OOMs even though the prior tensors are no longer referenced. + # release cached blocks, else the next big shape can OOM on the previous op's buffers try: import torch as _torch if _torch.cuda.is_available(): @@ -286,7 +237,6 @@ def main() -> int: write_run_result(out_path, run, results) print(f"\n[run_smoke] {counts} -> {out_path}") - # Multi-rank cleanup (torch.distributed used by NVIDIA + Trainium) if ws > 1: try: import torch.distributed as dist diff --git a/operatorx/ops/__init__.py b/operatorx/ops/__init__.py index 0c56618cf6..4a211b5fdc 100644 --- a/operatorx/ops/__init__.py +++ b/operatorx/ops/__init__.py @@ -1,4 +1 @@ -from operatorx.ops import gemm -from operatorx.ops import attention -from operatorx.ops import collective -from operatorx.ops import layer +from operatorx.ops import attention, gemm, moe diff --git a/operatorx/ops/attention.py b/operatorx/ops/attention.py index 36aa0479d4..d42a24c302 100644 --- a/operatorx/ops/attention.py +++ b/operatorx/ops/attention.py @@ -1,57 +1,447 @@ +"""Attention modules: one op type per module, each timed whole. + +From post-input_layernorm x [T, hidden] (+ positions) to the module output before the +residual add: projections, norms, RoPE, cache/state writes, selection and attention; the +tp all-reduce, residual, hyper-connections and MTP are outside. The cache/state holds +each request's ctx tokens of random, format-valid data. + +Shared args: + proj: {"": {"a": operand, "b": operand}} (gemm operands); omitted: bf16 x bf16. + batch: {"groups": [{"count", "q", "ctx"}], "pages"?: "contiguous" | "shuffled", "seed"?} + count requests of q new tokens on ctx cached; ctx is an int or {"dist": "uniform", "min", + "max"} drawn from seed. Prefill vs decode path is the backend's choice, recorded. + selection (sparse modules): natural = the module's indexer on the random cache; + uniform / recent / clustered force the indices. +""" from __future__ import annotations -from dataclasses import dataclass +from dataclasses import dataclass, replace +from typing import Any +from operatorx.core import parallel from operatorx.core.op import OpSpec from operatorx.core.op_registry import register +from operatorx.ops.gemm import check_operand +SELECTIONS = {"natural", "uniform", "recent", "clustered"} +PAGES = {"contiguous", "shuffled"} +STATE_DTYPES = {"fp32", "bf16"} -@dataclass(frozen=True) -class AttentionMhaArgs: - batch_size: int - seq_len_q: int - seq_len_kv: int - num_heads: int - num_heads_kv: int - head_dim: int - dtype_q: str = "bf16" - dtype_k: str = "bf16" - dtype_v: str = "bf16" - dtype_o: str = "bf16" - causal: bool = True - sliding_window: int | None = None - kv_layout: str = "contig" # "contig" | "paged" - block_size: int | None = None + +def _pos_int(v: Any, where: str, zero: bool = False) -> None: + if not isinstance(v, int) or isinstance(v, bool) or v < (0 if zero else 1): + raise ValueError(f"{where} must be a {'non-negative' if zero else 'positive'} int, got {v!r}") + + +def _number(v: Any, where: str) -> None: + if not isinstance(v, (int, float)) or isinstance(v, bool) or v <= 0: + raise ValueError(f"{where} must be a positive number, got {v!r}") + + +def _choice(v: Any, where: str, allowed: set) -> None: + if v not in allowed: + raise ValueError(f"{where} must be one of {sorted(allowed, key=str)}, got {v!r}") + + +def _bool(v: Any, where: str) -> None: + if not isinstance(v, bool): + raise ValueError(f"{where} must be a bool, got {v!r}") + + +def _keys(d: Any, where: str, required: set, optional: set = frozenset()) -> None: + if not isinstance(d, dict): + raise ValueError(f"{where} must be a dict, got {d!r}") + missing, extra = required - set(d), set(d) - required - set(optional) + if missing or extra: + raise ValueError(f"{where}: missing {sorted(missing)}, unknown {sorted(extra)}") + + +def _check_proj(p: Any, names: tuple[str, ...]) -> None: + _keys(p, "proj", set(), set(names)) + for name, pair in p.items(): + _keys(pair, f"proj.{name}", {"a", "b"}) + check_operand(pair["a"], f"proj.{name}.a") + check_operand(pair["b"], f"proj.{name}.b") + + +def _check_batch(b: Any) -> None: + _keys(b, "batch", {"groups"}, {"pages", "seed"}) + groups = b["groups"] + if not isinstance(groups, list) or not groups: + raise ValueError("batch.groups must be a non-empty list") + for i, g in enumerate(groups): + where = f"batch.groups[{i}]" + _keys(g, where, {"count", "q", "ctx"}) + _pos_int(g["count"], f"{where}.count") + _pos_int(g["q"], f"{where}.q") + ctx = g["ctx"] + if isinstance(ctx, dict): + _keys(ctx, f"{where}.ctx", {"dist", "min", "max"}) + _choice(ctx["dist"], f"{where}.ctx.dist", {"uniform"}) + _pos_int(ctx["min"], f"{where}.ctx.min", zero=True) + _pos_int(ctx["max"], f"{where}.ctx.max", zero=True) + if ctx["min"] > ctx["max"]: + raise ValueError(f"{where}.ctx.min exceeds ctx.max") + else: + _pos_int(ctx, f"{where}.ctx", zero=True) + _choice(b.get("pages", "contiguous"), "batch.pages", PAGES) + if not isinstance(b.get("seed", 0), int) or isinstance(b.get("seed", 0), bool): + raise ValueError("batch.seed must be an int") + + +def _check_rope_scaling(s: Any) -> None: + if s is None: + return + _keys(s, "rope_scaling", {"type", "factor", "original_max"}, {"beta_fast", "beta_slow", "mscale", "mscale_all_dim"}) + _choice(s["type"], "rope_scaling.type", {"yarn"}) + _number(s["factor"], "rope_scaling.factor") + _pos_int(s["original_max"], "rope_scaling.original_max") + for k in ("beta_fast", "beta_slow", "mscale", "mscale_all_dim"): + if k in s: + _number(s[k], f"rope_scaling.{k}") + + +def _check_mrope(m: Any, rope_dim: int) -> None: + if m is None: + return + if not (isinstance(m, list) and len(m) == 3 and all(isinstance(x, int) and x >= 0 for x in m)): + raise ValueError(f"mrope_section must be three non-negative ints, got {m!r}") + if 2 * sum(m) != rope_dim: + raise ValueError(f"mrope_section {m} must cover rope_dim/2 = {rope_dim // 2}") + + +MLA_PROJ = ("q_a_proj", "kv_a_proj_with_mqa", "q_b_proj", "kv_b_proj", "o_proj", "g_proj") @dataclass(frozen=True) -class AttentionMlaArgs: - batch_size: int - seq_len_q: int - seq_len_kv: int - num_heads: int - head_dim_qk_nope: int - head_dim_qk_rope: int - head_dim_v: int +class MlaArgs: + """DeepSeek multi-head latent attention (DeepSeek-R1, Kimi-K3's MLA layers). + + Timed: q_a_proj + kv_a_proj_with_mqa (+ g_proj when gate) -> q_a / kv_a RMSNorm -> + q_b_proj -> RoPE on q_pe and k_pe (when rope) -> latent cache write (kv_c, k_pe) -> + prefill: kv_b_proj up-projection of new and cached tokens, MHA, merge of context + chunks; decode: q_nope x W_UK, MQA over the latent cache, x W_UV -> sigmoid output + gate (when gate) -> o_proj. + + rope false: NoPE; the rope_dim part is still cached and used, never rotated. + kv_cache_dtype: vLLM's --kv-cache-dtype (auto = the model dtype). + """ + hidden: int + heads: int + q_lora_rank: int kv_lora_rank: int - dtype_q: str = "bf16" - dtype_kv: str = "bf16" # compressed cache + rope share one dtype in practice - dtype_o: str = "bf16" + nope: int + rope_dim: int + v: int + batch: dict rope: bool = True - causal: bool = True + rope_theta: float | None = None + rope_scaling: dict | None = None + gate: bool = False + kv_cache_dtype: str = "auto" + proj: dict | None = None + + def __post_init__(self): + for k in ("hidden", "heads", "q_lora_rank", "kv_lora_rank", "nope", "rope_dim", "v"): + _pos_int(getattr(self, k), k) + _bool(self.rope, "rope") + _bool(self.gate, "gate") + if self.rope: + _number(self.rope_theta, "rope_theta") + _check_rope_scaling(self.rope_scaling) + elif self.rope_theta is not None or self.rope_scaling is not None: + raise ValueError("rope_theta / rope_scaling need rope") + _choice(self.kv_cache_dtype, "kv_cache_dtype", {"auto", "bf16", "fp8"}) + _check_proj(self.proj or {}, MLA_PROJ if self.gate else MLA_PROJ[:-1]) + _check_batch(self.batch) + + +MLA_DSA_PROJ = ("q_a_proj", "kv_a_proj_with_mqa", "q_b_proj", "kv_b_proj", "o_proj", + "indexer.wq_b", "indexer.wk", "indexer.weights_proj") + + +@dataclass(frozen=True) +class MlaDsaArgs: + """MLA with DeepSeek sparse attention (GLM-5.1 / 5.2 / 5.3). + + Timed: q_a_proj + kv_a_proj_with_mqa (+ indexer.wk / weights_proj when indexer is + own) -> q_a / kv_a RMSNorm, RoPE, latent cache write (and index-K cache write when + own) -> q_b_proj -> q_nope x W_UK -> indexer (own: indexer.wq_b, index q RoPE and + quant, logits over the index-K cache, top-k; reuse: the top-k of an earlier layer, + pre-filled) -> MQA over the top-k tokens (dense MHA prefill where the backend picks + it for short sequences) -> x W_UV -> o_proj. + + kv_cache_dtype: vLLM's --kv-cache-dtype; fp8_ds_mla is the fp8 latent with per-128 + scales and a bf16 rope part. + """ + hidden: int + heads: int + q_lora_rank: int + kv_lora_rank: int + nope: int + rope_dim: int + v: int + rope_theta: float + topk: int + index_heads: int + index_dim: int + batch: dict + indexer: str = "own" + rope_scaling: dict | None = None + kv_cache_dtype: str = "auto" + selection: str = "natural" + proj: dict | None = None + + def __post_init__(self): + for k in ("hidden", "heads", "q_lora_rank", "kv_lora_rank", "nope", "rope_dim", "v", "topk", + "index_heads", "index_dim"): + _pos_int(getattr(self, k), k) + _number(self.rope_theta, "rope_theta") + _check_rope_scaling(self.rope_scaling) + _choice(self.indexer, "indexer", {"own", "reuse"}) + _choice(self.kv_cache_dtype, "kv_cache_dtype", {"auto", "bf16", "fp8", "fp8_ds_mla"}) + _choice(self.selection, "selection", SELECTIONS) + _check_proj(self.proj or {}, MLA_DSA_PROJ if self.indexer == "own" else MLA_DSA_PROJ[:5]) + _check_batch(self.batch) + + +DSV4_PROJ = ("wq_a", "wkv", "wq_b", "wo_a", "wo_b", "compressor.wkv", "compressor.wgate", + "indexer.wq_b", "indexer.wk", "indexer.weights_proj", + "indexer.compressor.wkv", "indexer.compressor.wgate") +DSV4_RATIOS = {0, 1, 2, 4, 128} -ATTENTION_MHA = OpSpec( - type="attention_mha", - arg_schema=AttentionMhaArgs, - description="Vanilla multi-head attention", -) +@dataclass(frozen=True) +class Dsv4AttnArgs: + """DeepSeek-V4 attention (DeepSeek-V4-Pro, DeepSeek-V4.1-Flash): a sliding window of + raw tokens plus compressed tokens, one latent used as both K and V, learned sinks. + + Timed: wq_a + wkv -> q_norm / kv_norm -> wq_b -> compressor (compress_ratio > 0 and + source: compressor.wkv (+ wgate), pooling of compress_ratio tokens, RMSNorm, RoPE, + compressed cache write, compressor state update) -> indexer (own: indexer.wq_b and + weights_proj, index keys (ratio 4: its own compressor; ratio 1 / 2: indexer.wk over + the source latent), logits, top-k; shared: own queries over a pre-filled index-K + cache; reuse: a pre-filled top-k) -> per-head q RMSNorm, RoPE, quant and window + cache write -> sparse MQA over window + compressed tokens with attn_sink -> inverse + RoPE -> wo_a (o_groups grouped) -> wo_b. + + compress_ratio: 4 / 128 V4-Pro (4 with an indexer, 128 attends every compressed token); + 0 (window only), 1 / 2 V4.1; compressor overlap/pos-emb/gate follow the ratio. + source false: reads another layer's pre-filled compressed cache, no compressor. + rope_theta: window-only layers; compress_rope_theta (+ rope_scaling): compressed ones. + index_cache_dtype: the indexer K cache (vLLM indexer_kv_dtype). + """ + hidden: int + heads: int + head_dim: int + rope_dim: int + q_lora_rank: int + window: int + o_groups: int + o_lora_rank: int + compress_ratio: int + batch: dict + source: bool = True + indexer: str | None = None + topk: int | None = None + index_heads: int | None = None + index_dim: int | None = None + rope_theta: float = 10000.0 + compress_rope_theta: float | None = None + rope_scaling: dict | None = None + kv_cache_dtype: str = "auto" + index_cache_dtype: str | None = None + selection: str = "natural" + proj: dict | None = None + + def __post_init__(self): + for k in ("hidden", "heads", "head_dim", "rope_dim", "q_lora_rank", "window", "o_groups", "o_lora_rank"): + _pos_int(getattr(self, k), k) + if self.heads % self.o_groups: + raise ValueError("o_groups must divide heads") + _choice(self.compress_ratio, "compress_ratio", DSV4_RATIOS) + _bool(self.source, "source") + _choice(self.indexer, "indexer", {None, "own", "shared", "reuse"}) + if self.compress_ratio in (0, 128) and self.indexer is not None: + raise ValueError(f"compress_ratio {self.compress_ratio} has no indexer") + if self.compress_ratio == 4 and self.indexer != "own": + raise ValueError("compress_ratio 4 runs its own indexer") + if self.compress_ratio == 0 and not self.source: + raise ValueError("a window-only layer has no compressed cache to share") + index = ("topk", "index_heads", "index_dim") + if self.indexer is None: + if any(getattr(self, k) is not None for k in index) or self.index_cache_dtype is not None: + raise ValueError("topk / index_heads / index_dim / index_cache_dtype need an indexer") + else: + for k in index: + _pos_int(getattr(self, k), k) + _choice(self.index_cache_dtype or "fp8", "index_cache_dtype", {"bf16", "fp8", "mxfp4", "nvfp4"}) + _number(self.rope_theta, "rope_theta") + if self.compress_ratio: + _number(self.compress_rope_theta, "compress_rope_theta") + _check_rope_scaling(self.rope_scaling) + elif self.compress_rope_theta is not None or self.rope_scaling is not None: + raise ValueError("compress_rope_theta / rope_scaling need compress_ratio > 0") + _choice(self.kv_cache_dtype, "kv_cache_dtype", {"auto", "bf16", "fp8", "fp8_ds_mla"}) + _choice(self.selection, "selection", SELECTIONS) + _check_proj(self.proj or {}, DSV4_PROJ) + _check_batch(self.batch) + + +GQA_PROJ = ("q_proj", "k_proj", "v_proj", "o_proj") + + +@dataclass(frozen=True) +class GqaArgs: + """Dense grouped-query attention (MiniMax-M3 layers 0-2, Qwen3.5 full attention). + + Timed: q_proj (with the gate packed in when gate), k_proj, v_proj -> per-head + q / k RMSNorm (when qk_norm) -> RoPE on the first rope_dim dims (M-RoPE when + mrope_section) -> KV cache write -> causal attention -> sigmoid output gate (when + gate) -> o_proj. + """ + hidden: int + q_heads: int + kv_heads: int + head_dim: int + rope_dim: int + rope_theta: float + batch: dict + mrope_section: list | None = None + qk_norm: bool = False + gate: bool = False + kv_cache_dtype: str = "auto" + proj: dict | None = None + + def __post_init__(self): + for k in ("hidden", "q_heads", "kv_heads", "head_dim", "rope_dim"): + _pos_int(getattr(self, k), k) + if self.q_heads % self.kv_heads: + raise ValueError("kv_heads must divide q_heads") + if self.rope_dim > self.head_dim: + raise ValueError("rope_dim exceeds head_dim") + _number(self.rope_theta, "rope_theta") + _check_mrope(self.mrope_section, self.rope_dim) + _bool(self.qk_norm, "qk_norm") + _bool(self.gate, "gate") + _choice(self.kv_cache_dtype, "kv_cache_dtype", {"auto", "bf16", "fp8"}) + _check_proj(self.proj or {}, GQA_PROJ) + _check_batch(self.batch) + + +QSA_PROJ = (*GQA_PROJ, "indexer.index_qk_proj") + + +@dataclass(frozen=True) +class QsaArgs: + """Qwen3.8-Flash-Next gated sparse attention. + + Timed: q_proj (with the gate), k_proj, v_proj -> q / k RMSNorm, RoPE -> KV cache + write -> indexer (indexer.index_qk_proj to index_heads queries and one key, + RMSNorm, RoPE; keys pooled compress tokens at a time into the compressed index-K + cache; score = sum over index heads of ReLU(q . k); top budget/compress groups, + expanded to budget tokens) -> attention over the selected tokens with the sigmoid + output gate fused -> o_proj. The KV cache is bf16. + """ + hidden: int + q_heads: int + kv_heads: int + head_dim: int + rope_dim: int + rope_theta: float + index_heads: int + index_dim: int + compress: int + budget: int + batch: dict + mrope_section: list | None = None + selection: str = "natural" + proj: dict | None = None + + def __post_init__(self): + for k in ("hidden", "q_heads", "kv_heads", "head_dim", "rope_dim", "index_heads", "index_dim", + "compress", "budget"): + _pos_int(getattr(self, k), k) + if self.q_heads % self.kv_heads: + raise ValueError("kv_heads must divide q_heads") + if self.budget % self.compress: + raise ValueError("compress must divide budget") + _number(self.rope_theta, "rope_theta") + _check_mrope(self.mrope_section, self.rope_dim) + _choice(self.selection, "selection", SELECTIONS) + _check_proj(self.proj or {}, QSA_PROJ) + _check_batch(self.batch) + + +GDN_PROJ = ("in_proj_qkv", "in_proj_z", "in_proj_a", "in_proj_b", "out_proj") + + +@dataclass(frozen=True) +class GdnArgs: + """Gated DeltaNet linear attention (Qwen3.5 / Qwen3.8 linear_attn layers). + + Timed: in_proj_qkv, in_proj_z, in_proj_a, in_proj_b -> causal depthwise conv1d + (conv_kernel) with conv-state update -> decay g and beta gates -> q / k L2 norm -> + delta rule (chunked for prefill, recurrent for decode) reading and writing the + recurrent state -> RMSNorm gated by z (norm_act) -> out_proj. ctx > 0 means the + request starts from a pre-filled state. + """ + hidden: int + qk_heads: int + v_heads: int + head_dim: int + conv_kernel: int + batch: dict + norm_act: str = "silu" + state_dtype: str = "fp32" + proj: dict | None = None + + def __post_init__(self): + for k in ("hidden", "qk_heads", "v_heads", "head_dim", "conv_kernel"): + _pos_int(getattr(self, k), k) + if self.v_heads % self.qk_heads: + raise ValueError("qk_heads must divide v_heads") + _choice(self.norm_act, "norm_act", {"silu", "sigmoid"}) + _choice(self.state_dtype, "state_dtype", STATE_DTYPES) + _check_proj(self.proj or {}, GDN_PROJ) + _check_batch(self.batch) + + +KDA_PROJ = ("q_proj", "k_proj", "v_proj", "g_proj", "f_a_proj", "f_b_proj", "b_proj", "o_proj") + + +@dataclass(frozen=True) +class KdaArgs: + """Kimi Delta Attention (Kimi-K3 linear layers). + + Timed: q_proj, k_proj, v_proj, g_proj, f_a_proj, b_proj -> f_b_proj (the + head_dim-rank per-channel decay gate) -> causal depthwise conv1d on q / k / v with + conv-state update -> decay and beta gates -> gated delta rule with q / k L2 norm, + reading and writing the recurrent state -> RMSNorm gated by g -> o_proj. + """ + hidden: int + heads: int + head_dim: int + conv_kernel: int + batch: dict + state_dtype: str = "fp32" + proj: dict | None = None + + def __post_init__(self): + for k in ("hidden", "heads", "head_dim", "conv_kernel"): + _pos_int(getattr(self, k), k) + _choice(self.state_dtype, "state_dtype", STATE_DTYPES) + _check_proj(self.proj or {}, KDA_PROJ) + _check_batch(self.batch) -ATTENTION_MLA = OpSpec( - type="attention_mla", - arg_schema=AttentionMlaArgs, - description="Multi-head latent attention with compressed KV cache.", -) -register(ATTENTION_MHA) -register(ATTENTION_MLA) +for _spec in ( + OpSpec(type="mla", arg_schema=MlaArgs, description="DeepSeek MLA module"), + OpSpec(type="mla_dsa", arg_schema=MlaDsaArgs, description="MLA with DeepSeek sparse attention"), + OpSpec(type="dsv4_attn", arg_schema=Dsv4AttnArgs, description="DeepSeek-V4 window + compressed attention"), + OpSpec(type="gqa", arg_schema=GqaArgs, description="dense grouped-query attention module"), + OpSpec(type="qsa", arg_schema=QsaArgs, description="Qwen3.8 gated sparse attention"), + OpSpec(type="gdn", arg_schema=GdnArgs, description="Gated DeltaNet linear attention"), + OpSpec(type="kda", arg_schema=KdaArgs, description="Kimi Delta Attention"), +): + register(replace(_spec, parallel_axes=parallel.ATTENTION_AXES)) diff --git a/operatorx/ops/collective.py b/operatorx/ops/collective.py deleted file mode 100644 index bdeb8ec75e..0000000000 --- a/operatorx/ops/collective.py +++ /dev/null @@ -1,87 +0,0 @@ -from __future__ import annotations - -from dataclasses import dataclass - -from operatorx.core.op import OpSpec -from operatorx.core.op_registry import register - - -@dataclass(frozen=True) -class AllreduceArgs: - num_elements: int - dtype: str - world_size: int - - -@dataclass(frozen=True) -class AllgatherArgs: - num_elements_per_rank: int - dtype: str - world_size: int - - -@dataclass(frozen=True) -class ReduceScatterArgs: - num_elements: int # total elements; output per rank is num_elements // world_size - dtype: str - world_size: int - - -@dataclass(frozen=True) -class AlltoallArgs: - num_elements_per_rank: int # total elements per rank, split into world_size chunks - dtype: str - world_size: int - - -@dataclass(frozen=True) -class DispatchArgs: - num_tokens: int - num_experts: int - top_k: int - hidden: int - dtype: str - world_size: int - - -@dataclass(frozen=True) -class CombineArgs: - num_tokens: int - num_experts: int - top_k: int - hidden: int - dtype: str - world_size: int - - -ALLREDUCE = OpSpec( - type="allreduce", arg_schema=AllreduceArgs, - description="AllReduce", -) -ALLGATHER = OpSpec( - type="allgather", arg_schema=AllgatherArgs, - description="AllGather", -) -REDUCE_SCATTER = OpSpec( - type="reduce_scatter", arg_schema=ReduceScatterArgs, - description="ReduceScatter", -) -ALLTOALL = OpSpec( - type="alltoall", arg_schema=AlltoallArgs, - description="AlltoAll", -) -DISPATCH = OpSpec( - type="dispatch", arg_schema=DispatchArgs, - description="MoE Dispatch", -) -COMBINE = OpSpec( - type="combine", arg_schema=CombineArgs, - description="MoE Combine", -) - -register(ALLREDUCE) -register(ALLGATHER) -register(REDUCE_SCATTER) -register(ALLTOALL) -register(DISPATCH) -register(COMBINE) diff --git a/operatorx/ops/gemm.py b/operatorx/ops/gemm.py index e9b0bb98aa..afd10b7234 100644 --- a/operatorx/ops/gemm.py +++ b/operatorx/ops/gemm.py @@ -1,27 +1,107 @@ from __future__ import annotations from dataclasses import dataclass +from typing import Any +from operatorx.core import parallel from operatorx.core.op import OpSpec from operatorx.core.op_registry import register +ELEMENT_DTYPES = {"bf16", "fp16", "fp32", "e4m3", "e5m2", "e2m1", "int4", "int8"} +SCALE_DTYPES = {"fp32", "bf16", "fp16", "e4m3", "ue8m0"} + + +def quant(dtype: str, scale: dict | None = None, scale2: dict | None = None, + symmetric: bool = True, input: str = "bf16") -> dict[str, Any]: + """Canonical operand descriptor; defaults are omitted so equal operands hash equal.""" + d: dict[str, Any] = {"dtype": dtype} + if input != "bf16": + d["input"] = input + if scale is not None: + d["scale"] = scale + if scale2 is not None: + d["scale2"] = scale2 + if not symmetric: + d["symmetric"] = False + return d + + +def scale(dtype: str, static: bool, group: tuple[int, int] | list[int]) -> dict[str, Any]: + return {"dtype": dtype, "static": static, "group": [int(group[0]), int(group[1])]} + + +def _check_scale(s: Any, where: str) -> None: + if not isinstance(s, dict) or set(s) != {"dtype", "static", "group"}: + raise ValueError(f"{where} must be {{dtype, static, group}}, got {s!r}") + if s["dtype"] not in SCALE_DTYPES: + raise ValueError(f"{where}.dtype {s['dtype']!r} not in {sorted(SCALE_DTYPES)}") + if not isinstance(s["static"], bool): + raise ValueError(f"{where}.static must be a bool") + g = s["group"] + if not (isinstance(g, list) and len(g) == 2 and all(isinstance(x, int) and (x == -1 or x >= 1) for x in g)): + raise ValueError(f"{where}.group must be [rows, cols] with each -1 or >= 1, got {g!r}") + + +def check_operand(d: Any, where: str) -> None: + if not isinstance(d, dict) or "dtype" not in d or set(d) - {"dtype", "scale", "scale2", "symmetric", "input"}: + raise ValueError(f"{where} must be {{dtype[, scale, scale2, symmetric, input]}}, got {d!r}") + if d.get("input", "bf16") not in ELEMENT_DTYPES: + raise ValueError(f"{where}.input {d['input']!r} not in {sorted(ELEMENT_DTYPES)}") + if d["dtype"] not in ELEMENT_DTYPES: + raise ValueError(f"{where}.dtype {d['dtype']!r} not in {sorted(ELEMENT_DTYPES)}") + if "scale" in d: + _check_scale(d["scale"], f"{where}.scale") + elif "scale2" in d: + raise ValueError(f"{where}.scale2 needs a scale") + if "scale2" in d: + _check_scale(d["scale2"], f"{where}.scale2") + if d.get("symmetric", True) is not True and d["symmetric"] is not False: + raise ValueError(f"{where}.symmetric must be a bool") + + +def plain_dtype(args: dict) -> str | None: + """The shared element dtype when neither operand is quantized, else None.""" + qa, qb = args["a"], args["b"] + return qa["dtype"] if qa == qb == {"dtype": qa["dtype"]} else None + @dataclass(frozen=True) class GemmArgs: + """C[M,N] = activation(A[M,K] @ B[N,K]^T + bias); A = activation, B = weight. + + a, b: {"dtype", "scale"?, "scale2"?, "symmetric"?, "input"?}; unquantized: {"dtype": "bf16"} + dtype: storage element type (bf16, e4m3, e2m1, int4, ...) + scale: {"dtype", "static", "group": [rows, cols]}; group is the block of the stored + operand (A [M, K], B [N, K]) sharing one scale, -1 spans the dim: [-1, -1] per-tensor, + [1, -1] per-token, [-1, 1] per-channel, [1, g] groups of g along K, [r, c] 2-D blocks. + static=False: computed at runtime (quantization inside the op). + scale2: second-level scale (e.g. NVFP4's per-tensor fp32 global scale). + symmetric: False when the format has zero points. + input: dtype the operand arrives in (default bf16); != dtype means quantizing is in the op. + parallel: {"tp": T} - B split over K, partial C summed across T devices (in the op); + m, n, k are the full layer shape. + """ m: int n: int k: int - dtype_a: str = "bf16" - dtype_b: str = "bf16" - dtype_out: str = "bf16" + a: dict + b: dict + out: str = "bf16" bias: bool = False activation: str | None = None + def __post_init__(self): + check_operand(self.a, "a") + check_operand(self.b, "b") + if self.out not in ELEMENT_DTYPES: + raise ValueError(f"out {self.out!r} not in {sorted(ELEMENT_DTYPES)}") + GEMM = OpSpec( type="gemm", arg_schema=GemmArgs, - description="C = activation(A[M,K] @ B[K,N] + bias)", + description="C = activation(A[M,K] @ B[N,K]^T + bias)", + parallel_axes=parallel.GEMM_AXES, ) register(GEMM) diff --git a/operatorx/ops/layer.py b/operatorx/ops/layer.py deleted file mode 100644 index 0ba4f2cd35..0000000000 --- a/operatorx/ops/layer.py +++ /dev/null @@ -1,63 +0,0 @@ -from __future__ import annotations - -from dataclasses import dataclass - -from operatorx.core.op import OpSpec -from operatorx.core.op_registry import register - - -@dataclass(frozen=True) -class MoeForwardArgs: - num_tokens: int - hidden: int - intermediate: int - num_experts: int - top_k: int - dtype_act: str = "bf16" - dtype_weight: str = "bf16" - world_size: int = 1 - expert_parallel_size: int = 1 - routed_tensor_parallel_size: int = 1 - shared_tensor_parallel_size: int = 1 - n_shared_experts: int = 0 - # How tokens are routed across experts at benchmark time. - # "uniform" — uniform-random topk (best-case load balance, default). - # "zipf" — Zipfian (s=1) bias on routing logits — a few experts hot, - # tail cold. Approximates pre-aux-loss DSv3-style skew. - # "single_hot" — every token routes to expert 0 + (top_k-1) random — worst - # case for load imbalance and dispatch. - expert_distribution: str = "uniform" - - -MOE_FORWARD = OpSpec( - type="moe_forward", - arg_schema=MoeForwardArgs, - description="Full MoE forward (gate + dispatch + experts + combine), timed as one unit.", -) - -register(MOE_FORWARD) - - -@dataclass(frozen=True) -class MoeGemmArgs: - num_tokens: int - hidden: int - intermediate: int - num_experts: int - top_k: int - dtype_act: str = "bf16" - dtype_weight: str = "bf16" - expert_parallel_size: int = 1 - routed_tensor_parallel_size: int = 1 - shared_tensor_parallel_size: int = 1 - n_shared_experts: int = 0 - expert_distribution: str = "uniform" - - -MOE_GEMM = OpSpec( - type="moe_gemm", - arg_schema=MoeGemmArgs, - description="Per-rank routed-expert grouped GEMM (gate_up + activation + down, effective single-rank shapes) with no comms. ", -) - -register(MOE_GEMM) diff --git a/operatorx/ops/moe.py b/operatorx/ops/moe.py new file mode 100644 index 0000000000..3644a6fccd --- /dev/null +++ b/operatorx/ops/moe.py @@ -0,0 +1,181 @@ +from __future__ import annotations + +from dataclasses import dataclass +from typing import Any + +from operatorx.core import parallel +from operatorx.core.op import OpSpec +from operatorx.core.op_registry import register +from operatorx.ops.gemm import ELEMENT_DTYPES, check_operand + +SCORING = {"softmax", "sigmoid", "sqrtsoftplus"} +ACTIVATIONS = {"silu", "gelu", "gelu_tanh", "swigluoai", "situ", "swiglustep"} +GATE_DTYPES = {"bf16", "fp32"} +DISTRIBUTIONS = {"natural", "balanced", "single_hot"} +# vLLM's FusedMoEQuantConfig slots +EXPERT_OPERANDS = ("a1", "w1", "w2", "a2") +SHARED_OPERANDS = ("a1", "w1", "w2") + + +def _keys(d: Any, where: str, required: set[str], optional: set[str] = frozenset()) -> None: + if not isinstance(d, dict): + raise ValueError(f"{where} must be a dict, got {d!r}") + missing, extra = required - set(d), set(d) - required - set(optional) + if missing or extra: + raise ValueError(f"{where}: missing {sorted(missing)}, unknown {sorted(extra)}") + + +def _pos_int(v: Any, where: str) -> None: + if not isinstance(v, int) or isinstance(v, bool) or v < 1: + raise ValueError(f"{where} must be a positive int, got {v!r}") + + +def _number(v: Any, where: str) -> None: + if v is not None and (not isinstance(v, (int, float)) or isinstance(v, bool)): + raise ValueError(f"{where} must be a number or null, got {v!r}") + + +def _check_operands(d: dict, where: str, operands: tuple[str, ...]) -> None: + for name in operands: + check_operand(d[name], f"{where}.{name}") + + +def _check_experts(e: Any) -> None: + _keys(e, "experts", {"num", "top_k", "inter", *EXPERT_OPERANDS}, {"bias", "latent", "latent_norm", "zero"}) + for k in ("num", "top_k", "inter"): + _pos_int(e[k], f"experts.{k}") + if e["top_k"] > e["num"]: + raise ValueError("experts.top_k exceeds experts.num") + if e.get("latent") is not None: + _pos_int(e["latent"], "experts.latent") + if not isinstance(e.get("zero", 0), int) or e.get("zero", 0) < 0: + raise ValueError("experts.zero must be a non-negative int") + for k in ("bias", "latent_norm"): + if not isinstance(e.get(k, False), bool): + raise ValueError(f"experts.{k} must be a bool") + if e.get("latent_norm") and not e.get("latent"): + raise ValueError("experts.latent_norm needs experts.latent") + _check_operands(e, "experts", EXPERT_OPERANDS) + + +def _check_router(r: Any, num_experts: int) -> None: + _keys(r, "router", {"gate", "scoring", "select"}, {"bias", "renormalize", "scale", "weight_on_input"}) + _keys(r["gate"], "router.gate", {"dtype"}, {"logits"}) + for k in ("dtype", "logits"): + if r["gate"].get(k, "bf16") not in GATE_DTYPES: + raise ValueError(f"router.gate.{k} must be one of {sorted(GATE_DTYPES)}") + if r["scoring"] not in SCORING: + raise ValueError(f"router.scoring must be one of {sorted(SCORING)}") + s = r["select"] + kind = s.get("kind") if isinstance(s, dict) else None + if kind == "topk": + _keys(s, "router.select", {"kind"}) + elif kind == "grouped_topk": + _keys(s, "router.select", {"kind", "groups", "topk_groups"}) + _pos_int(s["groups"], "router.select.groups") + _pos_int(s["topk_groups"], "router.select.topk_groups") + if num_experts % s["groups"] or s["topk_groups"] > s["groups"]: + raise ValueError("router.select: groups must divide experts.num and topk_groups <= groups") + elif kind == "hash": + _keys(s, "router.select", {"kind", "vocab"}) + _pos_int(s["vocab"], "router.select.vocab") + else: + raise ValueError(f"router.select.kind must be topk, grouped_topk or hash, got {s!r}") + for k in ("bias", "renormalize", "weight_on_input"): + if not isinstance(r.get(k, False), bool): + raise ValueError(f"router.{k} must be a bool") + _number(r.get("scale"), "router.scale") + + +def _check_activation(a: Any) -> None: + _keys(a, "activation", {"kind"}, {"gated", "interleaved", "limit", "alpha", "beta"}) + if a["kind"] not in ACTIVATIONS: + raise ValueError(f"activation.kind must be one of {sorted(ACTIVATIONS)}") + for k, default in (("gated", True), ("interleaved", False)): + if not isinstance(a.get(k, default), bool): + raise ValueError(f"activation.{k} must be a bool") + for k in ("limit", "alpha", "beta"): + _number(a.get(k), f"activation.{k}") + + +def _check_shared(s: Any) -> None: + _keys(s, "shared", {"count", "inter", *SHARED_OPERANDS}, {"gate"}) + _pos_int(s["count"], "shared.count") + _pos_int(s["inter"], "shared.inter") + if s.get("gate") not in (None, "sigmoid"): + raise ValueError("shared.gate must be null or 'sigmoid'") + _check_operands(s, "shared", SHARED_OPERANDS) + + +def _check_routing(r: Any) -> None: + _keys(r, "routing", {"distribution", "seed"}) + d = r["distribution"] + if isinstance(d, dict): + _keys(d, "routing.distribution", {"kind", "s"}) + if d["kind"] != "zipf": + raise ValueError("routing.distribution dict must be {'kind': 'zipf', 's': ...}") + _number(d["s"], "routing.distribution.s") + elif d not in DISTRIBUTIONS: + raise ValueError(f"routing.distribution must be one of {sorted(DISTRIBUTIONS)} or a zipf dict") + if not isinstance(r["seed"], int) or isinstance(r["seed"], bool): + raise ValueError("routing.seed must be an int") + + +@dataclass(frozen=True) +class MoeArgs: + """y[T,H] = shared(x) + sum over the selected experts e of w_e * expert_e(x). + + From the router GEMM on normed x [T, H] to the combined output (residual/norms outside). + Operands are gemm operand descriptors. + + experts: {"num": E, "top_k": K, "inter": I, "a1", "w1", "w2", "a2", + "bias"?: bool, "latent"?: L, "latent_norm"?: bool, "zero"?: n} + a1 expert input, w1 fused gate/up [2I, H], w2 down [H, I], a2 intermediate activation. + latent: experts at width L with bf16 H->L / L->H projections (latent_norm: RMSNorm + before L->H). zero: identity experts. + router: {"gate": {"dtype", "logits"?}, "scoring": softmax|sigmoid|sqrtsoftplus, + "select": {"kind": "topk"} | {"kind": "grouped_topk", "groups", "topk_groups"} + | {"kind": "hash", "vocab"}, + "bias"?: score-correction bias, "renormalize"?, "scale"?: routed scaling factor, + "weight_on_input"?: router weight applied to the expert input} + gate.dtype: router weight dtype; gate.logits: logits dtype (default gate.dtype). + activation: {"kind", "gated"?: default true, "interleaved"?: gate/up rows interleaved + in w1 (default false: [gate; up]), "limit"?, "alpha"?, "beta"?} + swigluoai: alpha scales the gate sigmoid, beta offsets up, limit clamps. situ: alpha and beta + soft-cap the gate and up halves (alpha*tanh(g/alpha)*sigmoid(g) * beta*tanh(u/beta)). + shared: null | {"count", "inter", "a1", "w1", "w2", "gate"?: null|"sigmoid"} + routing: {"distribution": "natural" | "balanced" | "single_hot" | {"kind": "zipf", "s"}, "seed"} + natural: the router's own choice on seeded random inputs; otherwise forced: balanced + evenly, zipf p ~ 1/rank^s, single_hot expert 0 + top_k-1 random. Weights stay the router's. + parallel: {"tp", "dp", "ep"} - tokens is each dp group's batch; hidden/experts the full layer. + """ + tokens: int + hidden: int + experts: dict + router: dict + activation: dict + shared: dict | None = None + routing: dict | None = None + out: str = "bf16" + + def __post_init__(self): + _pos_int(self.tokens, "tokens") + _pos_int(self.hidden, "hidden") + _check_experts(self.experts) + _check_router(self.router, self.experts["num"]) + _check_activation(self.activation) + if self.shared is not None: + _check_shared(self.shared) + _check_routing(self.routing if self.routing is not None else {"distribution": "natural", "seed": 0}) + if self.out not in ELEMENT_DTYPES: + raise ValueError(f"out {self.out!r} not in {sorted(ELEMENT_DTYPES)}") + + +MOE = OpSpec( + type="moe", + arg_schema=MoeArgs, + description="y = shared(x) + sum_{e in topk(route(x))} w_e * expert_e(x)", + parallel_axes=parallel.MOE_AXES, +) + +register(MOE) diff --git a/operatorx/platforms.json b/operatorx/platforms.json index 0eea726729..c900052503 100644 --- a/operatorx/platforms.json +++ b/operatorx/platforms.json @@ -1,20 +1,26 @@ { "base": "../collectivex/configs/platform_config.json", "platforms": { - "mi300x": { + "cluster:mi300x-amd": { "arch": "gfx942", "gpus_per_node": 8, "image_platform": "linux/amd64", - "operator": {"partition": "compute", "cpus_per_node": 128} + "operator": { + "cpus_per_node": 128 + } }, - "mi325x": { + "cluster:mi325x-amds": { "arch": "gfx942", "gpus_per_node": 8, "image_platform": "linux/amd64", - "operator": {"partition": "compute", "cpus_per_node": 256} + "operator": { + "cpus_per_node": 256 + } }, - "mi355x": { - "operator": {"partition": "compute", "cpus_per_node": 128} + "cluster:mi355x-amds": { + "operator": { + "cpus_per_node": 128 + } } } } diff --git a/operatorx/recipes.py b/operatorx/recipes.py new file mode 100644 index 0000000000..c229e26721 --- /dev/null +++ b/operatorx/recipes.py @@ -0,0 +1,139 @@ +"""The InferenceX recipe a case runs under. + +InferenceX serves every single-node configuration from a configs/*-master.yaml entry (model, +image, runner) whose search space names native srt-slurm recipes +(inferencex-e2e/benchmarks/single_node/srt-slurm-recipes//). +A case takes the recipe InferenceX runs for its source checkpoint on the runner's +hardware - the variant with the case's parallel split, else any of the checkpoint's - for +the recipe's image, launch env and server arguments. InferenceX's own code expands a +recipe into its variants (infx.srt_slurm.synthetic_acceptance.selected_recipes). +Without a recipe the case runs on the backend's default image. +""" +from __future__ import annotations + +import functools +import json +import sys +from pathlib import Path +from typing import Any + +from operatorx.core import parallel + +REPO = Path(__file__).resolve().parents[1] / "inferencex-e2e" # infx, its recipes and srt-slurm +RECIPES = REPO / "benchmarks/single_node/srt-slurm-recipes" +FRAMEWORKS = ("vllm",) + +# Server arguments a layer benchmark does not apply: artifacts it never loads (weights, +# tokenizer and the parsers built on it, draft model), the parallel layout (a case's own), +# the KV tiers and prefix cache of a long-running server, and the frontend. +_NOT_APPLIED = {"model", "tokenizer", "tokenizer-mode", "reasoning-parser", "tool-call-parser", + "enable-auto-tool-choice", "speculative-config", "served-model-name", "load-format", + "safetensors-load-strategy", "skip-tokenizer-init", "tensor-parallel-size", + "data-parallel-size", "pipeline-parallel-size", "enable-expert-parallel", + "expert-parallel-size", "decode-context-parallel-size", "dcp-comm-backend", + "distributed-executor-backend", "kv-transfer-config", "kv-events-config", + "enable-prefix-caching", "prefix-match-unit", "enable-cumem-allocator", + "gpu-memory-utilization", "kv-cache-memory-bytes", "max-num-seqs", "max-model-len", + "enforce-eager"} + + +@functools.cache +def _entries(framework: str, hardware: str) -> tuple[dict, ...]: + """The single-node configs/*-master.yaml entries for a framework on a hardware family.""" + import yaml + + out = [] + for master in sorted((REPO / "configs").glob("*-master.yaml")): + for entry in (yaml.safe_load(master.read_text()) or {}).values(): + if (isinstance(entry, dict) and not entry.get("multinode") and entry.get("framework") == framework + and entry.get("runner", "").removeprefix("cluster:").split("-")[0] == hardware): + out.append(entry) + return tuple(out) + + +def default_image(framework: str, hardware: str) -> str | None: + """The image InferenceX serves this framework with on this hardware family: the one most + of its entries use (the first listed on a tie).""" + images = [e["image"] for e in _entries(framework, hardware) if e.get("image")] + return max(dict.fromkeys(images), key=images.count) if images else None + + +@functools.cache +def _variants(framework: str, hardware: str) -> tuple[tuple[str, dict], ...]: + """Every expanded variant InferenceX runs for a framework on a hardware family: the + single-node entries of configs/*-master.yaml (model, image, runner) and the srt-slurm + recipes their search spaces name, expanded by InferenceX's own code.""" + import yaml + + for path in (str(REPO), str(REPO / "utils/srt-slurm/src")): + if path not in sys.path: + sys.path.append(path) + from infx.srt_slurm.synthetic_acceptance import selected_recipes + + out, seen = [], set() + for entry in _entries(framework, hardware): + if not entry.get("srt-recipe-dir"): + continue + refs = {s["srt-recipe"] for arms in (entry.get("scenarios") or {}).values() for arm in arms or () + for s in arm.get("search-space") or () if s.get("srt-recipe")} + for ref in sorted(refs): + path, _, selector = ref.partition(":") + f = RECIPES / entry["srt-recipe-dir"] / path + key = (str(f), selector, entry["model"], entry["image"]) + if key in seen or not f.is_file(): + continue + seen.add(key) + for name, recipe in selected_recipes(yaml.safe_load(f.read_text()), selector or None): + recipe = {**recipe, "model": {"path": f"hf:{entry['model']}", "container": entry["image"]}} + out.append((str(f.relative_to(REPO)) + (f":{name}" if name else ""), recipe)) + return tuple(out) + + +def _vllm_split(args: dict[str, Any]) -> dict: + tp, dp = int(args.get("tensor-parallel-size", 1)), int(args.get("data-parallel-size", 1)) + return {"tp": tp, "dp": dp, "ep": tp * dp if args.get("enable-expert-parallel") else 1, + "dcp": int(args.get("decode-context-parallel-size", 1))} + + +# per framework: the parallel split its serve arguments spell +SPLITS = {"vllm": _vllm_split} + + +def find(framework: str, hardware: str, checkpoints: list[str], split: dict | None = None, + axes: tuple[str, ...] = parallel.AXES) -> dict | None: + """The recipe InferenceX runs for the first of the checkpoints served on this hardware, + preferring a variant whose split equals the case's on `axes`.""" + if framework not in FRAMEWORKS or not RECIPES.is_dir(): + return None + want = parallel.normalize(split) + variants = _variants(framework, hardware) + for ckpt in checkpoints: + mine = [(k, r) for k, r in variants if r["model"]["path"] == f"hf:{ckpt}"] + exact = [(k, r) for k, r in mine + if all(SPLITS[framework](r["roles"]["agg"]["args"])[a] == want[a] for a in axes)] + for match, pool in (("split", exact), ("checkpoint", mine)): + if pool: + key, r = pool[0] + role = r["roles"]["agg"] + return {"recipe": key, "match": match, "checkpoint": ckpt, "image": r["model"]["container"], + "env": {k: str(v) for k, v in (role.get("env") or {}).items()}, + "engine_args": role.get("args") or {}} + return None + + +def checkpoints(sources) -> list[str]: + """The distinct checkpoints ("org/model") a case's "//" sources name.""" + return list(dict.fromkeys(s.rsplit("/", 1)[0] for s in sources or ())) + + +def serve_argv(args: dict[str, Any]) -> list[str]: + """A recipe's server arguments as `vllm serve` command-line arguments, less the ones a + layer benchmark does not apply.""" + out = [] + for k, v in args.items(): + if k in _NOT_APPLIED or v is None or v is False: + continue + out.append(f"--{k}") + if v is not True: + out.append(v if isinstance(v, str) else json.dumps(v)) + return out diff --git a/operatorx/runners/amd/backends/aiter.py b/operatorx/runners/amd/backends/aiter.py deleted file mode 100644 index 2d07d4c6c5..0000000000 --- a/operatorx/runners/amd/backends/aiter.py +++ /dev/null @@ -1,41 +0,0 @@ -"""AITER fused MHA/GQA and materialized MLA attention (no torch fallback).""" - -from __future__ import annotations - -import aiter - -from operatorx.core import BackendImpl, Op, UnsupportedOpError, lookup_versions -from operatorx.runners import attention - - -def versions() -> dict[str, str]: - return lookup_versions("amd-aiter", "torch") - - -def prepare(op: Op) -> dict: - a = op.args - if a["dtype_q"] not in {"bf16", "fp16"}: - raise UnsupportedOpError("AITER attention currently supports bf16/fp16") - dq = ( - a["head_dim"] - if op.type == "attention_mha" - else a["head_dim_qk_nope"] + a["head_dim_qk_rope"] - ) - dv = a["head_dim"] if op.type == "attention_mha" else a["head_dim_v"] - if dq > 256 or dv > 256 or dq % 8 or dv % 8: - raise UnsupportedOpError( - "AITER attention requires head dimensions divisible by 8 and <=256" - ) - return attention.prepare(op, layout="bshd") - - -def kernel(ctx: dict) -> None: - ctx["out"] = aiter.flash_attn_func( - ctx["q"], ctx["k"], ctx["v"], dropout_p=0.0, causal=ctx["causal"] - ) - - -IMPLS = [ - BackendImpl(op_type=name, prepare=prepare, kernel=kernel) - for name in ("attention_mha", "attention_mla") -] diff --git a/operatorx/runners/amd/backends/torch.py b/operatorx/runners/amd/backends/torch.py index 4f361668ba..520f4e0821 100644 --- a/operatorx/runners/amd/backends/torch.py +++ b/operatorx/runners/amd/backends/torch.py @@ -1,11 +1,9 @@ -"""Dense GEMM through ROCm PyTorch, including architecture-correct FP8.""" +"""Dense GEMM through ROCm PyTorch (FP8 dtype chosen per gfx arch).""" from __future__ import annotations import torch -from operatorx.runners import attention - from operatorx.core import BackendImpl, Op, UnsupportedOpError, lookup_versions @@ -17,7 +15,7 @@ def resolve_dtype(name: str) -> torch.dtype: ordinary = {"bf16": torch.bfloat16, "fp16": torch.float16, "fp32": torch.float32} if name in ordinary: return ordinary[name] - if name == "fp8": + if name == "e4m3": arch = torch.cuda.get_device_properties(torch.cuda.current_device()).gcnArchName arch = arch.split(":")[0] if arch == "gfx942": @@ -32,15 +30,24 @@ def prepare(op: Op) -> dict: a = op.args if a.get("activation") is not None: raise UnsupportedOpError("ROCm torch GEMM has no native activation fusion") - if a["dtype_a"] != a["dtype_b"]: + qa, qb = a["a"], a["b"] + if qa["dtype"] != qb["dtype"]: raise UnsupportedOpError("ROCm torch GEMM requires matching input dtypes") - dtype = resolve_dtype(a["dtype_a"]) - out_name = a.get("dtype_out") or "bf16" + fp8 = qa["dtype"] == "e4m3" + # _scaled_mm here takes one static fp32 scale per operand; unscaled GEMMs take none + per_tensor = {"dtype": "fp32", "static": True, "group": [-1, -1]} + # the FP8 path multiplies pre-quantized operands: activation quantization is not in the op + want = (({"dtype": "e4m3", "scale": per_tensor, "input": "e4m3"}, {"dtype": "e4m3", "scale": per_tensor}) + if fp8 else ({"dtype": qa["dtype"]},) * 2) + if (qa, qb) != want: + raise UnsupportedOpError(f"ROCm torch GEMM supports unscaled or static per-tensor fp8; got a={qa} b={qb}") + dtype = resolve_dtype(qa["dtype"]) + out_name = a.get("out") or "bf16" if out_name not in {"bf16", "fp16", "fp32"}: raise UnsupportedOpError(f"unsupported GEMM output dtype={out_name!r}") out_dtype = resolve_dtype(out_name) m, n, k = a["m"], a["n"], a["k"] - if a["dtype_a"] == "fp8": + if fp8: left = torch.randn(m, k, device="cuda", dtype=torch.bfloat16).to(dtype) right = torch.randn(n, k, device="cuda", dtype=torch.bfloat16).to(dtype).t() scale_a = torch.tensor(1.0, device="cuda") @@ -59,7 +66,7 @@ def prepare(op: Op) -> dict: "out_dtype": out_dtype, "scale_a": scale_a, "scale_b": scale_b, - "fp8": a["dtype_a"] == "fp8", + "fp8": fp8, } @@ -81,10 +88,4 @@ def kernel(ctx: dict) -> None: IMPLS = [ BackendImpl(op_type="gemm", prepare=prepare, kernel=kernel), - BackendImpl( - op_type="attention_mha", prepare=attention.prepare, kernel=attention.kernel - ), - BackendImpl( - op_type="attention_mla", prepare=attention.prepare, kernel=attention.kernel - ), ] diff --git a/operatorx/runners/amd/backends/vllm.py b/operatorx/runners/amd/backends/vllm.py index d029ff36b2..80dab41e40 100644 --- a/operatorx/runners/amd/backends/vllm.py +++ b/operatorx/runners/amd/backends/vllm.py @@ -1,5 +1,11 @@ -"""vLLM routed expert kernels on ROCm.""" +"""GEMM, MoE and attention through vLLM's layers (ROCm), AITER on as in InferenceX's ROCm launches.""" +import os -from operatorx.runners.moe import IMPLS, versions +os.environ.setdefault("VLLM_ROCM_USE_AITER", "1") + +from operatorx.runners.common.vllm import attention, linear, moe # noqa: E402 +from operatorx.runners.common.vllm.linear import versions # noqa: E402 + +IMPLS = [*linear.IMPLS, *moe.IMPLS, *attention.IMPLS] __all__ = ["IMPLS", "versions"] diff --git a/operatorx/runners/amd/runner.py b/operatorx/runners/amd/runner.py index 83c2b07ee3..7384019aaf 100644 --- a/operatorx/runners/amd/runner.py +++ b/operatorx/runners/amd/runner.py @@ -2,19 +2,25 @@ from __future__ import annotations +import os +import time from importlib import import_module import torch from operatorx.core import Op, Result, UnsupportedOpError +from operatorx.runners.common import profiling, ranks, telemetry, timing + +_COOLDOWN_RATIO = float(os.environ.get("OPERATORX_COOLDOWN_RATIO", "4")) +_COOLDOWN_MAX_S = float(os.environ.get("OPERATORX_COOLDOWN_MAX_S", "1.0")) +# ROCm reports only the per-XCD L2 slice: size the flush buffer for the whole hierarchy +_FLUSH_MB = int(os.environ.get("OPERATORX_FLUSH_MB", "512")) _L2_BUF: dict[int, torch.Tensor] = {} -_WARMUP = 5 -_ITERS = 10 def run(op: Op) -> Result: - if op.backend not in {"torch", "aiter", "vllm"}: + if op.backend not in ("torch", "vllm"): raise UnsupportedOpError(f"unknown AMD backend: {op.backend}") backend = import_module(f"operatorx.runners.amd.backends.{op.backend}") impl = next((item for item in backend.IMPLS if item.op_type == op.type), None) @@ -27,18 +33,26 @@ def run(op: Op) -> Result: size = torch.cuda.get_device_properties(device).L2_cache_size if size <= 0: raise RuntimeError("ROCm did not report a positive L2 cache size") + size = max(size, _FLUSH_MB << 20) _L2_BUF[device] = torch.empty(size, dtype=torch.int8, device="cuda") ctx = impl.prepare(op) - for _ in range(_WARMUP): - impl.kernel(ctx) - torch.cuda.synchronize() - starts = [torch.cuda.Event(enable_timing=True) for _ in range(_ITERS)] - ends = [torch.cuda.Event(enable_timing=True) for _ in range(_ITERS)] - for start, end in zip(starts, ends): - _L2_BUF[device].zero_() - start.record() - impl.kernel(ctx) - end.record() - torch.cuda.synchronize() - times = sorted(start.elapsed_time(end) * 1000.0 for start, end in zip(starts, ends)) - return Result(op=op, metrics={"latency_us": times[_ITERS // 2]}) + + fn, cuda_graph = impl.launcher(ctx) if impl.launcher else ((lambda: impl.kernel(ctx)), False) + median_us, telem = telemetry.measure( + op, lambda sleep_s: timing.time_op(fn, _L2_BUF[device].zero_, sleep_s)) + + if _COOLDOWN_RATIO > 0.0: + time.sleep(min(median_us * 1e-6 * (timing.ITERS + timing.WARMUP) * _COOLDOWN_RATIO, + _COOLDOWN_MAX_S)) + metrics = {"latency_us": median_us, "cuda_graph": cuda_graph, "telemetry": telem} + if isinstance(ctx, dict) and ctx.get("meta"): + metrics["backend_meta"] = ctx["meta"] + prof = profiling.profile_op(fn) + if prof is not None: + metrics["profile"] = prof + per_rank = ranks.summary(metrics) # every rank's timeline and telemetry, on rank 0 + if per_rank is not None: + metrics["ranks"] = per_rank + if per_rank["capped_ranks"] and metrics.get("telemetry"): + metrics["telemetry"]["capped"] = True # any rank capped caps the op + return Result(op=op, metrics=metrics) diff --git a/operatorx/runners/attention.py b/operatorx/runners/attention.py deleted file mode 100644 index f6345a304b..0000000000 --- a/operatorx/runners/attention.py +++ /dev/null @@ -1,74 +0,0 @@ -"""Shared contiguous attention inputs and bottom-right causal SDPA semantics. - -MLA measures materialized Q/K/V attention only; cache projection and RoPE are -outside this operator. MHA expands grouped KV heads outside the torch timer. -""" - -from __future__ import annotations - -import torch -import torch.nn.functional as F - -from operatorx.core import Op, UnsupportedOpError - - -def prepare(op: Op, *, layout: str = "bhsd") -> dict: - a = op.args - if a.get("kv_layout", "contig") != "contig" or a.get("sliding_window") is not None: - raise UnsupportedOpError( - "attention requires contiguous KV without a sliding window" - ) - mla = op.type == "attention_mla" - names = ( - [a["dtype_q"], a["dtype_kv"]] - if mla - else [a[k] for k in ("dtype_q", "dtype_k", "dtype_v")] - ) - names.append(a.get("dtype_o", "bf16")) - if len(set(names)) != 1 or names[0] not in {"bf16", "fp16", "fp32"}: - raise UnsupportedOpError( - "attention requires uniform bf16/fp16/fp32 inputs and output" - ) - dtype = {"bf16": torch.bfloat16, "fp16": torch.float16, "fp32": torch.float32}[ - names[0] - ] - b, sq, sk, h = ( - a[k] for k in ("batch_size", "seq_len_q", "seq_len_kv", "num_heads") - ) - hkv = h if mla else a["num_heads_kv"] - dq = a["head_dim_qk_nope"] + a["head_dim_qk_rope"] if mla else a["head_dim"] - dv = a["head_dim_v"] if mla else dq - if ( - any(type(n) is not int or n <= 0 for n in (b, sq, sk, h, hkv, dq, dv)) - or h % hkv - ): - raise ValueError( - "attention dimensions must be positive; query heads must divide by KV heads" - ) - dims = ((b, h, sq, dq), (b, hkv, sk, dq), (b, hkv, sk, dv)) - if layout == "bshd": - dims = tuple((batch, seq, heads, dim) for batch, heads, seq, dim in dims) - elif layout != "bhsd": - raise ValueError(f"invalid attention layout: {layout}") - q, k, v = (torch.randn(*shape, dtype=dtype, device="cuda") for shape in dims) - causal = bool(a.get("causal", True)) - ctx = {"q": q, "k": k, "v": v, "causal": causal, "mask": None} - if layout == "bhsd": - if hkv != h: - ctx["k"] = k.repeat_interleave(h // hkv, dim=1) - ctx["v"] = v.repeat_interleave(h // hkv, dim=1) - # PyTorch is_causal aligns upper-left. Decode queries refer to the last - # positions of the KV sequence, so rectangular masks align bottom-right. - if causal and sq != sk: - ctx["causal"] = False - if sq != 1 or sq > sk: - ctx["mask"] = torch.ones(sq, sk, dtype=torch.bool, device="cuda").tril( - sk - sq - ) - return ctx - - -def kernel(ctx: dict) -> None: - ctx["out"] = F.scaled_dot_product_attention( - ctx["q"], ctx["k"], ctx["v"], attn_mask=ctx["mask"], is_causal=ctx["causal"] - ) diff --git a/operatorx/runners/common/__init__.py b/operatorx/runners/common/__init__.py new file mode 100644 index 0000000000..bd63371b62 --- /dev/null +++ b/operatorx/runners/common/__init__.py @@ -0,0 +1 @@ +"""Code shared by the platform runners.""" diff --git a/operatorx/runners/common/collectivex.py b/operatorx/runners/common/collectivex.py new file mode 100644 index 0000000000..90683e565e --- /dev/null +++ b/operatorx/runners/common/collectivex.py @@ -0,0 +1,26 @@ +"""CollectiveX modules OperatorX reuses as-is (the sibling collectivex/ tree).""" +from __future__ import annotations + +import functools +import importlib.util +import sys +from pathlib import Path + +_ROOT = Path(__file__).resolve().parents[3] / "collectivex" + + +def _load(name: str, rel: str): + folder = str((_ROOT / rel).parent) # its sibling imports (ep_harness -> ep_oracle) + if folder not in sys.path: + sys.path.append(folder) + spec = importlib.util.spec_from_file_location(name, _ROOT / rel) + module = importlib.util.module_from_spec(spec) + sys.modules[name] = module # its dataclasses resolve their module by name + spec.loader.exec_module(module) + return module + + +@functools.cache +def harness(): + return _load("collectivex_ep_harness", "bench/ep_harness.py") + diff --git a/operatorx/runners/common/profiling.py b/operatorx/runners/common/profiling.py new file mode 100644 index 0000000000..d22142ca1a --- /dev/null +++ b/operatorx/runners/common/profiling.py @@ -0,0 +1,192 @@ +"""Per-op kernel breakdown under torch.profiler (CUPTI / rocprofiler), after timing. + +metrics["profile"]: iters, op_index, kernels [{name, cat, count_per_call, us_per_call, grid, block, +regs, smem, blocks_per_sm, warps_per_sm, occupancy_pct}] (by time; launch fields where reported), +gpu_us_per_call, span/busy/gap/overlap_us, streams, timeline (median-span replay), trace. + +Env OPERATORX_PROFILE_*: =0 off; _ITERS (3); _FLUSH_MB per replay (512, 0 = warm); _TRACE_DIR, +_TRACE_EVERY (200); _MARKERS=1: nvtx/roctx range "opx" for ncu/rocprofv3 instead. +""" +from __future__ import annotations + +import os +import shutil +import tempfile + +import torch + +from operatorx.runners.common import trace + +PROFILE = os.environ.get("OPERATORX_PROFILE", "1") == "1" +_ITERS = int(os.environ.get("OPERATORX_PROFILE_ITERS", "3")) +_FLUSH_MB = int(os.environ.get("OPERATORX_PROFILE_FLUSH_MB", "512")) +_TRACE_DIR = os.environ.get("OPERATORX_PROFILE_TRACE_DIR") or None +_TRACE_EVERY = int(os.environ.get("OPERATORX_PROFILE_TRACE_EVERY", "200")) +_MARKERS = os.environ.get("OPERATORX_PROFILE_MARKERS", "") == "1" +# name of the kernel the int8 zero_() flush dispatches +_FLUSH_KERNEL_MARKER = "FillFunctor" +# spin after each flush so launches queue behind it and the timeline shows device time, +# not host gaps; torch.cuda._sleep's kernel +_SHIELD_CYCLES = int(os.environ.get("OPERATORX_SHIELD_CYCLES", "4000000")) +_SHIELD_KERNEL_MARKER = "spin_kernel" + +_counter = 0 + + +class _Cuda: + """CUDA and ROCm: kineto kernel events, replays delimited by the harness.""" + + iters = _ITERS + arg_fields = ( + ("grid", "grid"), + ("block", "block"), + ("registers per thread", "regs"), + ("shared memory", "smem"), + ("blocks per SM", "blocks_per_sm"), + ("warps per SM", "warps_per_sm"), + ("est. achieved occupancy %", "occupancy_pct"), + ) + _flush_buf = None + + def activities(self): + return [torch.profiler.ProfilerActivity.CPU, torch.profiler.ProfilerActivity.CUDA] + + def experimental_config(self): + return None + + def flush(self) -> None: + if _FLUSH_MB <= 0: + return + if _Cuda._flush_buf is None: + _Cuda._flush_buf = torch.empty(_FLUSH_MB << 20, dtype=torch.int8, device="cuda") + _Cuda._flush_buf.zero_() + + def pre_replay(self) -> None: + self.flush() + if _SHIELD_CYCLES > 0: + torch.cuda._sleep(_SHIELD_CYCLES) + + def sync(self) -> None: + torch.cuda.synchronize() + + def stream_of(self, e): + return (e.get("args") or {}).get("stream") + + def cat_of(self, e): + return e["cat"] + + def name_of(self, e): + return e.get("name", "")[:200] + + def harness_events(self, events: list[dict]) -> tuple[set[int], set[int]]: + """(flush, spin) event ids. The op may launch fills itself, so the flush is the last fill + before each spin (flush, spin, replay); without a spin every fill is a flush.""" + flush, spin, last_fill = set(), set(), None + for i, e in enumerate(events): + name = e.get("name", "") + if _SHIELD_KERNEL_MARKER in name: + spin.add(i) + if last_fill is not None: + flush.add(last_fill) + last_fill = None + elif _FLUSH_MB > 0 and _FLUSH_KERNEL_MARKER in name: + last_fill = i + if _SHIELD_CYCLES <= 0: + flush.add(i) + return flush, spin + + def decompose(self, events: list[dict], iters: int) -> tuple[list[dict], list[list[dict]], int]: + device = sorted((e for e in events + if e.get("ph") == "X" and e.get("cat") in ("kernel", "gpu_memcpy", "gpu_memset")), + key=trace.ts) + flush, spin = self.harness_events(device) + kernels = [e for i, e in enumerate(device) if i not in flush and i not in spin] + replays = None + if _FLUSH_MB > 0: + bounds = spin if spin else flush + replays, cur = [], [] + for i, e in enumerate(device): + if i in bounds: + if cur: + replays.append(cur) + cur = [] + elif i not in flush and i not in spin: + cur.append(e) + if cur: + replays.append(cur) + return kernels, (replays or []), len(flush) + + +def _platform(): + return _Cuda() + + +def _markers_pass(kernel_fn, plat) -> dict: + marker = f"opx{_counter:06d}" + plat.flush() # outside the range: not attributed + torch.cuda.synchronize() + torch.cuda.nvtx.range_push(marker) + try: + for _ in range(_ITERS): + kernel_fn() + torch.cuda.synchronize() + finally: + torch.cuda.nvtx.range_pop() + return {"iters": _ITERS, "op_index": _counter, "marker": marker} + + +def profile_op(kernel_fn) -> dict | None: + global _counter + if not PROFILE: + return None + _counter += 1 + plat = _platform() + iters = plat.iters + if _MARKERS: + try: + return _markers_pass(kernel_fn, plat) + except Exception as e: + return {"error": f"{type(e).__name__}: {e}"[:200], "op_index": _counter} + + try: + with torch.profiler.profile(activities=plat.activities(), + experimental_config=plat.experimental_config()) as prof: + for _ in range(iters): + plat.pre_replay() + kernel_fn() + plat.sync() + except Exception as e: + return {"error": f"{type(e).__name__}: {e}"[:200]} + + fd, path = tempfile.mkstemp(suffix=".json") + os.close(fd) + try: + prof.export_chrome_trace(path) + events = trace.load_events(path) + except Exception as e: + os.unlink(path) + return {"error": f"trace: {type(e).__name__}: {e}"[:200]} + + kernels, replays, flush_excluded = plat.decompose(events, iters) + out = trace.aggregate(kernels, iters, plat.arg_fields, plat.cat_of, plat.name_of) + summary = {"iters": iters, "kernels": out, + "gpu_us_per_call": round(sum(k["us_per_call"] for k in out), 3), + "op_index": _counter} + if flush_excluded: + summary["flush_kernels_excluded"] = flush_excluded + stats = trace.replay_stats(replays, plat.stream_of) + if stats: + summary.update(stats) + + try: + # "== 1 % N" so that TRACE_EVERY=1 keeps every trace + if _TRACE_DIR and _counter % _TRACE_EVERY == 1 % _TRACE_EVERY: + os.makedirs(_TRACE_DIR, exist_ok=True) + dest = os.path.join(_TRACE_DIR, f"op{_counter:06d}.json") + shutil.move(path, dest) # tmp may be on another filesystem + summary["trace"] = dest + else: + os.unlink(path) + except OSError as e: + summary["trace_error"] = str(e)[:120] + return summary diff --git a/operatorx/runners/common/ranks.py b/operatorx/runners/common/ranks.py new file mode 100644 index 0000000000..d11f4cfc88 --- /dev/null +++ b/operatorx/runners/common/ranks.py @@ -0,0 +1,101 @@ +"""Agreement and reduction across ranks; no-ops on one rank. + +Every rank must take the same branch wherever a collective launches, or the others hang. +""" +from __future__ import annotations + +from contextlib import contextmanager + +import torch + +from operatorx.core import UnsupportedOpError + + +def _group(): + import torch.distributed as dist + return dist if dist.is_available() and dist.is_initialized() and dist.get_world_size() > 1 else None + + +def world() -> int: + d = _group() + return d.get_world_size() if d else 1 + + +def agree(ok: bool) -> bool: + """True only when every rank passes True.""" + d = _group() + if d is None: + return ok + t = torch.tensor([1 if ok else 0], dtype=torch.int32, device="cuda") + d.all_reduce(t, op=d.ReduceOp.MIN) + return bool(t.item()) + + +def any_(flag: bool) -> bool: + return not agree(not flag) + + +_TOKEN: dict[int, torch.Tensor] = {} + + +def align(cycles: int) -> None: + """Device-side rank barrier then a spin, without a host sync (CollectiveX _graph_align).""" + d = _group() + if d is None: + return + dev = torch.cuda.current_device() + if dev not in _TOKEN: + _TOKEN[dev] = torch.zeros(1, device="cuda") + d.all_reduce(_TOKEN[dev]) + torch.cuda._sleep(cycles) + + +@contextmanager +def together(what: str): + """Every rank reports once whether it got through; all stop if any did not.""" + try: + yield + except BaseException: + agree(False) + raise + if not agree(True): + raise UnsupportedOpError(f"another rank could not {what}") + + +LAST: dict = {} # the last timing's reduction, for summary() + + +def iterations(times: list[float]) -> dict: + """Per-iteration latencies reduced across ranks (CollectiveX ep_harness._Collectives.vec).""" + d = _group() + if d is None: + out = {"max": list(times), "min": list(times), "spread": [0.0] * len(times)} + else: + from operatorx.runners.common import collectivex + h = collectivex.harness() + dev = torch.device("cuda", torch.cuda.current_device()) + reduce = h._Collectives(torch, d, dev) + hi = reduce.vec(times, d.ReduceOp.MAX) + lo = reduce.vec(times, d.ReduceOp.MIN) + out = {"max": hi, "min": lo, "spread": [a - b for a, b in zip(hi, lo)]} + LAST.clear() + LAST.update(out, own=list(times)) + return out + + +def summary(metrics: dict) -> dict | None: + """Every rank's latency, timeline and telemetry, gathered on rank 0; None on one device.""" + d = _group() + if d is None: + return None + + def med(xs): + return sorted(xs)[len(xs) // 2] if xs else None + + mine = {"rank": d.get_rank(), "latency_us": med(LAST.get("own", [])), + "profile": metrics.get("profile"), "telemetry": metrics.get("telemetry")} + gathered = [None] * d.get_world_size() if d.get_rank() == 0 else None + d.gather_object(mine, gathered, dst=0) + capped = [p["rank"] for p in gathered or () if (p.get("telemetry") or {}).get("capped")] + return {"world": d.get_world_size(), "latency_us_min": med(LAST.get("min", [])), + "skew_us": med(LAST.get("spread", [])), "capped_ranks": capped, "per_rank": gathered} diff --git a/operatorx/runners/common/telemetry.py b/operatorx/runners/common/telemetry.py new file mode 100644 index 0000000000..70f8abfad5 --- /dev/null +++ b/operatorx/runners/common/telemetry.py @@ -0,0 +1,320 @@ +"""Per-op GPU telemetry (sampled every OPERATORX_TELEMETRY_MS) and the throttle retry. + +An attempt is capped when a power/thermal reason is active with the SM clock below +OPERATORX_CAP_CLOCK_FRACTION x rated boost (power cap at full boost is ordinary DVFS); +capped attempts retry with doubling sleeps (OPERATORX_RETRY_SLEEP_MS, up to +OPERATORX_THROTTLE_RETRIES) and the lowest median is kept. + +metrics["telemetry"]: provider, interval_ms, n_samples, window_ms, rated_sm_clock_mhz, +: {min, p50, max}, counters (window deltas), throttle_reasons, capped, attempts, +inter_kernel_sleep_ms. Raw samples: OPERATORX_TELEMETRY_DIR/telemetry-rank.jsonl. +""" +from __future__ import annotations + +import json +import os +import threading +import time + +import torch + +from operatorx.runners.common import ranks + +_INTERVAL_MS = float(os.environ.get("OPERATORX_TELEMETRY_MS", "5")) +_DIR = os.environ.get("OPERATORX_TELEMETRY_DIR") +_CAP_CLOCK_FRACTION = float(os.environ.get("OPERATORX_CAP_CLOCK_FRACTION", "0.985")) +_RETRIES = int(os.environ.get("OPERATORX_THROTTLE_RETRIES", "3")) +_RETRY_SLEEP_MS = float(os.environ.get("OPERATORX_RETRY_SLEEP_MS", "2")) + +_NVML_REASONS = { + 0x001: "GpuIdle", + 0x002: "ApplicationsClocksSetting", + 0x004: "SwPowerCap", + 0x008: "HwSlowdown", + 0x010: "SyncBoost", + 0x020: "SwThermalSlowdown", + 0x040: "HwThermalSlowdown", + 0x080: "HwPowerBrakeSlowdown", + 0x100: "DisplayClockSetting", +} +_NVML_CAP_REASONS = {"SwPowerCap", "HwSlowdown", "SwThermalSlowdown", + "HwThermalSlowdown", "HwPowerBrakeSlowdown"} +# cumulative nanosecond counters (field values), read at window start/end +_NVML_TIME_COUNTERS = { + "throttle_sw_power_cap_ms": "NVML_FI_DEV_CLOCKS_EVENT_REASON_SW_POWER_CAP", + "throttle_hw_slowdown_ms": "NVML_FI_DEV_CLOCKS_EVENT_REASON_HW_THERM_SLOWDOWN", + "throttle_sw_thermal_ms": "NVML_FI_DEV_CLOCKS_EVENT_REASON_SW_THERM_SLOWDOWN", + "throttle_hw_power_brake_ms": "NVML_FI_DEV_CLOCKS_EVENT_REASON_HW_POWER_BRAKE_SLOWDOWN", + "throttle_sync_boost_ms": "NVML_FI_DEV_CLOCKS_EVENT_REASON_SYNC_BOOST", + "limit_power_ms": "NVML_FI_DEV_PERF_POLICY_POWER", + "limit_thermal_ms": "NVML_FI_DEV_PERF_POLICY_THERMAL", + "limit_board_ms": "NVML_FI_DEV_PERF_POLICY_BOARD_LIMIT", + "limit_reliability_ms": "NVML_FI_DEV_PERF_POLICY_RELIABILITY", + "limit_low_utilization_ms": "NVML_FI_DEV_PERF_POLICY_LOW_UTILIZATION", + "below_app_clocks_ms": "NVML_FI_DEV_PERF_POLICY_TOTAL_APP_CLOCKS", + "below_base_clocks_ms": "NVML_FI_DEV_PERF_POLICY_TOTAL_BASE_CLOCKS", +} +# the driver advances these counters (and energy) in 100 ms steps +_COUNTER_MIN_WINDOW_NS = 1_000_000_000 + + +class _Provider: + + name = "null" + rated_mhz: int | None = None + cap_reasons: set[str] | None = None # None: every reported reason caps + + def __init__(self) -> None: + self._samples: list[dict] = [] + self._stop = threading.Event() + self._thread: threading.Thread | None = None + self._t0_ns = 0 + + def start(self) -> None: + self._samples = [] + self._t0_ns = time.time_ns() + self._stop.clear() + self._thread = threading.Thread(target=self._poll, daemon=True) + self._thread.start() + + def stop(self) -> list[dict]: + self._stop.set() + self._thread.join() + self._samples.extend(self._history(self._t0_ns)) + self._samples.sort(key=lambda s: s["t_ns"]) + return self._samples + + def counters(self) -> dict[str, float]: + return {} + + def _poll(self) -> None: + while not self._stop.is_set(): + try: + self._samples.append(self._sample()) + except Exception: + pass + self._stop.wait(_INTERVAL_MS / 1e3) + + def _sample(self) -> dict: + raise NotImplementedError + + def _history(self, since_ns: int) -> list[dict]: + return [] + + +class _Null(_Provider): + def start(self) -> None: + pass + + def stop(self) -> list[dict]: + return [] + + +class _Nvml(_Provider): + # FIXME(hbarclay): switch to DCGM for ~1 ms sampling. + name = "nvml" + cap_reasons = _NVML_CAP_REASONS + + def __init__(self, device: int) -> None: + super().__init__() + import pynvml as nv + nv.nvmlInit() + # NVML ignores CUDA_VISIBLE_DEVICES: match by UUID + uuid = str(torch.cuda.get_device_properties(device).uuid) + self._nv = nv + self._h = nv.nvmlDeviceGetHandleByUUID(uuid if uuid.startswith("GPU-") else f"GPU-{uuid}") + self._reasons = getattr(nv, "nvmlDeviceGetCurrentClocksEventReasons", None) \ + or nv.nvmlDeviceGetCurrentClocksThrottleReasons + self.rated_mhz = nv.nvmlDeviceGetMaxClockInfo(self._h, nv.NVML_CLOCK_SM) + self._fast_fields = [nv.NVML_FI_DEV_POWER_INSTANT, nv.NVML_FI_DEV_MEMORY_TEMP] + self._counter_fields = {k: getattr(nv, f) for k, f in _NVML_TIME_COUNTERS.items() + if hasattr(nv, f)} + self._counter_fields["energy_j"] = nv.NVML_FI_DEV_TOTAL_ENERGY_CONSUMPTION + + def _fields(self, ids: list[int]) -> list: + out = [] + for v in self._nv.nvmlDeviceGetFieldValues(self._h, ids): + if v.nvmlReturn != 0: + out.append(None) + elif v.valueType == 1: + out.append(v.value.uiVal) + else: + out.append(v.value.ullVal) + return out + + def _sample(self) -> dict: + nv, h = self._nv, self._h + mask = self._reasons(h) + util = nv.nvmlDeviceGetUtilizationRates(h) + power_inst, mem_temp = self._fields(self._fast_fields) + return { + "t_ns": time.time_ns(), + "sm_clock_mhz": nv.nvmlDeviceGetClockInfo(h, nv.NVML_CLOCK_SM), + "gr_clock_mhz": nv.nvmlDeviceGetClockInfo(h, nv.NVML_CLOCK_GRAPHICS), + "mem_clock_mhz": nv.nvmlDeviceGetClockInfo(h, nv.NVML_CLOCK_MEM), + "video_clock_mhz": nv.nvmlDeviceGetClockInfo(h, nv.NVML_CLOCK_VIDEO), + "power_w": nv.nvmlDeviceGetPowerUsage(h) / 1e3, + "power_instant_w": power_inst / 1e3 if power_inst is not None else None, + "gpu_temp_c": nv.nvmlDeviceGetTemperature(h, nv.NVML_TEMPERATURE_GPU), + "mem_temp_c": mem_temp, + "pstate": nv.nvmlDeviceGetPerformanceState(h), + "gpu_util_pct": util.gpu, + "mem_util_pct": util.memory, + "reasons": [n for bit, n in _NVML_REASONS.items() if mask & bit], + } + + def counters(self) -> dict[str, float]: + vals = self._fields(list(self._counter_fields.values())) + out = {} + for key, v in zip(self._counter_fields, vals): + if v is not None: + out[key] = v / 1e3 if key == "energy_j" else v / 1e6 # mJ, ns + return out + + def _history(self, since_ns: int) -> list[dict]: + """The driver's own sample buffers for the window.""" + nv, out = self._nv, [] + for kind, key, scale in ((nv.NVML_TOTAL_POWER_SAMPLES, "power_w", 1e-3), + (nv.NVML_GPU_UTILIZATION_SAMPLES, "gpu_util_pct", 1), + (nv.NVML_MEMORY_UTILIZATION_SAMPLES, "mem_util_pct", 1)): + try: + _, buf = nv.nvmlDeviceGetSamples(self._h, kind, since_ns // 1000) + except nv.NVMLError: + continue + for s in buf: + t = s.timeStamp * 1000 + if t >= since_ns: + out.append({"t_ns": t, key: s.sampleValue.uiVal * scale, "source": "driver"}) + return out + + +class _AmdSmi(_Provider): + name = "amdsmi" + + def __init__(self, device: int) -> None: + super().__init__() + import amdsmi + amdsmi.amdsmi_init() + # amdsmi ignores *_VISIBLE_DEVICES: match by PCI address + p = torch.cuda.get_device_properties(device) + bdf = f"{p.pci_domain_id:04x}:{p.pci_bus_id:02x}:{p.pci_device_id:02x}." + self._a = amdsmi + self._h = next(h for h in amdsmi.amdsmi_get_processor_handles() + if amdsmi.amdsmi_get_gpu_device_bdf(h).lower().startswith(bdf)) + self.rated_mhz = amdsmi.amdsmi_get_clock_info( + self._h, amdsmi.AmdSmiClkType.SYS).get("max_clk") or None + + def _sample(self) -> dict: + m = self._a.amdsmi_get_gpu_metrics_info(self._h) + + def val(key): + v = m.get(key) + return None if v in (None, "N/A", 0xFFFF, 0xFFFFFFFF) else v + + def top(key): + vals = [v for v in (m.get(key) or []) if v not in (None, "N/A", 0xFFFF)] + return max(vals) if vals else None + + status = val("throttle_status") + power = val("current_socket_power") or val("average_socket_power") + return { + "t_ns": time.time_ns(), + "sm_clock_mhz": top("current_gfxclks") or val("current_gfxclk"), + "mem_clock_mhz": val("current_uclk"), + "soc_clock_mhz": top("current_socclks") or val("current_socclk"), + "power_w": float(power) if power is not None else None, + "gpu_temp_c": val("temperature_hotspot"), + "mem_temp_c": val("temperature_mem"), + "gpu_util_pct": val("average_gfx_activity"), + "mem_util_pct": val("average_umc_activity"), + "reasons": [f"throttle_status=0x{int(status):x}"] if status else [], + } + + +_PROVIDER: _Provider | None = None + + +def _provider() -> _Provider: + global _PROVIDER + if _PROVIDER is None: + cls = _AmdSmi if torch.version.hip else _Nvml + try: + _PROVIDER = cls(torch.cuda.current_device()) + except Exception: + _PROVIDER = _Null() + return _PROVIDER + + +def _envelope(values: list) -> dict: + s = sorted(values) + return {"min": s[0], "p50": s[len(s) // 2], "max": s[-1]} + + +def _summarize(provider: _Provider, samples: list[dict], counters: dict, window_ns: int) -> dict: + rated = provider.rated_mhz + fields = sorted({k for s in samples for k in s} - {"t_ns", "reasons", "source"}) + summary = { + "provider": provider.name, + "interval_ms": _INTERVAL_MS, + "n_samples": len(samples), + "window_ms": round(window_ns / 1e6, 3), + "rated_sm_clock_mhz": rated, + } + for f in fields: + vals = [s[f] for s in samples if s.get(f) is not None] + if vals: + summary[f] = _envelope([round(v, 1) if isinstance(v, float) else v for v in vals]) + if counters: + summary["counters"] = {k: round(v, 3) for k, v in counters.items()} + if "energy_j" in counters and window_ns >= _COUNTER_MIN_WINDOW_NS: + summary["counters"]["avg_power_w"] = round(counters["energy_j"] / (window_ns / 1e9), 1) + + def caps(s): + rs = set(s.get("reasons", [])) + return rs & provider.cap_reasons if provider.cap_reasons is not None else rs + + summary["throttle_reasons"] = sorted({r for s in samples for r in s.get("reasons", [])}) + summary["capped"] = rated is not None and any( + caps(s) and s.get("sm_clock_mhz") is not None + and s["sm_clock_mhz"] < rated * _CAP_CLOCK_FRACTION for s in samples) + return summary + + +def _dump(op, attempt: int, summary: dict, samples: list[dict]) -> None: + if not _DIR: + return + os.makedirs(_DIR, exist_ok=True) + rec = {"op": {"type": op.type, "backend": op.backend, "args": dict(op.args)}, + "attempt": attempt, "counters": summary.get("counters"), "samples": samples} + path = os.path.join(_DIR, f"telemetry-rank{os.environ.get('RANK', '0')}.jsonl") + with open(path, "a") as f: + f.write(json.dumps(rec) + "\n") + + +def measure(op, time_once) -> tuple[float, dict]: + """time_once(sleep_s) -> median_us, retried while capped; the best median and its telemetry.""" + provider = _provider() + best = None + for attempt in range(1 + max(_RETRIES, 0)): + sleep_s = _RETRY_SLEEP_MS * 2 ** (attempt - 1) / 1e3 if attempt else 0.0 + before = provider.counters() + t0 = time.time_ns() + provider.start() + try: + median_us = time_once(sleep_s) + finally: + # a faulting kernel must not leave the sampler polling into the next op + samples = provider.stop() + t1 = time.time_ns() + after = provider.counters() + delta = {k: after[k] - before[k] for k in after if k in before} + summary = _summarize(provider, samples, delta, t1 - t0) + _dump(op, attempt, summary, samples) + if best is None or median_us < best[0]: + best = (median_us, summary, sleep_s) + if not ranks.any_(summary["capped"]): # every rank retries, or none does + break + median_us, summary, sleep_s = best + summary["attempts"] = attempt + 1 + summary["inter_kernel_sleep_ms"] = sleep_s * 1e3 + return median_us, summary diff --git a/operatorx/runners/common/timing.py b/operatorx/runners/common/timing.py new file mode 100644 index 0000000000..76f1789b56 --- /dev/null +++ b/operatorx/runners/common/timing.py @@ -0,0 +1,56 @@ +"""Event timing of one op on one or many devices. + +Per iteration: flush, rank align, start, op, end. A spin ahead of the loop lets the host +enqueue every iteration first; multi-device iterations start behind a device-side barrier ++ spin (CollectiveX _graph_align) so the cross-rank max is the op, not launch skew. +""" +from __future__ import annotations + +import os +import time + +import torch + +from operatorx.runners.common import ranks + +WARMUP = 5 +ITERS = 10 +_WARMUP_MIN_S = float(os.environ.get("OPERATORX_WARMUP_MIN_S", "0.025")) +_FIRST_WARMUP_S = float(os.environ.get("OPERATORX_FIRST_WARMUP_S", "1.0")) +_SHIELD_CYCLES = int(os.environ.get("OPERATORX_SHIELD_CYCLES", "100000000")) # ~50 ms +_ALIGN_CYCLES = int(os.environ.get("OPERATORX_ALIGN_CYCLES", "10000000")) # ~5 ms +_RETRY_SPIN_CYCLES = 2_000_000 +_FIRST = True + + +def time_op(fn, flush, sleep_s: float) -> float: + """Median over ITERS cold iterations of fn, in us (each iteration's slowest rank).""" + global _FIRST + for _ in range(WARMUP): + fn() + torch.cuda.synchronize() + floor, _FIRST = (max(_WARMUP_MIN_S, _FIRST_WARMUP_S) if _FIRST else _WARMUP_MIN_S), False + t0 = time.perf_counter() + while ranks.any_(time.perf_counter() - t0 < floor): # same round count on every rank + fn() + torch.cuda.synchronize() + + align = _ALIGN_CYCLES if ranks.world() > 1 else 0 + starts = [torch.cuda.Event(enable_timing=True) for _ in range(ITERS)] + ends = [torch.cuda.Event(enable_timing=True) for _ in range(ITERS)] + if sleep_s <= 0.0: + torch.cuda._sleep(_SHIELD_CYCLES) + for start, end in zip(starts, ends): + if sleep_s > 0.0: # throttle retry + time.sleep(sleep_s) + torch.cuda._sleep(_RETRY_SPIN_CYCLES) + flush() + ranks.align(align) + start.record() + fn() + end.record() + if sleep_s > 0.0: + torch.cuda.synchronize() + torch.cuda.synchronize() + times = [s.elapsed_time(e) * 1000.0 for s, e in zip(starts, ends)] + return sorted(ranks.iterations(times)["max"])[ITERS // 2] diff --git a/operatorx/runners/common/trace.py b/operatorx/runners/common/trace.py new file mode 100644 index 0000000000..ee7e35cda6 --- /dev/null +++ b/operatorx/runners/common/trace.py @@ -0,0 +1,78 @@ +"""Chrome-trace arithmetic shared by the per-platform profilers (CUPTI, rocprofiler): +a replay is a list of device events with start, duration and lane.""" +from __future__ import annotations + +import json +from typing import Callable + +TIMELINE_MAX = 64 + +def load_events(path: str) -> list[dict]: + return json.loads(open(path).read()).get("traceEvents", []) + + +def ts(e: dict) -> float: + return float(e.get("ts", 0.0)) + + +def dur(e: dict) -> float: + return float(e.get("dur", 0.0)) + + +def busy_us(events: list[dict]) -> float: + """Union of the events' time, so concurrent lanes are not counted twice.""" + total, end = 0.0, None + for a, b in sorted((ts(e), ts(e) + dur(e)) for e in events): + if end is None or a > end: + total += b - a + end = b + elif b > end: + total += b - end + end = b + return total + + +def replay_stats(replays: list[list[dict]], stream_of: Callable[[dict], object]) -> dict | None: + """Median replay by span: span first start to last end, busy the union of device time, + gap idle inside the span, overlap time counted twice by concurrent lanes.""" + rows = [] + for r in replays: + if not r: + continue + busy = busy_us(r) + span = max(ts(e) + dur(e) for e in r) - min(ts(e) for e in r) + rows.append((span, busy, span - busy, sum(dur(e) for e in r) - busy, + len({stream_of(e) for e in r}), r)) + if not rows: + return None + span, busy, gap, overlap, streams, r = sorted(rows, key=lambda x: x[0])[len(rows) // 2] + t0 = min(ts(e) for e in r) + timeline = [{"name": e.get("name", "")[:120], "stream": stream_of(e), + "start_us": round(ts(e) - t0, 3), "dur_us": round(dur(e), 3)} + for e in sorted(r, key=ts)[:TIMELINE_MAX]] + # + 0.0 so a value that is zero to rounding is never reported as -0.0 + return {"span_us": round(span, 3) + 0.0, "busy_us": round(busy, 3) + 0.0, + "gap_us": round(gap, 3) + 0.0, "overlap_us": round(overlap, 3) + 0.0, + "streams": streams, "timeline": timeline} + + +def aggregate(events: list[dict], calls: int, arg_fields: tuple, + cat_of: Callable[[dict], str], name_of: Callable[[dict], str]) -> list[dict]: + """Per-kernel totals over all replays, divided down to one call, longest first.""" + kernels: dict[str, dict] = {} + for e in events: + name = name_of(e) + a = e.get("args") or {} + k = kernels.setdefault(name, {"name": name, "cat": cat_of(e), "count": 0, "total_us": 0.0}) + k["count"] += 1 + k["total_us"] += dur(e) + for src, dst in arg_fields: + if src in a and dst not in k: + k[dst] = a[src] + out = [] + for k in kernels.values(): + k["count_per_call"] = round(k.pop("count") / calls, 2) + k["us_per_call"] = round(k.pop("total_us") / calls, 3) + out.append(k) + out.sort(key=lambda x: -x["us_per_call"]) + return out diff --git a/operatorx/runners/common/vllm/__init__.py b/operatorx/runners/common/vllm/__init__.py new file mode 100644 index 0000000000..01d16d5385 --- /dev/null +++ b/operatorx/runners/common/vllm/__init__.py @@ -0,0 +1 @@ +"""vLLM layers as operatorx backends: linear (GEMM), moe, attention.""" diff --git a/operatorx/runners/common/vllm/attention.py b/operatorx/runners/common/vllm/attention.py new file mode 100644 index 0000000000..d463ce999a --- /dev/null +++ b/operatorx/runners/common/vllm/attention.py @@ -0,0 +1,575 @@ +"""Attention modules through vLLM's own model code, KV cache and scheduling. + +The op's checkpoint family (attention_models.json) with its sizes, cut to the layers the module +needs, dummy weights; the batch is scheduled through the model runner with ctx tokens already +computed, and the module's forward is timed with its decoder layer's arguments and forward context. +""" +from __future__ import annotations + +import copy +import gc +import json +import os +import random +import sys +import tempfile +from dataclasses import dataclass, field, fields +from pathlib import Path + +import torch + +from operatorx.core import BackendImpl, Op, UnsupportedOpError, parallel +from operatorx.runners.common import ranks +from operatorx.runners.common.vllm import engine as vllm_engine +from operatorx.runners.common.vllm import linear as vllm_linear + +_MODELS = json.loads((Path(__file__).with_name("attention_models.json")).read_text()) +_MLP = 256 # width of the (untimed) MLP in the cut-down model +_VOCAB = 1024 +_MAX_BATCHED_TOKENS = 32768 # the largest max-num-batched-tokens of InferenceX's recipes +_MAX_SEQS = 1024 +_KV_FRACTION = float(os.environ.get("OPERATORX_ATTN_KV_FRACTION", "0.5")) # of device memory +# startup headroom check only; the KV cache is sized by _KV_FRACTION +_GPU_UTIL = float(os.environ.get("OPERATORX_ATTN_GPU_UTIL", "0.6")) +_KV_DTYPES = {"auto": "auto", "bf16": "bfloat16", "fp8": "fp8", "fp8_ds_mla": "fp8_ds_mla"} + + +@dataclass +class _Build: + family: str + config: dict + module: str # module path suffix, e.g. "layers.0.self_attn" + engine: dict = field(default_factory=dict) + without: tuple = () # recipe arguments for model parts the cut-down model leaves out + + +def _family(name: str) -> dict: + return copy.deepcopy(_MODELS[name]["config"]) + + +def _yarn(s: dict | None) -> dict | None: + if s is None: + return None + out = {"type": "yarn", "factor": s["factor"], "original_max_position_embeddings": s["original_max"]} + for k in ("beta_fast", "beta_slow", "mscale", "mscale_all_dim"): + if k in s: + out[k] = s[k] + return out + + +def _quant(op: Op, family: str) -> dict | None: + """The family checkpoint variant's quantization_config whose projections match proj.""" + proj = op.args.get("proj") or {} + if not proj: + return None + for v in _MODELS[family].get("variants", []): + if all(v["proj"].get(name) == pair for name, pair in proj.items()): + return copy.deepcopy(v["quantization_config"]) + raise UnsupportedOpError(f"no {family} checkpoint quantizes these projections: {sorted(proj)}") + + +def _kimi_k3(mla: dict, kda: dict, target: str) -> _Build: + """Kimi-K3: one KDA layer then one MLA layer (1-based layer lists), dense MLPs.""" + c = _family("kimi_k3") + t = c["text_config"] + lin = dict(t["linear_attn_config"], kda_layers=[1], full_attn_layers=[2], **kda) + t.update(num_hidden_layers=2, linear_attn_config=lin, first_k_dense_replace=2, intermediate_size=_MLP, + num_nextn_predict_layers=0, vocab_size=_VOCAB, **mla) + return _Build("kimi_k3", c, target, {"language_model_only": True}) + + +def _mla_dims(a: dict) -> dict: + return dict(hidden_size=a["hidden"], num_attention_heads=a["heads"], num_key_value_heads=a["heads"], + q_lora_rank=a["q_lora_rank"], kv_lora_rank=a["kv_lora_rank"], qk_nope_head_dim=a["nope"], + qk_rope_head_dim=a["rope_dim"], v_head_dim=a["v"]) + + +def _build_mla(op: Op) -> _Build: + a = op.args + if not a.get("rope", True) or a.get("gate"): + if a.get("rope", True): + raise UnsupportedOpError("the gated MLA module (Kimi-K3) has no RoPE") + return _kimi_k3(dict(_mla_dims(a), mla_use_nope=True, mla_use_output_gate=bool(a.get("gate"))), {}, + "layers.1.self_attn") + c = _family("deepseek_v3") + c.update(hidden_size=a["hidden"], num_attention_heads=a["heads"], num_key_value_heads=a["heads"], + q_lora_rank=a["q_lora_rank"], kv_lora_rank=a["kv_lora_rank"], qk_nope_head_dim=a["nope"], + qk_rope_head_dim=a["rope_dim"], v_head_dim=a["v"], rope_theta=a["rope_theta"], + rope_scaling=_yarn(a.get("rope_scaling")), num_hidden_layers=1, first_k_dense_replace=1, + intermediate_size=_MLP, num_nextn_predict_layers=0, vocab_size=_VOCAB) + return _Build("deepseek_v3", c, "layers.0.self_attn", {}) + + +def _qwen35(a: dict, full: dict, linear: dict, target: str) -> _Build: + """Qwen3.5: one Gated DeltaNet then one gated full-attention layer (serving's hybrid KV layout).""" + c = _family("qwen3_5_moe") + t = c["text_config"] + t.update(num_hidden_layers=2, layer_types=["linear_attention", "full_attention"], mtp_num_hidden_layers=0, + num_experts=8, num_experts_per_tok=2, moe_intermediate_size=_MLP, shared_expert_intermediate_size=_MLP, + vocab_size=_VOCAB, **full, **linear) + return _Build("qwen3_5_moe", c, target, {"language_model_only": True}) + + +def _qwen35_full(a: dict) -> dict: + rope = {"rope_type": "default", "rope_theta": a["rope_theta"], "partial_rotary_factor": a["rope_dim"] / a["head_dim"]} + if a.get("mrope_section"): + rope.update(mrope_section=a["mrope_section"], mrope_interleaved=True) + return dict(hidden_size=a["hidden"], num_attention_heads=a["q_heads"], num_key_value_heads=a["kv_heads"], + head_dim=a["head_dim"], attn_output_gate=True, rope_parameters=rope) + + +def _build_gqa(op: Op) -> _Build: + a = op.args + if a.get("gate"): + if not a.get("qk_norm"): + raise UnsupportedOpError("the gated GQA module (Qwen3.5) always has QK norm") + return _qwen35(a, _qwen35_full(a), {}, "layers.1.self_attn") + if a.get("mrope_section"): + raise UnsupportedOpError("M-RoPE without the output gate is not a module of these models") + c = _family("minimax_m3") + t = c["text_config"] + sparse = dict(t["sparse_attention_config"], sparse_attention_freq=[0], sparse_disable_index_value=[0]) + t.update(hidden_size=a["hidden"], num_attention_heads=a["q_heads"], num_key_value_heads=a["kv_heads"], + head_dim=a["head_dim"], rotary_dim=a["rope_dim"], partial_rotary_factor=a["rope_dim"] / a["head_dim"], + rope_theta=a["rope_theta"], use_qk_norm=bool(a.get("qk_norm")), attention_output_gate=False, + num_hidden_layers=1, moe_layer_freq=[0], dense_intermediate_size=_MLP, num_mtp_modules=0, + sparse_attention_config=sparse, vocab_size=_VOCAB) + return _Build("minimax_m3", c, "layers.0.self_attn", {"language_model_only": True}) + + +def _build_gdn(op: Op) -> _Build: + """silu: Qwen3.5's Gated DeltaNet; sigmoid: Qwen3.8's.""" + a = op.args + linear = dict(hidden_size=a["hidden"], linear_num_key_heads=a["qk_heads"], linear_num_value_heads=a["v_heads"], + linear_key_head_dim=a["head_dim"], linear_value_head_dim=a["head_dim"], + linear_conv_kernel_dim=a["conv_kernel"], + mamba_ssm_dtype={"fp32": "float32", "bf16": "bfloat16"}[a.get("state_dtype", "fp32")]) + if a.get("norm_act", "silu") == "sigmoid": + return _build_gdn_qwen38(a, linear) + b = _qwen35(a, {}, linear, "layers.0.linear_attn") + b.engine["mamba_ssm_cache_dtype"] = linear["mamba_ssm_dtype"] + return b + + +def _build_kda(op: Op) -> _Build: + a = op.args + b = _kimi_k3({"hidden_size": a["hidden"]}, + {"num_heads": a["heads"], "head_dim": a["head_dim"], "short_conv_kernel_size": a["conv_kernel"]}, + "layers.0.self_attn") + b.engine["mamba_ssm_cache_dtype"] = {"fp32": "float32", "bf16": "bfloat16"}[a.get("state_dtype", "fp32")] + return b + + +def _build_mla_dsa(op: Op) -> _Build: + """GLM-5.x: a layer with its own indexer, then one reusing its top-k ("FS").""" + a = op.args + c = _family("glm_moe_dsa") + rope = {"rope_type": "default", "rope_theta": a["rope_theta"]} + if a.get("rope_scaling"): + raise UnsupportedOpError("rope scaling for the GLM sparse MLA module is not wired") + c.update(_mla_dims(a), qk_head_dim=a["nope"] + a["rope_dim"], rope_parameters=rope, index_topk=a["topk"], + index_n_heads=a["index_heads"], index_head_dim=a["index_dim"], num_hidden_layers=2, + index_topk_pattern="FS", indexer_types=["full", "shared"], mlp_layer_types=["dense", "dense"], + first_k_dense_replace=2, intermediate_size=_MLP, num_nextn_predict_layers=0, vocab_size=_VOCAB) + return _Build("glm_moe_dsa", c, "layers.0.self_attn" if a.get("indexer", "own") == "own" else "layers.1.self_attn") + + +def _qwen38(text: dict, target: str) -> _Build: + """Qwen3.8-Flash-Next: one Gated DeltaNet layer then one QSA layer.""" + c = _family("qwen4_exp") + t = c["text_config"] + t.update(num_hidden_layers=2, layer_types=["linear_attention", "full_attention"], mtp_num_hidden_layers=0, + ple_layer_ids=[], num_experts=8, num_experts_per_tok=2, moe_intermediate_size=_MLP, + shared_expert_intermediate_size=_MLP, **text) + return _Build("qwen4_exp", c, target, {"language_model_only": True}) + + +def _build_qsa(op: Op) -> _Build: + a = op.args + rope = {"rope_type": "default", "rope_theta": a["rope_theta"], "partial_rotary_factor": a["rope_dim"] / a["head_dim"]} + if a.get("mrope_section"): + rope.update(mrope_section=a["mrope_section"], mrope_interleaved=True) + return _qwen38(dict(hidden_size=a["hidden"], num_attention_heads=a["q_heads"], num_key_value_heads=a["kv_heads"], + head_dim=a["head_dim"], partial_rotary_factor=rope["partial_rotary_factor"], + rope_parameters=rope, indexer_n_heads=a["index_heads"], indexer_kv_heads=1, + indexer_head_dim=a["index_dim"], indexer_budget=a["budget"], + indexer_compress_ratio=a["compress"]), "layers.1.self_attn") + + +# DeepSeek-V4.1's layer roles, in checkpoint order: window-only; ratio-2 source, top-k reuser; +# ratio-1 source (publishes candidate blocks), consumer, index source over the shared index cache +_V41_RATIOS = [0, 2, 2, 1, 1, 1] +_V41_LAYER = {(0, True, None): 0, (2, True, "own"): 1, (2, False, "reuse"): 2, (1, True, "own"): 3, + (1, False, "reuse"): 4, (1, False, "shared"): 5} + + +def _build_dsv41(a: dict) -> _Build: + layer = _V41_LAYER.get((a["compress_ratio"], a.get("source", True), a.get("indexer"))) + if layer is None: + raise UnsupportedOpError("DeepSeek-V4.1 has no layer with this compress ratio / source / indexer") + c = _family("deepseek_v41") + t = c["text_config"] + t.update(hidden_size=a["hidden"], num_attention_heads=a["heads"], head_dim=a["head_dim"], + qk_rope_head_dim=a["rope_dim"], q_lora_rank=a["q_lora_rank"], sliding_window=a["window"], + o_groups=a["o_groups"], o_lora_rank=a["o_lora_rank"], compress_ratios=_V41_RATIOS, + kv_source_layer_ids=[1, 3], index_source_layer_ids=[1, 3, 5], candidate_source_layer_id=3, + rope_theta=a.get("rope_theta", 10000.0), num_hidden_layers=len(_V41_RATIOS), + num_nextn_predict_layers=0, engram_layer_ids=[], dspark_target_layer_ids=[], n_routed_experts=8, + num_experts_per_tok=2, moe_intermediate_size=_MLP, vocab_size=_VOCAB) + if a.get("compress_rope_theta"): + t["compress_rope_theta"] = a["compress_rope_theta"] + if a.get("rope_scaling"): + t["rope_scaling"] = dict(_yarn(a["rope_scaling"]), rope_type="yarn") + if a.get("indexer"): + t.update(index_topk=a["topk"], index_n_heads=a["index_heads"], index_head_dim=a["index_dim"]) + return _Build("deepseek_v41", c, f"layers.{layer}.attn", {"language_model_only": True}, + without=("engram-config",)) + + +def _build_dsv4(op: Op) -> _Build: + """compress ratio 4 / 128: DeepSeek-V4-Pro; 0 / 1 / 2: DeepSeek-V4.1-Flash.""" + a = op.args + if a["compress_ratio"] in (0, 1, 2): + b = _build_dsv41(a) + else: + b = _build_dsv4_pro(a) + if a.get("index_cache_dtype"): + b.engine["attention_config"] = {"indexer_kv_dtype": a["index_cache_dtype"]} + return b + + +def _build_dsv4_pro(a: dict) -> _Build: + c = _family("deepseek_v4") + c.pop("expert_dtype", None) + # the C4A and C128A layers share one KV cache layout, so either is built next to the other + ratios = [4, 128] + c.update(hidden_size=a["hidden"], num_attention_heads=a["heads"], head_dim=a["head_dim"], + qk_rope_head_dim=a["rope_dim"], q_lora_rank=a["q_lora_rank"], sliding_window=a["window"], + o_groups=a["o_groups"], o_lora_rank=a["o_lora_rank"], compress_ratios=ratios, + rope_theta=a.get("rope_theta", 10000.0), num_hidden_layers=len(ratios), num_nextn_predict_layers=0, + num_hash_layers=0, n_routed_experts=8, num_experts_per_tok=2, vocab_size=_VOCAB) + # the checkpoint's expert width: DeepGEMM's MegaMoE (DEP recipes) rejects a narrower one + if a["compress_ratio"]: + c["compress_rope_theta"] = a["compress_rope_theta"] + if a.get("rope_scaling"): + c["rope_scaling"] = _yarn(a["rope_scaling"]) + if a.get("indexer"): + c.update(index_topk=a["topk"], index_n_heads=a["index_heads"], index_head_dim=a["index_dim"]) + return _Build("deepseek_v4", c, f"layers.{ratios.index(a['compress_ratio'])}.attn") + + +def _build_gdn_qwen38(a: dict, linear: dict) -> _Build: + b = _qwen38(linear, "layers.0.linear_attn") + b.engine["mamba_ssm_cache_dtype"] = linear["mamba_ssm_dtype"] + return b + + +_BUILDERS = {"mla": _build_mla, "mla_dsa": _build_mla_dsa, "dsv4_attn": _build_dsv4, "gqa": _build_gqa, + "qsa": _build_qsa, "gdn": _build_gdn, "kda": _build_kda} + + +class _Engine: + """One vLLM engine (in-process) for one module config; reused while ops share it.""" + + def __init__(self, key: str, b: _Build, split: dict | None): + os.environ["VLLM_ENABLE_V1_MULTIPROCESSING"] = "0" + from vllm import LLM + self.key = key + self.dir = tempfile.mkdtemp(prefix="opx-attn-") + Path(self.dir, "config.json").write_text(json.dumps(b.config)) + vllm_engine.launch() + kwargs, self.recipe = vllm_engine.engine_args( + split, without=b.without, eager=True, defaults={"max_num_batched_tokens": _MAX_BATCHED_TOKENS}, + axes=parallel.ATTENTION_AXES, + model=self.dir, load_format="dummy", skip_tokenizer_init=True, enforce_eager=True, + enable_prefix_caching=False, max_num_seqs=_MAX_SEQS, kv_cache_memory_bytes=_kv_bytes(), + gpu_memory_utilization=_GPU_UTIL, **b.engine) + # JIT kernels on first use (the untimed step), not every shape up front, where the + # image's vLLM has that setting + from vllm.config.kernel import KernelConfig + if "enable_jit_warmup" in {f.name for f in fields(KernelConfig)}: + kernel = kwargs.get("kernel_config") + if kernel is None or isinstance(kernel, dict): + kwargs["kernel_config"] = {**(kernel or {}), "enable_jit_warmup": False} + else: + kernel.enable_jit_warmup = False + # the engine runs eagerly: the full graphs serving would capture, from vLLM + self.capture = vllm_engine.full_graph_sizes(kwargs, self.recipe["compilation_config"]) + self.reqs: list = [] + self.n = 0 + try: + self.llm = LLM(**kwargs) + core = self.llm.llm_engine.engine_core.engine_core + self.runner = core.model_executor.driver_worker.worker.model_runner + self.kvm = core.scheduler.kv_cache_manager + _fill_caches(self.runner) + except BaseException: + self.close() + raise + + def close(self) -> None: + # keep the process group for the next engine + from vllm.distributed.parallel_state import destroy_model_parallel + try: + self.release() + if hasattr(self, "llm"): + self.llm.llm_engine.engine_core.shutdown() + except Exception as e: # noqa: BLE001 - teardown is best effort + print(f"[vllm.attention] engine shutdown: {type(e).__name__}: {e}", file=sys.stderr) + for k in ("llm", "runner", "kvm"): + self.__dict__.pop(k, None) + destroy_model_parallel() + gc.collect() + torch.cuda.empty_cache() + + def _output(self, new: list, scheduled: dict, finished: set): + from vllm.v1.core.sched.output import CachedRequestData, SchedulerOutput + return SchedulerOutput( + scheduled_new_reqs=new, scheduled_cached_reqs=CachedRequestData.make_empty(), + num_scheduled_tokens=scheduled, total_num_scheduled_tokens=sum(scheduled.values()), + scheduled_spec_decode_tokens={}, scheduled_encoder_inputs={}, + num_common_prefix_blocks=[0] * len(self.runner.kv_cache_config.kv_cache_groups), + finished_req_ids=finished, free_encoder_mm_hashes=[]) + + def _configured(self): + from vllm.config import set_current_vllm_config + return set_current_vllm_config(self.runner.vllm_config) + + def release(self) -> None: + """Finish the previous op's requests in the runner and free their blocks.""" + if not self.reqs or not hasattr(self, "runner"): + return + with self._configured(): + self.runner.execute_model(self._output([], {}, {r.request_id for r in self.reqs})) + for r in self.reqs: + self.kvm.free(r) + self.reqs = [] + + def step(self, batch: dict, path: str) -> dict: + """Schedule the batch; return the model's call to the module at path, with its forward context.""" + from vllm import SamplingParams + from vllm.forward_context import get_forward_context + from vllm.v1.core.sched.output import NewRequestData + from vllm.v1.request import Request + self.release() + budget = self.runner.vllm_config.scheduler_config.max_num_batched_tokens + if sum(g["count"] * g["q"] for g in batch["groups"]) > budget: + raise UnsupportedOpError(f"the batch schedules more than max-num-batched-tokens={budget} new tokens") + rng = random.Random(batch.get("seed", 0)) + for g in batch["groups"]: + for _ in range(g["count"]): + ctx = g["ctx"] if isinstance(g["ctx"], int) else rng.randint(g["ctx"]["min"], g["ctx"]["max"]) + self.n += 1 + r = Request(f"opx{self.n}", [0] * (ctx + g["q"]), SamplingParams(max_tokens=1), None) + # as the scheduler allocates computed ctx: every block (full), window only (sliding) + r.num_computed_tokens = ctx + self.reqs.append(r) + if self.kvm.allocate_slots(r, g["q"]) is None: + raise UnsupportedOpError(f"the batch needs more KV cache than {_kv_bytes() >> 30} GiB") + blocks = [self.kvm.get_block_ids(r.request_id) for r in self.reqs] + if batch.get("pages", "contiguous") == "shuffled": + blocks = _shuffle(blocks, rng) + new = [NewRequestData.from_request(r, b, r._all_token_ids) for r, b in zip(self.reqs, blocks)] + module = next(m for n, m in self.runner.model.named_modules() if n.endswith(path)) + forward = module.forward + seen: dict = {"forward": forward} + + def capture(*args, **kwargs): + if "fc" not in seen: + seen.update(args=args, kwargs=kwargs, fc=get_forward_context()) + return forward(*args, **kwargs) + + module.forward = capture + try: + with self._configured(): + self.runner.execute_model(self._output(new, {r.request_id: r.num_tokens - r.num_computed_tokens + for r in self.reqs}, set())) + finally: + module.forward = forward + self.runner.execute_model_state = None + if "fc" not in seen: + raise RuntimeError("the model never called the attention module") + return seen + + +def _kv_bytes() -> int: + return int(torch.cuda.get_device_properties(torch.cuda.current_device()).total_memory * _KV_FRACTION) + + +def _shuffle(blocks: list, rng: random.Random) -> list: + """Permute each KV cache group's blocks across the batch's requests.""" + out = [list(map(list, b)) for b in blocks] + for gi in range(len(blocks[0])): + pool = [x for b in blocks for x in b[gi]] + rng.shuffle(pool) + it = iter(pool) + for b in out: + b[gi] = [next(it) for _ in b[gi]] + return [tuple(b) for b in out] + + +def _fill_caches(runner) -> None: + """Random bytes below 0x40 in every KV cache/state buffer: small and finite as fp32, bf16, + fp8 e4m3 / ue8m0 and int8 alike (caches are multi-dtype views over raw bytes).""" + ctx = runner.vllm_config.compilation_config.static_forward_context + seen: set[int] = set() + with torch.no_grad(): + for layer in ctx.values(): + kc = getattr(layer, "kv_cache", None) + for t in kc if isinstance(kc, (list, tuple)) else [kc]: + if not isinstance(t, torch.Tensor): + continue + storage = t.untyped_storage() + if storage.data_ptr() in seen: + continue + seen.add(storage.data_ptr()) + raw = torch.empty(0, dtype=torch.uint8, device=t.device).set_(storage) + for c in raw.split(1 << 30): + c.random_(0, 0x40) + + +_ENGINE: _Engine | None = None + + +def _engine(b: _Build, split: dict | None) -> _Engine: + global _ENGINE + key = json.dumps([b.family, b.config, b.engine, split], sort_keys=True) + if _ENGINE is not None and _ENGINE.key == key: + return _ENGINE + if _ENGINE is not None: + _ENGINE.close() + _ENGINE = None + _ENGINE = _Engine(key, b, split) + return _ENGINE + + +def _backends(runner) -> dict[str, str]: + return {", ".join(g.layer_names): getattr(g.backend, "__name__", type(g.backend).__name__) + for gs in runner.attn_groups for g in gs} + + +def _prepare(op: Op) -> dict: + a = op.args + if a.get("selection", "natural") != "natural": + raise UnsupportedOpError("forced token selection is not wired yet") + b = _BUILDERS[op.type](op) + q = _quant(op, b.family) + if q is not None: + b.config["quantization_config"] = q + kv = a.get("kv_cache_dtype") + if kv is not None: + b.engine["kv_cache_dtype"] = _KV_DTYPES[kv] + split = a.get("parallel") + with ranks.together("build this engine"): + try: + eng = _engine(b, split) + # vLLM's startup (build, profiling, warmup) failing on this config + except (ValueError, NotImplementedError, AssertionError, RuntimeError) as e: + if vllm_linear._is_fault(e): + raise + raise UnsupportedOpError(f"vLLM rejected the {b.family} module: {type(e).__name__}: {e}"[:400]) from e + with ranks.together("schedule this batch"): + try: + seen = eng.step(a["batch"], b.module) + except (ValueError, NotImplementedError, AssertionError) as e: + if vllm_linear._is_fault(e): + raise + raise UnsupportedOpError(f"vLLM rejected the batch: {type(e).__name__}: {e}"[:400]) from e + ctx = {"engine": eng, **seen, + "meta": {"vllm_engine": vllm_engine.meta(), "vllm_family": b.family, "vllm_repo": _MODELS[b.family]["repo"], "vllm_module": b.module, + "vllm_backends": _backends(eng.runner), + "vllm_attn_metadata": {k: type(v).__name__ for k, v in (seen["fc"].attn_metadata or {}).items()}, + "kv_cache_groups": [ + {"spec": type(g.kv_cache_spec).__name__, "block_size": g.kv_cache_spec.block_size, + "dtype": str(getattr(g.kv_cache_spec, "dtype", "")).removeprefix("torch.")} + for g in eng.runner.kv_cache_config.kv_cache_groups]}} + _kernel(ctx) + torch.cuda.synchronize() + return ctx + + +def _replaying(ctx: dict): + """The model runner step's context: inference mode, vLLM's current config, forward context.""" + import contextlib + + from vllm.config import set_current_vllm_config + from vllm.forward_context import override_forward_context + stack = contextlib.ExitStack() + stack.enter_context(torch.inference_mode()) + stack.enter_context(set_current_vllm_config(ctx["engine"].runner.vllm_config)) + stack.enter_context(override_forward_context(ctx["fc"])) + return stack + + +def _kernel(ctx: dict) -> None: + with _replaying(ctx): + ctx["out"] = ctx["forward"](*ctx["args"], **ctx["kwargs"]) + + +def _cudagraph(ctx: dict) -> bool: + """vLLM would replay this as a full CUDA graph: uniform decode within capture sizes, backend support.""" + with ctx["engine"]._configured(): # backends read vLLM's current config to answer + return _full_graph(ctx["engine"]) + + +def _full_graph(eng: _Engine) -> bool: + from vllm.v1.attention.backend import AttentionCGSupport + qs = {r.num_tokens - r.num_computed_tokens for r in eng.reqs} + if len(qs) != 1: + return False + q = qs.pop() + if not eng.capture or q * len(eng.reqs) > eng.capture[-1]: + return False + need = AttentionCGSupport.UNIFORM_SINGLE_TOKEN_DECODE if q == 1 else AttentionCGSupport.UNIFORM_BATCH + for gs in eng.runner.attn_groups: + for g in gs: + support = g.backend.get_builder_cls().get_cudagraph_support(eng.runner.vllm_config, g.kv_cache_spec) + if support.value < need.value: + return False + return True + + +def _ready_upstream(ctx: dict) -> None: + """A top-k reusing layer waits on events its source layer records in the same graph; + captured alone, record them here (already satisfied) instead.""" + stream = torch.cuda.current_stream() + for m in ctx["engine"].runner.model.modules(): + group = getattr(getattr(m, "impl", None), "index_group", None) or getattr(m, "index_group", None) + for name in ("logical_topk_ready", "physical_topk_ready"): + ev = getattr(group, name, None) + if ev is not None: + ev.record(stream) + + +def _launcher(ctx: dict): + from vllm.distributed.parallel_state import graph_capture + eager = (lambda: _kernel(ctx)), False + if not _cudagraph(ctx): + return eager + err = None + try: + with _replaying(ctx): + for _ in range(2): + ctx["forward"](*ctx["args"], **ctx["kwargs"]) + torch.cuda.synchronize() + g = torch.cuda.CUDAGraph() + with graph_capture(torch.device("cuda", torch.cuda.current_device())) as gc_, \ + torch.cuda.graph(g, pool=torch.cuda.graph_pool_handle(), stream=gc_.stream): + _ready_upstream(ctx) + ctx["graph_out"] = ctx["forward"](*ctx["args"], **ctx["kwargs"]) + torch.cuda.synchronize() + except Exception as e: # fall back to eager, as the linear launcher does + if vllm_linear._is_fault(e): + raise + torch.cuda.synchronize() + err = e + if not ranks.agree(err is None): # every rank replays, or every rank runs eagerly + print(f"[vllm.attention] CUDA-graph capture failed, timing eagerly: {type(err).__name__}: {err}"[:300] + if err else "[vllm.attention] another rank's CUDA-graph capture failed, timing eagerly", + file=sys.stderr) + return eager + ctx["graph"] = g + return g.replay, True + + +IMPLS = [BackendImpl(op_type=t, prepare=_prepare, kernel=_kernel, launcher=_launcher) for t in _BUILDERS] diff --git a/operatorx/runners/common/vllm/attention_models.json b/operatorx/runners/common/vllm/attention_models.json new file mode 100644 index 0000000000..d278569613 --- /dev/null +++ b/operatorx/runners/common/vllm/attention_models.json @@ -0,0 +1,3902 @@ +{ + "deepseek_v3": { + "config": { + "architectures": [ + "DeepseekV3ForCausalLM" + ], + "attention_bias": false, + "attention_dropout": 0.0, + "bos_token_id": 0, + "eos_token_id": 1, + "ep_size": 1, + "first_k_dense_replace": 3, + "hidden_act": "silu", + "hidden_size": 7168, + "initializer_range": 0.02, + "intermediate_size": 18432, + "kv_lora_rank": 512, + "max_position_embeddings": 163840, + "model_type": "deepseek_v3", + "moe_intermediate_size": 2048, + "moe_layer_freq": 1, + "n_group": 8, + "n_routed_experts": 256, + "n_shared_experts": 1, + "norm_topk_prob": true, + "num_attention_heads": 128, + "num_experts_per_tok": 8, + "num_hidden_layers": 61, + "num_key_value_heads": 128, + "num_nextn_predict_layers": 1, + "q_lora_rank": 1536, + "qk_nope_head_dim": 128, + "qk_rope_head_dim": 64, + "rms_norm_eps": 1e-06, + "rope_scaling": { + "beta_fast": 32, + "beta_slow": 1, + "factor": 40, + "mscale": 1.0, + "mscale_all_dim": 1.0, + "original_max_position_embeddings": 4096, + "type": "yarn" + }, + "rope_theta": 10000, + "routed_scaling_factor": 2.5, + "scoring_func": "sigmoid", + "tie_word_embeddings": false, + "topk_group": 4, + "topk_method": "noaux_tc", + "torch_dtype": "bfloat16", + "use_cache": true, + "v_head_dim": 128, + "vocab_size": 129280 + }, + "repo": "deepseek-ai/DeepSeek-R1-0528", + "variants": [ + { + "proj": { + "kv_a_proj_with_mqa": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "group": [ + 1, + 128 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "group": [ + 128, + 128 + ], + "static": true + } + } + }, + "kv_b_proj": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "group": [ + 1, + 128 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "group": [ + 128, + 128 + ], + "static": true + } + } + }, + "o_proj": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "group": [ + 1, + 128 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "group": [ + 128, + 128 + ], + "static": true + } + } + }, + "q_a_proj": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "group": [ + 1, + 128 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "group": [ + 128, + 128 + ], + "static": true + } + } + }, + "q_b_proj": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "group": [ + 1, + 128 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "group": [ + 128, + 128 + ], + "static": true + } + } + } + }, + "quantization_config": { + "activation_scheme": "dynamic", + "fmt": "e4m3", + "quant_method": "fp8", + "weight_block_size": [ + 128, + 128 + ] + }, + "repo": "deepseek-ai/DeepSeek-R1-0528" + }, + { + "proj": { + "kv_a_proj_with_mqa": { + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + } + }, + "kv_b_proj": { + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + } + }, + "o_proj": { + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + } + }, + "q_a_proj": { + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + } + }, + "q_b_proj": { + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + } + } + }, + "quantization_config": { + "algo_config": null, + "exclude": [ + "re:model.decoder.*", + "lm_head" + ], + "export": { + "kv_cache_group": [], + "min_kv_scale": 0.0, + "pack_method": "reorder", + "weight_format": "real_quantized", + "weight_merge_groups": null + }, + "global_quant_config": { + "bias": null, + "input_tensors": { + "ch_axis": -1, + "dtype": "fp4", + "group_size": 32, + "is_dynamic": true, + "is_scale_quant": false, + "mx_element_dtype": null, + "observer_cls": "PerBlockMXObserver", + "qscheme": "per_group", + "round_method": "half_even", + "scale_calculation_mode": "even", + "scale_format": "e8m0", + "scale_type": "float", + "symmetric": null + }, + "output_tensors": null, + "target_device": null, + "weight": { + "ch_axis": -1, + "dtype": "fp4", + "group_size": 32, + "is_dynamic": false, + "is_scale_quant": false, + "mx_element_dtype": null, + "observer_cls": "PerBlockMXObserver", + "qscheme": "per_group", + "round_method": "half_even", + "scale_calculation_mode": "even", + "scale_format": "e8m0", + "scale_type": "float", + "symmetric": null + } + }, + "kv_cache_quant_config": {}, + "layer_quant_config": {}, + "layer_type_quant_config": {}, + "quant_method": "quark", + "quant_mode": "eager_mode", + "softmax_quant_spec": null, + "version": "0.10" + }, + "repo": "amd/DeepSeek-R1-0528-MXFP4-v2" + } + ] + }, + "deepseek_v4": { + "config": { + "architectures": [ + "DeepseekV4ForCausalLM" + ], + "attention_bias": false, + "attention_dropout": 0.0, + "bos_token_id": 0, + "compress_ratios": [ + 128, + 128, + 4, + 128, + 4, + 128, + 4, + 128, + 4, + 128, + 4, + 128, + 4, + 128, + 4, + 128, + 4, + 128, + 4, + 128, + 4, + 128, + 4, + 128, + 4, + 128, + 4, + 128, + 4, + 128, + 4, + 128, + 4, + 128, + 4, + 128, + 4, + 128, + 4, + 128, + 4, + 128, + 4, + 128, + 4, + 128, + 4, + 128, + 4, + 128, + 4, + 128, + 4, + 128, + 4, + 128, + 4, + 128, + 4, + 128, + 4, + 0 + ], + "compress_rope_theta": 160000, + "eos_token_id": 1, + "expert_dtype": "fp4", + "hc_eps": 1e-06, + "hc_mult": 4, + "hc_sinkhorn_iters": 20, + "head_dim": 512, + "hidden_act": "silu", + "hidden_size": 7168, + "index_head_dim": 128, + "index_n_heads": 64, + "index_topk": 1024, + "initializer_range": 0.02, + "max_position_embeddings": 1048576, + "model_type": "deepseek_v4", + "moe_intermediate_size": 3072, + "n_routed_experts": 384, + "n_shared_experts": 1, + "norm_topk_prob": true, + "num_attention_heads": 128, + "num_experts_per_tok": 6, + "num_hash_layers": 3, + "num_hidden_layers": 61, + "num_key_value_heads": 1, + "num_nextn_predict_layers": 1, + "o_groups": 16, + "o_lora_rank": 1024, + "q_lora_rank": 1536, + "qk_rope_head_dim": 64, + "rms_norm_eps": 1e-06, + "rope_scaling": { + "beta_fast": 32, + "beta_slow": 1, + "factor": 16, + "original_max_position_embeddings": 65536, + "type": "yarn" + }, + "rope_theta": 10000, + "routed_scaling_factor": 2.5, + "scoring_func": "sqrtsoftplus", + "sliding_window": 128, + "swiglu_limit": 10.0, + "tie_word_embeddings": false, + "topk_method": "noaux_tc", + "torch_dtype": "bfloat16", + "use_cache": true, + "vocab_size": 129280 + }, + "repo": "deepseek-ai/DeepSeek-V4-Pro", + "variants": [ + { + "proj": { + "indexer.wq_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 128 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + } + }, + "wkv": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 128 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + } + }, + "wo_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 128 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + } + }, + "wq_a": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 128 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + } + }, + "wq_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 128 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + } + } + }, + "quantization_config": { + "activation_scheme": "dynamic", + "fmt": "e4m3", + "quant_method": "fp8", + "scale_fmt": "ue8m0", + "weight_block_size": [ + 128, + 128 + ] + }, + "repo": "deepseek-ai/DeepSeek-V4-Pro" + }, + { + "proj": { + "indexer.wq_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 128 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + } + }, + "wkv": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 128 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + } + }, + "wo_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 128 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + } + }, + "wq_a": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 128 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + } + }, + "wq_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 128 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + } + } + }, + "quantization_config": { + "activation_scheme": "dynamic", + "fmt": "e4m3", + "quant_method": "fp8", + "scale_fmt": "ue8m0", + "weight_block_size": [ + 128, + 128 + ] + }, + "repo": "deepseek-ai/DeepSeek-V4-Pro-0813" + } + ] + }, + "deepseek_v41": { + "config": { + "architectures": [ + "DeepseekV41ForCausalLM" + ], + "bos_token_id": 0, + "dtype": "bfloat16", + "eos_token_id": 1, + "image_token_id": 129264, + "model_type": "deepseek_v41", + "pad_token_id": 2, + "text_config": { + "attention_bias": false, + "attention_dropout": 0.0, + "candidate_block_size": 8, + "candidate_source_layer_id": 20, + "candidate_topk_blocks": 2048, + "compress_ratios": [ + 0, + 0, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 0, + 0, + 0 + ], + "compress_rope_theta": 160000, + "dspark_block_size": 5, + "dspark_markov_rank": 256, + "dspark_n_routed_experts": 128, + "dspark_noise_token_id": 128799, + "dspark_num_experts_per_tok": 3, + "dspark_target_layer_ids": [ + 37, + 38, + 39 + ], + "engram_compressed_vocab_size": 99092, + "engram_head_dim": 256, + "engram_layer_ids": [ + 1, + 14 + ], + "engram_max_ngram_size": 4, + "engram_n_heads": 8, + "engram_num_embeddings": [ + 384006168, + 384016682 + ], + "engram_pad_token_id": 2, + "engram_vocab_size": 16000000, + "hc_eps": 1e-06, + "hc_mult": 4, + "hc_sinkhorn_iters": 20, + "head_dim": 512, + "hidden_act": "silu", + "hidden_size": 5120, + "index_head_dim": 128, + "index_n_heads": 32, + "index_source_layer_ids": [ + 2, + 8, + 14, + 20, + 24, + 28, + 32, + 36 + ], + "index_topk": 512, + "initializer_range": 0.02, + "kv_source_layer_ids": [ + 2, + 8, + 14, + 20 + ], + "max_position_embeddings": 1048576, + "model_type": "deepseek_v41_text", + "moe_intermediate_size": 2304, + "n_routed_experts": 384, + "n_shared_experts": 1, + "norm_topk_prob": true, + "num_attention_heads": 64, + "num_experts_per_tok": 6, + "num_hidden_layers": 40, + "num_key_value_heads": 1, + "num_nextn_predict_layers": 3, + "o_groups": 8, + "o_lora_rank": 1024, + "q_lora_rank": 1280, + "qk_rope_head_dim": 64, + "rms_norm_eps": 1e-20, + "rope_scaling": { + "beta_fast": 32, + "beta_slow": 1, + "factor": 16, + "original_max_position_embeddings": 65536, + "rope_type": "yarn" + }, + "rope_theta": 10000, + "routed_scaling_factor": 1.5, + "scoring_func": "sqrtsoftplus", + "sliding_window": 128, + "swiglu_limit": 10.0, + "tie_word_embeddings": false, + "topk_method": "noaux_tc", + "use_cache": true, + "vocab_size": 129280 + }, + "vision_config": { + "downsample_ratio": 3, + "hidden_size": 1024, + "intermediate_size": 2816, + "max_image_tokens": 1024, + "max_wh_ratio": null, + "min_pixels": 295936, + "model_type": "deepseek_v41_vision", + "num_attention_heads": 16, + "num_hidden_layers": 32, + "patch_size": 14, + "rope_theta": 10000 + } + }, + "repo": "deepseek-ai/DeepSeek-V4.1-Flash", + "variants": [ + { + "proj": { + "indexer.wq_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + }, + "wkv": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + }, + "wo_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + }, + "wq_a": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + }, + "wq_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + } + }, + "quantization_config": { + "activation_scheme": "dynamic", + "expert_dtype": "fp4", + "quant_method": "fp8", + "scale_fmt": "ue8m0", + "weight_block_size": [ + 32, + 32 + ] + }, + "repo": "deepseek-ai/DeepSeek-V4.1-Flash" + } + ] + }, + "glm_moe_dsa": { + "config": { + "architectures": [ + "GlmMoeDsaForCausalLM" + ], + "attention_bias": false, + "attention_dropout": 0.0, + "dtype": "bfloat16", + "eos_token_id": [ + 154820, + 154827, + 154829 + ], + "ep_size": 1, + "first_k_dense_replace": 3, + "head_dim": 192, + "hidden_act": "silu", + "hidden_size": 6144, + "index_head_dim": 128, + "index_n_heads": 32, + "index_share_for_mtp_iteration": true, + "index_skip_topk_offset": 3, + "index_topk": 2048, + "index_topk_freq": 4, + "index_topk_pattern": null, + "indexer_rope_interleave": true, + "indexer_types": [ + "full", + "full", + "full", + "shared", + "shared", + "shared", + "full", + "shared", + "shared", + "shared", + "full", + "shared", + "shared", + "shared", + "full", + "shared", + "shared", + "shared", + "full", + "shared", + "shared", + "shared", + "full", + "shared", + "shared", + "shared", + "full", + "shared", + "shared", + "shared", + "full", + "shared", + "shared", + "shared", + "full", + "shared", + "shared", + "shared", + "full", + "shared", + "shared", + "shared", + "full", + "shared", + "shared", + "shared", + "full", + "shared", + "shared", + "shared", + "full", + "shared", + "shared", + "shared", + "full", + "shared", + "shared", + "shared", + "full", + "shared", + "shared", + "shared", + "full", + "shared", + "shared", + "shared", + "full", + "shared", + "shared", + "shared", + "full", + "shared", + "shared", + "shared", + "full", + "shared", + "shared", + "shared" + ], + "initializer_range": 0.02, + "intermediate_size": 12288, + "kv_lora_rank": 512, + "max_position_embeddings": 1048576, + "mlp_layer_types": [ + "dense", + "dense", + "dense", + "sparse", + "sparse", + "sparse", + "sparse", + "sparse", + "sparse", + "sparse", + "sparse", + "sparse", + "sparse", + "sparse", + "sparse", + "sparse", + "sparse", + "sparse", + "sparse", + "sparse", + "sparse", + "sparse", + "sparse", + "sparse", + "sparse", + "sparse", + "sparse", + "sparse", + "sparse", + "sparse", + "sparse", + "sparse", + "sparse", + "sparse", + "sparse", + "sparse", + "sparse", + "sparse", + "sparse", + "sparse", + "sparse", + "sparse", + "sparse", + "sparse", + "sparse", + "sparse", + "sparse", + "sparse", + "sparse", + "sparse", + "sparse", + "sparse", + "sparse", + "sparse", + "sparse", + "sparse", + "sparse", + "sparse", + "sparse", + "sparse", + "sparse", + "sparse", + "sparse", + "sparse", + "sparse", + "sparse", + "sparse", + "sparse", + "sparse", + "sparse", + "sparse", + "sparse", + "sparse", + "sparse", + "sparse", + "sparse", + "sparse", + "sparse" + ], + "model_type": "glm_moe_dsa", + "moe_intermediate_size": 2048, + "moe_layer_freq": 1, + "moe_router_dtype": "float32", + "n_group": 1, + "n_routed_experts": 256, + "n_shared_experts": 1, + "norm_topk_prob": true, + "num_attention_heads": 64, + "num_experts_per_tok": 8, + "num_hidden_layers": 78, + "num_key_value_heads": 64, + "num_nextn_predict_layers": 1, + "pad_token_id": 154820, + "pretraining_tp": 1, + "q_lora_rank": 2048, + "qk_head_dim": 256, + "qk_nope_head_dim": 192, + "qk_rope_head_dim": 64, + "rms_norm_eps": 1e-05, + "rope_interleave": true, + "rope_parameters": { + "rope_theta": 8000000, + "rope_type": "default" + }, + "routed_scaling_factor": 2.5, + "scoring_func": "sigmoid", + "tie_word_embeddings": false, + "topk_group": 1, + "topk_method": "noaux_tc", + "use_cache": true, + "v_head_dim": 256, + "vocab_size": 154880 + }, + "repo": "zai-org/GLM-5.2-FP8", + "variants": [ + { + "proj": { + "indexer.wk": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "group": [ + 1, + 128 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "group": [ + 128, + 128 + ], + "static": true + } + } + }, + "indexer.wq_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "group": [ + 1, + 128 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "group": [ + 128, + 128 + ], + "static": true + } + } + }, + "kv_a_proj_with_mqa": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "group": [ + 1, + 128 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "group": [ + 128, + 128 + ], + "static": true + } + } + }, + "kv_b_proj": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "group": [ + 1, + 128 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "group": [ + 128, + 128 + ], + "static": true + } + } + }, + "o_proj": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "group": [ + 1, + 128 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "group": [ + 128, + 128 + ], + "static": true + } + } + }, + "q_a_proj": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "group": [ + 1, + 128 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "group": [ + 128, + 128 + ], + "static": true + } + } + }, + "q_b_proj": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "group": [ + 1, + 128 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "group": [ + 128, + 128 + ], + "static": true + } + } + } + }, + "quantization_config": { + "activation_scheme": "dynamic", + "fmt": "e4m3", + "modules_to_not_convert": [ + "lm_head", + "model.embed_tokens", + "model.layers.0.input_layernorm", + "model.layers.0.post_attention_layernorm", + "model.layers.0.self_attn.indexer.k_norm", + "model.layers.0.self_attn.indexer.k_norm.bias", + "model.layers.0.self_attn.indexers_proj", + "model.layers.0.self_attn.kv_a_layernorm", + "model.layers.0.self_attn.q_a_layernorm", + "model.layers.1.input_layernorm", + "model.layers.1.post_attention_layernorm", + "model.layers.1.self_attn.indexer.k_norm", + "model.layers.1.self_attn.indexer.k_norm.bias", + "model.layers.1.self_attn.indexers_proj", + "model.layers.1.self_attn.kv_a_layernorm", + "model.layers.1.self_attn.q_a_layernorm", + "model.layers.2.input_layernorm", + "model.layers.2.post_attention_layernorm", + "model.layers.2.self_attn.indexer.k_norm", + "model.layers.2.self_attn.indexer.k_norm.bias", + "model.layers.2.self_attn.indexers_proj", + "model.layers.2.self_attn.kv_a_layernorm", + "model.layers.2.self_attn.q_a_layernorm", + "model.layers.3.input_layernorm", + "model.layers.3.mlp.gate", + "model.layers.3.mlp.gate.e_score_correction_bias", + "model.layers.3.post_attention_layernorm", + "model.layers.3.self_attn.indexer.k_norm", + "model.layers.3.self_attn.indexer.k_norm.bias", + "model.layers.3.self_attn.indexers_proj", + "model.layers.3.self_attn.kv_a_layernorm", + "model.layers.3.self_attn.q_a_layernorm", + "model.layers.4.input_layernorm", + "model.layers.4.mlp.gate", + "model.layers.4.mlp.gate.e_score_correction_bias", + "model.layers.4.post_attention_layernorm", + "model.layers.4.self_attn.indexer.k_norm", + "model.layers.4.self_attn.indexer.k_norm.bias", + "model.layers.4.self_attn.indexers_proj", + "model.layers.4.self_attn.kv_a_layernorm", + "model.layers.4.self_attn.q_a_layernorm", + "model.layers.5.input_layernorm", + "model.layers.5.mlp.gate", + "model.layers.5.mlp.gate.e_score_correction_bias", + "model.layers.5.post_attention_layernorm", + "model.layers.5.self_attn.indexer.k_norm", + "model.layers.5.self_attn.indexer.k_norm.bias", + "model.layers.5.self_attn.indexers_proj", + "model.layers.5.self_attn.kv_a_layernorm", + "model.layers.5.self_attn.q_a_layernorm", + "model.norm" + ], + "quant_method": "fp8", + "weight_block_size": [ + 128, + 128 + ] + }, + "repo": "zai-org/GLM-5.1-FP8" + }, + { + "proj": { + "indexer.wk": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "group": [ + 1, + 128 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "group": [ + 128, + 128 + ], + "static": true + } + } + }, + "indexer.wq_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "group": [ + 1, + 128 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "group": [ + 128, + 128 + ], + "static": true + } + } + }, + "kv_a_proj_with_mqa": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "group": [ + 1, + 128 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "group": [ + 128, + 128 + ], + "static": true + } + } + }, + "kv_b_proj": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "group": [ + 1, + 128 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "group": [ + 128, + 128 + ], + "static": true + } + } + }, + "o_proj": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "group": [ + 1, + 128 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "group": [ + 128, + 128 + ], + "static": true + } + } + }, + "q_a_proj": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "group": [ + 1, + 128 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "group": [ + 128, + 128 + ], + "static": true + } + } + }, + "q_b_proj": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "group": [ + 1, + 128 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "group": [ + 128, + 128 + ], + "static": true + } + } + } + }, + "quantization_config": { + "activation_scheme": "dynamic", + "fmt": "e4m3", + "modules_to_not_convert": [ + "model.layers.4.self_attn.q_a_layernorm", + "model.layers.4.post_attention_layernorm", + "model.layers.4.input_layernorm", + "model.layers.3.mlp.gate.e_score_correction_bias", + "model.layers.3.mlp.gate", + "lm_head", + "model.layers.0.self_attn.indexer.k_norm", + "model.layers.4.mlp.gate.e_score_correction_bias", + "model.layers.2.input_layernorm", + "model.layers.5.mlp.gate", + "model.layers.0.self_attn.indexers_proj", + "model.layers.1.post_attention_layernorm", + "model.layers.4.mlp.gate", + "model.layers.5.self_attn.kv_a_layernorm", + "model.layers.5.post_attention_layernorm", + "model.layers.2.post_attention_layernorm", + "model.layers.5.input_layernorm", + "model.layers.4.self_attn.kv_a_layernorm", + "model.layers.3.self_attn.q_a_layernorm", + "model.layers.0.self_attn.q_a_layernorm", + "model.layers.1.self_attn.indexer.k_norm", + "model.layers.3.self_attn.kv_a_layernorm", + "model.layers.0.self_attn.kv_a_layernorm", + "model.norm", + "model.layers.1.self_attn.kv_a_layernorm", + "model.layers.0.input_layernorm", + "model.layers.1.self_attn.indexer.k_norm.bias", + "model.layers.2.self_attn.kv_a_layernorm", + "model.layers.2.self_attn.indexer.k_norm", + "model.layers.1.self_attn.q_a_layernorm", + "model.layers.5.mlp.gate.e_score_correction_bias", + "model.layers.3.post_attention_layernorm", + "model.layers.2.self_attn.q_a_layernorm", + "model.embed_tokens", + "model.layers.1.self_attn.indexers_proj", + "model.layers.0.post_attention_layernorm", + "model.layers.1.input_layernorm", + "model.layers.2.self_attn.indexers_proj", + "model.layers.0.self_attn.indexer.k_norm.bias", + "model.layers.2.self_attn.indexer.k_norm.bias", + "model.layers.3.input_layernorm", + "model.layers.5.self_attn.q_a_layernorm" + ], + "quant_method": "fp8", + "weight_block_size": [ + 128, + 128 + ] + }, + "repo": "zai-org/GLM-5.2-FP8" + }, + { + "proj": { + "indexer.wk": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "group": [ + 1, + 128 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "group": [ + 128, + 128 + ], + "static": true + } + } + }, + "indexer.wq_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "group": [ + 1, + 128 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "group": [ + 128, + 128 + ], + "static": true + } + } + }, + "kv_a_proj_with_mqa": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "group": [ + 1, + 128 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "group": [ + 128, + 128 + ], + "static": true + } + } + }, + "kv_b_proj": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "group": [ + 1, + 128 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "group": [ + 128, + 128 + ], + "static": true + } + } + }, + "o_proj": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "group": [ + 1, + 128 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "group": [ + 128, + 128 + ], + "static": true + } + } + }, + "q_a_proj": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "group": [ + 1, + 128 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "group": [ + 128, + 128 + ], + "static": true + } + } + }, + "q_b_proj": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "group": [ + 1, + 128 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "group": [ + 128, + 128 + ], + "static": true + } + } + } + }, + "quantization_config": { + "activation_scheme": "dynamic", + "fmt": "e4m3", + "modules_to_not_convert": [ + "model.layers.4.self_attn.q_a_layernorm", + "model.layers.4.post_attention_layernorm", + "model.layers.4.input_layernorm", + "model.layers.3.mlp.gate.e_score_correction_bias", + "model.layers.3.mlp.gate", + "lm_head", + "model.layers.0.self_attn.indexer.k_norm", + "model.layers.4.mlp.gate.e_score_correction_bias", + "model.layers.2.input_layernorm", + "model.layers.5.mlp.gate", + "model.layers.0.self_attn.indexers_proj", + "model.layers.1.post_attention_layernorm", + "model.layers.4.mlp.gate", + "model.layers.5.self_attn.kv_a_layernorm", + "model.layers.5.post_attention_layernorm", + "model.layers.2.post_attention_layernorm", + "model.layers.5.input_layernorm", + "model.layers.4.self_attn.kv_a_layernorm", + "model.layers.3.self_attn.q_a_layernorm", + "model.layers.0.self_attn.q_a_layernorm", + "model.layers.1.self_attn.indexer.k_norm", + "model.layers.3.self_attn.kv_a_layernorm", + "model.layers.0.self_attn.kv_a_layernorm", + "model.norm", + "model.layers.1.self_attn.kv_a_layernorm", + "model.layers.0.input_layernorm", + "model.layers.1.self_attn.indexer.k_norm.bias", + "model.layers.2.self_attn.kv_a_layernorm", + "model.layers.2.self_attn.indexer.k_norm", + "model.layers.1.self_attn.q_a_layernorm", + "model.layers.5.mlp.gate.e_score_correction_bias", + "model.layers.3.post_attention_layernorm", + "model.layers.2.self_attn.q_a_layernorm", + "model.embed_tokens", + "model.layers.1.self_attn.indexers_proj", + "model.layers.0.post_attention_layernorm", + "model.layers.1.input_layernorm", + "model.layers.2.self_attn.indexers_proj", + "model.layers.0.self_attn.indexer.k_norm.bias", + "model.layers.2.self_attn.indexer.k_norm.bias", + "model.layers.3.input_layernorm", + "model.layers.5.self_attn.q_a_layernorm" + ], + "quant_method": "fp8", + "weight_block_size": [ + 128, + 128 + ] + }, + "repo": "zai-org/GLM-5.3" + } + ] + }, + "kimi_k3": { + "config": { + "architectures": [ + "KimiK3ForConditionalGeneration" + ], + "bos_token_id": 163584, + "dtype": "bfloat16", + "eos_token_id": 163586, + "ignore_index": -100, + "image_placeholder": "<|kimi_image_placeholder|>", + "media_placeholder_token_id": 163605, + "model_type": "kimi_k3", + "pad_token_id": 163839, + "text_config": { + "activation_situ_beta": 4.0, + "activation_situ_linear_beta": 25.0, + "add_cross_attention": false, + "architectures": [ + "KimiLinearForCausalLM" + ], + "attn_res_block_size": 12, + "bad_words_ids": null, + "begin_suppress_tokens": null, + "bos_token_id": 163584, + "chunk_size_feed_forward": 0, + "cross_attention_hidden_size": null, + "decoder_start_token_id": null, + "diversity_penalty": 0.0, + "do_sample": false, + "dtype": "bfloat16", + "early_stopping": false, + "encoder_no_repeat_ngram_size": 0, + "eos_token_id": 163586, + "exponential_decay_length_penalty": null, + "finetuning_task": null, + "first_k_dense_replace": 1, + "forced_bos_token_id": null, + "forced_eos_token_id": null, + "hidden_act": "situ", + "hidden_size": 7168, + "initializer_range": 0.02, + "intermediate_size": 33792, + "is_decoder": false, + "is_encoder_decoder": false, + "kv_lora_rank": 512, + "latent_moe_use_norm": true, + "length_penalty": 1.0, + "linear_attn_config": { + "full_attn_layers": [ + 4, + 8, + 12, + 16, + 20, + 24, + 28, + 32, + 36, + 40, + 44, + 48, + 52, + 56, + 60, + 64, + 68, + 72, + 76, + 80, + 84, + 88, + 92, + 93 + ], + "gate_lower_bound": -5.0, + "head_dim": 128, + "kda_layers": [ + 1, + 2, + 3, + 5, + 6, + 7, + 9, + 10, + 11, + 13, + 14, + 15, + 17, + 18, + 19, + 21, + 22, + 23, + 25, + 26, + 27, + 29, + 30, + 31, + 33, + 34, + 35, + 37, + 38, + 39, + 41, + 42, + 43, + 45, + 46, + 47, + 49, + 50, + 51, + 53, + 54, + 55, + 57, + 58, + 59, + 61, + 62, + 63, + 65, + 66, + 67, + 69, + 70, + 71, + 73, + 74, + 75, + 77, + 78, + 79, + 81, + 82, + 83, + 85, + 86, + 87, + 89, + 90, + 91 + ], + "num_heads": 96, + "short_conv_kernel_size": 4, + "use_full_rank_gate": true + }, + "max_length": 20, + "max_position_embeddings": 1048576, + "min_length": 0, + "mla_use_nope": true, + "mla_use_output_gate": true, + "model_type": "kimi_linear", + "moe_intermediate_size": 3072, + "moe_layer_freq": 1, + "moe_renormalize": true, + "moe_router_activation_func": "sigmoid", + "no_repeat_ngram_size": 0, + "num_attention_heads": 96, + "num_beam_groups": 1, + "num_beams": 1, + "num_expert_group": 1, + "num_experts": 896, + "num_experts_per_token": 16, + "num_hidden_layers": 93, + "num_key_value_heads": 96, + "num_nextn_predict_layers": 0, + "num_return_sequences": 1, + "num_shared_experts": 2, + "output_attentions": false, + "output_hidden_states": false, + "output_scores": false, + "pad_token_id": 163839, + "prefix": null, + "problem_type": null, + "pruned_heads": {}, + "q_lora_rank": 1536, + "qk_nope_head_dim": 128, + "qk_rope_head_dim": 64, + "remove_invalid_values": false, + "repetition_penalty": 1.0, + "return_dict": true, + "return_dict_in_generate": false, + "rms_norm_eps": 1e-05, + "routed_expert_hidden_size": 3584, + "routed_scaling_factor": 1.0, + "sep_token_id": null, + "suppress_tokens": null, + "task_specific_params": null, + "temperature": 1.0, + "tf_legacy_loss": false, + "tie_encoder_decoder": false, + "tie_word_embeddings": false, + "tokenizer_class": null, + "top_k": 50, + "top_p": 1.0, + "topk_group": 1, + "topk_method": "noaux_tc", + "torchscript": false, + "typical_p": 1.0, + "use_bfloat16": false, + "use_cache": true, + "use_grouped_topk": true, + "v_head_dim": 128, + "vocab_size": 163840 + }, + "tie_word_embeddings": false, + "vision_config": { + "_attn_implementation": "flash_attention_2", + "activation_func": "gelu_pytorch_tanh", + "attn_bias": false, + "init_pos_emb_height": 64, + "init_pos_emb_time": 4, + "init_pos_emb_width": 64, + "linear_bias": false, + "merge_kernel_size": [ + 2, + 2 + ], + "merge_type": "sd2_tpool", + "mlp_type": "mlp2", + "mm_hidden_size": 1024, + "mm_projector_type": "patchmergerv2", + "norm_type": "rmsnorm", + "patch_embed_proj_bias": false, + "patch_size": 14, + "pos_emb_interpolation_mode": "bilinear", + "pos_emb_type": "divided_fixed", + "projector_hidden_act": "gelu", + "projector_ln_eps": 1e-05, + "qkv_hidden_size": 1536, + "text_hidden_size": 7168, + "vt_hidden_size": 1024, + "vt_intermediate_size": 4096, + "vt_num_attention_heads": 12, + "vt_num_hidden_layers": 27 + } + }, + "repo": "moonshotai/Kimi-K3", + "variants": [] + }, + "minimax_m3": { + "config": { + "architectures": [ + "MiniMaxM3SparseForConditionalGeneration" + ], + "image_grid_pinpoints": "[(336, 336), (336, 672), (336, 1008), (336, 1344), (336, 1680), (336, 2016), (672, 336), (672, 672), (672, 1008), (672, 1344), (672, 1680), (672, 2016), (1008, 336), (1008, 672), (1008, 1008), (1008, 1344), (1008, 1680), (1008, 2016), (1344, 336), (1344, 672), (1344, 1008), (1344, 1344), (1344, 1680), (1344, 2016), (1680, 336), (1680, 672), (1680, 1008), (1680, 1344), (1680, 1680), (1680, 2016), (2016, 336), (2016, 672), (2016, 1008), (2016, 1344), (2016, 1680), (2016, 2016)]", + "image_seq_length": 576, + "image_token_index": 200025, + "img_token_compression_config": { + "image_token_compression_method": "patch_merge", + "spatial_merge_size": 2, + "temporal_patch_size": 2 + }, + "model_type": "minimax_m3_vl", + "multimodal_projector_bias": true, + "num_reward_heads": 0, + "process_image_mode": "dynamic_res", + "projector_hidden_act": "gelu", + "projector_hidden_size": 6144, + "text_config": { + "architectures": [ + "MiniMaxM3SparseForCausalLM" + ], + "attention_output_gate": false, + "dense_intermediate_size": 12288, + "dtype": "bfloat16", + "head_dim": 128, + "hidden_act": "swigluoai", + "hidden_size": 6144, + "intermediate_size": 3072, + "max_position_embeddings": 1048576, + "moe_layer_freq": [ + 0, + 0, + 0, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1 + ], + "n_shared_experts": 1, + "num_attention_heads": 64, + "num_experts_per_tok": 4, + "num_hidden_layers": 60, + "num_key_value_heads": 4, + "num_local_experts": 128, + "num_mtp_modules": 1, + "partial_rotary_factor": 0.5, + "qk_norm_type": "per_head", + "rms_norm_eps": 1e-06, + "rope_theta": 5000000, + "rotary_dim": 64, + "routed_scaling_factor": 2.0, + "scoring_func": "sigmoid", + "shared_intermediate_size": 3072, + "sparse_attention_config": { + "sparse_attention_freq": [ + 0, + 0, + 0, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1 + ], + "sparse_block_size": 128, + "sparse_disable_index_value": [ + 0, + 0, + 0, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1 + ], + "sparse_index_dim": 128, + "sparse_init_block": 0, + "sparse_local_block": 1, + "sparse_num_index_heads": 4, + "sparse_score_type": "max", + "sparse_topk_blocks": 16, + "use_sparse_attention": true + }, + "swiglu_alpha": 1.702, + "swiglu_limit": 7.0, + "tie_word_embeddings": false, + "use_gemma_norm": true, + "use_qk_norm": true, + "use_routing_bias": true, + "vocab_size": 200064 + }, + "torch_dtype": "bfloat16", + "video_token_index": 200026, + "vision_config": { + "attention_dropout": 0.0, + "hidden_act": "gelu", + "hidden_size": 1280, + "image_size": 2016, + "img_token_compression_config": { + "image_token_compression_method": "patch_merge", + "spatial_merge_size": 2, + "temporal_patch_size": 2 + }, + "initializer_factor": 1.0, + "initializer_range": 0.02, + "intermediate_size": 5120, + "layer_norm_eps": 1e-05, + "model_type": "clip_vision_model", + "num_attention_heads": 16, + "num_channels": 3, + "num_hidden_layers": 32, + "patch_size": 14, + "position_embedding_type": "rope", + "projection_dim": 6144, + "rope_mode": "3d", + "rope_theta": 10000.0, + "vision_segment_max_frames": 4, + "vocab_size": 32000 + }, + "vision_feature_layer": -1, + "vision_feature_select_strategy": "full" + }, + "repo": "MiniMaxAI/MiniMax-M3-MXFP8", + "variants": [ + { + "proj": { + "index_k_proj": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + } + }, + "index_q_proj": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + } + }, + "k_proj": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + } + }, + "o_proj": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + } + }, + "q_proj": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + } + }, + "v_proj": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + } + } + }, + "quantization_config": { + "activation_scheme": "dynamic", + "ignored_layers": [ + "lm_head", + "model.embed_tokens", + "vision_tower", + "multi_modal_projector", + "patch_merge_mlp", + "language_model.model.layers.3.block_sparse_moe.gate", + "language_model.model.layers.4.block_sparse_moe.gate", + "language_model.model.layers.5.block_sparse_moe.gate" + ], + "quant_method": "mxfp8", + "weight_block_size": [ + 1, + 32 + ] + }, + "repo": "MiniMaxAI/MiniMax-M3-MXFP8" + }, + { + "proj": { + "index_k_proj": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + } + }, + "index_q_proj": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + } + }, + "k_proj": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + } + }, + "o_proj": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + } + }, + "q_proj": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + } + }, + "v_proj": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + } + } + }, + "quantization_config": { + "exclude_modules": [ + "lm_head", + "model.embed_tokens", + "vision_tower", + "multi_modal_projector", + "patch_merge_mlp", + "language_model.model.layers.3.block_sparse_moe.gate", + "language_model.model.layers.4.block_sparse_moe.gate", + "language_model.model.layers.5.block_sparse_moe.gate" + ], + "kv_cache_quant_algo": null, + "quant_algo": "MIXED_PRECISION", + "quant_method": "modelopt", + "quantized_layers": { + "language_model.model.layers.0.mlp.down_proj": { + "quant_algo": "MXFP8" + }, + "language_model.model.layers.0.mlp.gate_proj": { + "quant_algo": "MXFP8" + }, + "language_model.model.layers.0.mlp.up_proj": { + "quant_algo": "MXFP8" + }, + "language_model.model.layers.0.self_attn.k_proj": { + "quant_algo": "MXFP8" + }, + "language_model.model.layers.0.self_attn.o_proj": { + "quant_algo": "MXFP8" + }, + "language_model.model.layers.0.self_attn.q_proj": { + "quant_algo": "MXFP8" + }, + "language_model.model.layers.0.self_attn.v_proj": { + "quant_algo": "MXFP8" + }, + "language_model.model.layers.1.mlp.down_proj": { + "quant_algo": "MXFP8" + }, + "language_model.model.layers.1.mlp.gate_proj": { + "quant_algo": "MXFP8" + }, + "language_model.model.layers.1.mlp.up_proj": { + "quant_algo": "MXFP8" + }, + "language_model.model.layers.1.self_attn.k_proj": { + "quant_algo": "MXFP8" + }, + "language_model.model.layers.1.self_attn.o_proj": { + "quant_algo": "MXFP8" + }, + "language_model.model.layers.1.self_attn.q_proj": { + "quant_algo": "MXFP8" + }, + "language_model.model.layers.1.self_attn.v_proj": { + "quant_algo": "MXFP8" + }, + "language_model.model.layers.2.mlp.down_proj": { + "quant_algo": "MXFP8" + }, + "language_model.model.layers.2.mlp.gate_proj": { + "quant_algo": "MXFP8" + }, + "language_model.model.layers.2.mlp.up_proj": { + "quant_algo": "MXFP8" + }, + "language_model.model.layers.2.self_attn.k_proj": { + "quant_algo": "MXFP8" + }, + "language_model.model.layers.2.self_attn.o_proj": { + "quant_algo": "MXFP8" + }, + "language_model.model.layers.2.self_attn.q_proj": { + "quant_algo": "MXFP8" + }, + "language_model.model.layers.2.self_attn.v_proj": { + "quant_algo": "MXFP8" + }, + "language_model.model.layers.3.block_sparse_moe.experts.0.w1": { + "group_size": 16, + "quant_algo": "NVFP4" + }, + "language_model.model.layers.3.block_sparse_moe.experts.0.w2": { + "group_size": 16, + "quant_algo": "NVFP4" + }, + "language_model.model.layers.3.block_sparse_moe.experts.0.w3": { + "group_size": 16, + "quant_algo": "NVFP4" + }, + "language_model.model.layers.3.block_sparse_moe.experts.1.w1": { + "group_size": 16, + "quant_algo": "NVFP4" + }, + "language_model.model.layers.3.block_sparse_moe.experts.1.w2": { + "group_size": 16, + "quant_algo": "NVFP4" + }, + "language_model.model.layers.3.block_sparse_moe.experts.1.w3": { + "group_size": 16, + "quant_algo": "NVFP4" + }, + "language_model.model.layers.3.block_sparse_moe.experts.2.w1": { + "group_size": 16, + "quant_algo": "NVFP4" + }, + "language_model.model.layers.3.block_sparse_moe.experts.2.w2": { + "group_size": 16, + "quant_algo": "NVFP4" + }, + "language_model.model.layers.3.block_sparse_moe.experts.2.w3": { + "group_size": 16, + "quant_algo": "NVFP4" + }, + "language_model.model.layers.3.block_sparse_moe.experts.3.w1": { + "group_size": 16, + "quant_algo": "NVFP4" + }, + "language_model.model.layers.3.block_sparse_moe.experts.3.w2": { + "group_size": 16, + "quant_algo": "NVFP4" + }, + "language_model.model.layers.3.block_sparse_moe.experts.3.w3": { + "group_size": 16, + "quant_algo": "NVFP4" + }, + "language_model.model.layers.3.block_sparse_moe.experts.4.w1": { + "group_size": 16, + "quant_algo": "NVFP4" + }, + "language_model.model.layers.3.block_sparse_moe.experts.4.w2": { + "group_size": 16, + "quant_algo": "NVFP4" + }, + "language_model.model.layers.3.block_sparse_moe.experts.4.w3": { + "group_size": 16, + "quant_algo": "NVFP4" + }, + "language_model.model.layers.3.block_sparse_moe.experts.5.w1": { + "group_size": 16, + "quant_algo": "NVFP4" + }, + "language_model.model.layers.3.block_sparse_moe.experts.5.w2": { + "group_size": 16, + "quant_algo": "NVFP4" + }, + "language_model.model.layers.3.block_sparse_moe.experts.5.w3": { + "group_size": 16, + "quant_algo": "NVFP4" + }, + "language_model.model.layers.3.block_sparse_moe.experts.6.w1": { + "group_size": 16, + "quant_algo": "NVFP4" + }, + "language_model.model.layers.3.block_sparse_moe.experts.6.w2": { + "group_size": 16, + "quant_algo": "NVFP4" + }, + "language_model.model.layers.3.block_sparse_moe.experts.6.w3": { + "group_size": 16, + "quant_algo": "NVFP4" + }, + "language_model.model.layers.3.block_sparse_moe.experts.7.w1": { + "group_size": 16, + "quant_algo": "NVFP4" + }, + "language_model.model.layers.3.block_sparse_moe.experts.7.w2": { + "group_size": 16, + "quant_algo": "NVFP4" + }, + "language_model.model.layers.3.block_sparse_moe.experts.7.w3": { + "group_size": 16, + "quant_algo": "NVFP4" + }, + "language_model.model.layers.3.block_sparse_moe.shared_experts.down_proj": { + "quant_algo": "MXFP8" + }, + "language_model.model.layers.3.block_sparse_moe.shared_experts.gate_proj": { + "quant_algo": "MXFP8" + }, + "language_model.model.layers.3.block_sparse_moe.shared_experts.up_proj": { + "quant_algo": "MXFP8" + }, + "language_model.model.layers.3.self_attn.index_k_proj": { + "quant_algo": "MXFP8" + }, + "language_model.model.layers.3.self_attn.index_q_proj": { + "quant_algo": "MXFP8" + }, + "language_model.model.layers.3.self_attn.k_proj": { + "quant_algo": "MXFP8" + }, + "language_model.model.layers.3.self_attn.o_proj": { + "quant_algo": "MXFP8" + }, + "language_model.model.layers.3.self_attn.q_proj": { + "quant_algo": "MXFP8" + }, + "language_model.model.layers.3.self_attn.v_proj": { + "quant_algo": "MXFP8" + }, + "language_model.model.layers.4.block_sparse_moe.experts.0.w1": { + "group_size": 16, + "quant_algo": "NVFP4" + }, + "language_model.model.layers.4.block_sparse_moe.experts.0.w2": { + "group_size": 16, + "quant_algo": "NVFP4" + }, + "language_model.model.layers.4.block_sparse_moe.experts.0.w3": { + "group_size": 16, + "quant_algo": "NVFP4" + }, + "language_model.model.layers.4.block_sparse_moe.experts.1.w1": { + "group_size": 16, + "quant_algo": "NVFP4" + }, + "language_model.model.layers.4.block_sparse_moe.experts.1.w2": { + "group_size": 16, + "quant_algo": "NVFP4" + }, + "language_model.model.layers.4.block_sparse_moe.experts.1.w3": { + "group_size": 16, + "quant_algo": "NVFP4" + }, + "language_model.model.layers.4.block_sparse_moe.experts.2.w1": { + "group_size": 16, + "quant_algo": "NVFP4" + }, + "language_model.model.layers.4.block_sparse_moe.experts.2.w2": { + "group_size": 16, + "quant_algo": "NVFP4" + }, + "language_model.model.layers.4.block_sparse_moe.experts.2.w3": { + "group_size": 16, + "quant_algo": "NVFP4" + }, + "language_model.model.layers.4.block_sparse_moe.experts.3.w1": { + "group_size": 16, + "quant_algo": "NVFP4" + }, + "language_model.model.layers.4.block_sparse_moe.experts.3.w2": { + "group_size": 16, + "quant_algo": "NVFP4" + }, + "language_model.model.layers.4.block_sparse_moe.experts.3.w3": { + "group_size": 16, + "quant_algo": "NVFP4" + }, + "language_model.model.layers.4.block_sparse_moe.experts.4.w1": { + "group_size": 16, + "quant_algo": "NVFP4" + }, + "language_model.model.layers.4.block_sparse_moe.experts.4.w2": { + "group_size": 16, + "quant_algo": "NVFP4" + }, + "language_model.model.layers.4.block_sparse_moe.experts.4.w3": { + "group_size": 16, + "quant_algo": "NVFP4" + }, + "language_model.model.layers.4.block_sparse_moe.experts.5.w1": { + "group_size": 16, + "quant_algo": "NVFP4" + }, + "language_model.model.layers.4.block_sparse_moe.experts.5.w2": { + "group_size": 16, + "quant_algo": "NVFP4" + }, + "language_model.model.layers.4.block_sparse_moe.experts.5.w3": { + "group_size": 16, + "quant_algo": "NVFP4" + }, + "language_model.model.layers.4.block_sparse_moe.experts.6.w1": { + "group_size": 16, + "quant_algo": "NVFP4" + }, + "language_model.model.layers.4.block_sparse_moe.experts.6.w2": { + "group_size": 16, + "quant_algo": "NVFP4" + }, + "language_model.model.layers.4.block_sparse_moe.experts.6.w3": { + "group_size": 16, + "quant_algo": "NVFP4" + }, + "language_model.model.layers.4.block_sparse_moe.experts.7.w1": { + "group_size": 16, + "quant_algo": "NVFP4" + }, + "language_model.model.layers.4.block_sparse_moe.experts.7.w2": { + "group_size": 16, + "quant_algo": "NVFP4" + }, + "language_model.model.layers.4.block_sparse_moe.experts.7.w3": { + "group_size": 16, + "quant_algo": "NVFP4" + }, + "language_model.model.layers.4.block_sparse_moe.shared_experts.down_proj": { + "quant_algo": "MXFP8" + }, + "language_model.model.layers.4.block_sparse_moe.shared_experts.gate_proj": { + "quant_algo": "MXFP8" + }, + "language_model.model.layers.4.block_sparse_moe.shared_experts.up_proj": { + "quant_algo": "MXFP8" + }, + "language_model.model.layers.4.self_attn.index_k_proj": { + "quant_algo": "MXFP8" + }, + "language_model.model.layers.4.self_attn.index_q_proj": { + "quant_algo": "MXFP8" + }, + "language_model.model.layers.4.self_attn.k_proj": { + "quant_algo": "MXFP8" + }, + "language_model.model.layers.4.self_attn.o_proj": { + "quant_algo": "MXFP8" + }, + "language_model.model.layers.4.self_attn.q_proj": { + "quant_algo": "MXFP8" + }, + "language_model.model.layers.4.self_attn.v_proj": { + "quant_algo": "MXFP8" + }, + "language_model.model.layers.5.block_sparse_moe.experts.0.w1": { + "group_size": 16, + "quant_algo": "NVFP4" + }, + "language_model.model.layers.5.block_sparse_moe.experts.0.w2": { + "group_size": 16, + "quant_algo": "NVFP4" + }, + "language_model.model.layers.5.block_sparse_moe.experts.0.w3": { + "group_size": 16, + "quant_algo": "NVFP4" + }, + "language_model.model.layers.5.block_sparse_moe.experts.1.w1": { + "group_size": 16, + "quant_algo": "NVFP4" + }, + "language_model.model.layers.5.block_sparse_moe.experts.1.w2": { + "group_size": 16, + "quant_algo": "NVFP4" + }, + "language_model.model.layers.5.block_sparse_moe.experts.1.w3": { + "group_size": 16, + "quant_algo": "NVFP4" + }, + "language_model.model.layers.5.block_sparse_moe.experts.2.w1": { + "group_size": 16, + "quant_algo": "NVFP4" + }, + "language_model.model.layers.5.block_sparse_moe.experts.2.w2": { + "group_size": 16, + "quant_algo": "NVFP4" + }, + "language_model.model.layers.5.block_sparse_moe.experts.2.w3": { + "group_size": 16, + "quant_algo": "NVFP4" + }, + "language_model.model.layers.5.block_sparse_moe.experts.3.w1": { + "group_size": 16, + "quant_algo": "NVFP4" + }, + "language_model.model.layers.5.block_sparse_moe.experts.3.w2": { + "group_size": 16, + "quant_algo": "NVFP4" + }, + "language_model.model.layers.5.block_sparse_moe.experts.3.w3": { + "group_size": 16, + "quant_algo": "NVFP4" + }, + "language_model.model.layers.5.block_sparse_moe.experts.4.w1": { + "group_size": 16, + "quant_algo": "NVFP4" + }, + "language_model.model.layers.5.block_sparse_moe.experts.4.w2": { + "group_size": 16, + "quant_algo": "NVFP4" + }, + "language_model.model.layers.5.block_sparse_moe.experts.4.w3": { + "group_size": 16, + "quant_algo": "NVFP4" + }, + "language_model.model.layers.5.block_sparse_moe.experts.5.w1": { + "group_size": 16, + "quant_algo": "NVFP4" + }, + "language_model.model.layers.5.block_sparse_moe.experts.5.w2": { + "group_size": 16, + "quant_algo": "NVFP4" + }, + "language_model.model.layers.5.block_sparse_moe.experts.5.w3": { + "group_size": 16, + "quant_algo": "NVFP4" + }, + "language_model.model.layers.5.block_sparse_moe.experts.6.w1": { + "group_size": 16, + "quant_algo": "NVFP4" + }, + "language_model.model.layers.5.block_sparse_moe.experts.6.w2": { + "group_size": 16, + "quant_algo": "NVFP4" + }, + "language_model.model.layers.5.block_sparse_moe.experts.6.w3": { + "group_size": 16, + "quant_algo": "NVFP4" + }, + "language_model.model.layers.5.block_sparse_moe.experts.7.w1": { + "group_size": 16, + "quant_algo": "NVFP4" + }, + "language_model.model.layers.5.block_sparse_moe.experts.7.w2": { + "group_size": 16, + "quant_algo": "NVFP4" + }, + "language_model.model.layers.5.block_sparse_moe.experts.7.w3": { + "group_size": 16, + "quant_algo": "NVFP4" + }, + "language_model.model.layers.5.block_sparse_moe.shared_experts.down_proj": { + "quant_algo": "MXFP8" + }, + "language_model.model.layers.5.block_sparse_moe.shared_experts.gate_proj": { + "quant_algo": "MXFP8" + }, + "language_model.model.layers.5.block_sparse_moe.shared_experts.up_proj": { + "quant_algo": "MXFP8" + }, + "language_model.model.layers.5.self_attn.index_k_proj": { + "quant_algo": "MXFP8" + }, + "language_model.model.layers.5.self_attn.index_q_proj": { + "quant_algo": "MXFP8" + }, + "language_model.model.layers.5.self_attn.k_proj": { + "quant_algo": "MXFP8" + }, + "language_model.model.layers.5.self_attn.o_proj": { + "quant_algo": "MXFP8" + }, + "language_model.model.layers.5.self_attn.q_proj": { + "quant_algo": "MXFP8" + }, + "language_model.model.layers.5.self_attn.v_proj": { + "quant_algo": "MXFP8" + } + } + }, + "repo": "nvidia/MiniMax-M3-NVFP4" + } + ] + }, + "qwen3_5_moe": { + "config": { + "architectures": [ + "Qwen3_5MoeForConditionalGeneration" + ], + "image_token_id": 248056, + "model_type": "qwen3_5_moe", + "text_config": { + "attention_bias": false, + "attention_dropout": 0.0, + "attn_output_gate": true, + "dtype": "bfloat16", + "eos_token_id": 248044, + "full_attention_interval": 4, + "head_dim": 256, + "hidden_act": "silu", + "hidden_size": 4096, + "initializer_range": 0.02, + "layer_types": [ + "linear_attention", + "linear_attention", + "linear_attention", + "full_attention", + "linear_attention", + "linear_attention", + "linear_attention", + "full_attention", + "linear_attention", + "linear_attention", + "linear_attention", + "full_attention", + "linear_attention", + "linear_attention", + "linear_attention", + "full_attention", + "linear_attention", + "linear_attention", + "linear_attention", + "full_attention", + "linear_attention", + "linear_attention", + "linear_attention", + "full_attention", + "linear_attention", + "linear_attention", + "linear_attention", + "full_attention", + "linear_attention", + "linear_attention", + "linear_attention", + "full_attention", + "linear_attention", + "linear_attention", + "linear_attention", + "full_attention", + "linear_attention", + "linear_attention", + "linear_attention", + "full_attention", + "linear_attention", + "linear_attention", + "linear_attention", + "full_attention", + "linear_attention", + "linear_attention", + "linear_attention", + "full_attention", + "linear_attention", + "linear_attention", + "linear_attention", + "full_attention", + "linear_attention", + "linear_attention", + "linear_attention", + "full_attention", + "linear_attention", + "linear_attention", + "linear_attention", + "full_attention" + ], + "linear_conv_kernel_dim": 4, + "linear_key_head_dim": 128, + "linear_num_key_heads": 16, + "linear_num_value_heads": 64, + "linear_value_head_dim": 128, + "mamba_ssm_dtype": "float32", + "max_position_embeddings": 262144, + "mlp_only_layers": [], + "model_type": "qwen3_5_moe_text", + "moe_intermediate_size": 1024, + "mtp_num_hidden_layers": 1, + "mtp_use_dedicated_embeddings": false, + "num_attention_heads": 32, + "num_experts": 512, + "num_experts_per_tok": 10, + "num_hidden_layers": 60, + "num_key_value_heads": 2, + "rms_norm_eps": 1e-06, + "rope_parameters": { + "mrope_interleaved": true, + "mrope_section": [ + 11, + 11, + 10 + ], + "partial_rotary_factor": 0.25, + "rope_theta": 10000000, + "rope_type": "default" + }, + "router_aux_loss_coef": 0.001, + "shared_expert_intermediate_size": 1024, + "use_cache": true, + "vocab_size": 248320 + }, + "tie_word_embeddings": false, + "video_token_id": 248057, + "vision_config": { + "deepstack_visual_indexes": [], + "depth": 27, + "hidden_act": "gelu_pytorch_tanh", + "hidden_size": 1152, + "in_channels": 3, + "initializer_range": 0.02, + "intermediate_size": 4304, + "model_type": "qwen3_5_moe", + "num_heads": 16, + "num_position_embeddings": 2304, + "out_hidden_size": 4096, + "patch_size": 16, + "spatial_merge_size": 2, + "temporal_patch_size": 2 + }, + "vision_end_token_id": 248054, + "vision_start_token_id": 248053 + }, + "repo": "Qwen/Qwen3.5-397B-A17B-FP8", + "variants": [ + { + "proj": { + "in_proj_qkv": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "group": [ + 1, + 128 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "bf16", + "group": [ + 128, + 128 + ], + "static": true + } + } + }, + "in_proj_z": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "group": [ + 1, + 128 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "bf16", + "group": [ + 128, + 128 + ], + "static": true + } + } + }, + "k_proj": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "group": [ + 1, + 128 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "bf16", + "group": [ + 128, + 128 + ], + "static": true + } + } + }, + "o_proj": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "group": [ + 1, + 128 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "bf16", + "group": [ + 128, + 128 + ], + "static": true + } + } + }, + "out_proj": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "group": [ + 1, + 128 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "bf16", + "group": [ + 128, + 128 + ], + "static": true + } + } + }, + "q_proj": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "group": [ + 1, + 128 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "bf16", + "group": [ + 128, + 128 + ], + "static": true + } + } + }, + "v_proj": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "group": [ + 1, + 128 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "bf16", + "group": [ + 128, + 128 + ], + "static": true + } + } + } + }, + "quantization_config": { + "act_per_tensor": false, + "activation_scheme": "dynamic", + "modules_to_not_convert": [ + "lm_head", + "model.language_model.embed_tokens", + "model.language_model.layers.0.linear_attn.conv1d", + "model.language_model.layers.0.linear_attn.in_proj_a", + "model.language_model.layers.0.linear_attn.in_proj_b", + "model.language_model.layers.0.mlp.gate", + "model.language_model.layers.0.mlp.shared_expert_gate", + "model.language_model.layers.1.linear_attn.conv1d", + "model.language_model.layers.1.linear_attn.in_proj_a", + "model.language_model.layers.1.linear_attn.in_proj_b", + "model.language_model.layers.1.mlp.gate", + "model.language_model.layers.1.mlp.shared_expert_gate", + "model.language_model.layers.2.linear_attn.conv1d", + "model.language_model.layers.2.linear_attn.in_proj_a", + "model.language_model.layers.2.linear_attn.in_proj_b", + "model.language_model.layers.2.mlp.gate", + "model.language_model.layers.2.mlp.shared_expert_gate", + "model.language_model.layers.3.mlp.gate", + "model.language_model.layers.3.mlp.shared_expert_gate", + "model.language_model.layers.4.linear_attn.conv1d", + "model.language_model.layers.4.linear_attn.in_proj_a", + "model.language_model.layers.4.linear_attn.in_proj_b", + "model.language_model.layers.4.mlp.gate", + "model.language_model.layers.4.mlp.shared_expert_gate", + "model.language_model.layers.5.linear_attn.conv1d", + "model.language_model.layers.5.linear_attn.in_proj_a", + "model.language_model.layers.5.linear_attn.in_proj_b", + "model.language_model.layers.5.mlp.gate", + "model.language_model.layers.5.mlp.shared_expert_gate", + "model.visual.blocks.0.attn.proj", + "model.visual.blocks.0.attn.qkv", + "model.visual.blocks.0.mlp.linear_fc1", + "model.visual.blocks.0.mlp.linear_fc2", + "model.visual.blocks.1.attn.proj", + "model.visual.blocks.1.attn.qkv", + "model.visual.blocks.1.mlp.linear_fc1", + "model.visual.blocks.1.mlp.linear_fc2", + "model.visual.blocks.10.attn.proj", + "model.visual.blocks.10.attn.qkv", + "model.visual.blocks.10.mlp.linear_fc1", + "model.visual.blocks.10.mlp.linear_fc2", + "model.visual.blocks.11.attn.proj", + "model.visual.blocks.11.attn.qkv", + "model.visual.blocks.11.mlp.linear_fc1", + "model.visual.blocks.11.mlp.linear_fc2", + "model.visual.blocks.12.attn.proj", + "model.visual.blocks.12.attn.qkv", + "model.visual.blocks.12.mlp.linear_fc1", + "model.visual.blocks.12.mlp.linear_fc2", + "model.visual.blocks.13.attn.proj", + "model.visual.blocks.13.attn.qkv", + "model.visual.blocks.13.mlp.linear_fc1", + "model.visual.blocks.13.mlp.linear_fc2", + "model.visual.blocks.14.attn.proj", + "model.visual.blocks.14.attn.qkv", + "model.visual.blocks.14.mlp.linear_fc1", + "model.visual.blocks.14.mlp.linear_fc2", + "model.visual.blocks.15.attn.proj", + "model.visual.blocks.15.attn.qkv", + "model.visual.blocks.15.mlp.linear_fc1", + "model.visual.blocks.15.mlp.linear_fc2", + "model.visual.blocks.16.attn.proj", + "model.visual.blocks.16.attn.qkv", + "model.visual.blocks.16.mlp.linear_fc1", + "model.visual.blocks.16.mlp.linear_fc2", + "model.visual.blocks.17.attn.proj", + "model.visual.blocks.17.attn.qkv", + "model.visual.blocks.17.mlp.linear_fc1", + "model.visual.blocks.17.mlp.linear_fc2", + "model.visual.blocks.18.attn.proj", + "model.visual.blocks.18.attn.qkv", + "model.visual.blocks.18.mlp.linear_fc1", + "model.visual.blocks.18.mlp.linear_fc2", + "model.visual.blocks.19.attn.proj", + "model.visual.blocks.19.attn.qkv", + "model.visual.blocks.19.mlp.linear_fc1", + "model.visual.blocks.19.mlp.linear_fc2", + "model.visual.blocks.2.attn.proj", + "model.visual.blocks.2.attn.qkv", + "model.visual.blocks.2.mlp.linear_fc1", + "model.visual.blocks.2.mlp.linear_fc2", + "model.visual.blocks.20.attn.proj", + "model.visual.blocks.20.attn.qkv", + "model.visual.blocks.20.mlp.linear_fc1", + "model.visual.blocks.20.mlp.linear_fc2", + "model.visual.blocks.21.attn.proj", + "model.visual.blocks.21.attn.qkv", + "model.visual.blocks.21.mlp.linear_fc1", + "model.visual.blocks.21.mlp.linear_fc2", + "model.visual.blocks.22.attn.proj", + "model.visual.blocks.22.attn.qkv", + "model.visual.blocks.22.mlp.linear_fc1", + "model.visual.blocks.22.mlp.linear_fc2", + "model.visual.blocks.23.attn.proj", + "model.visual.blocks.23.attn.qkv", + "model.visual.blocks.23.mlp.linear_fc1", + "model.visual.blocks.23.mlp.linear_fc2", + "model.visual.blocks.24.attn.proj", + "model.visual.blocks.24.attn.qkv", + "model.visual.blocks.24.mlp.linear_fc1", + "model.visual.blocks.24.mlp.linear_fc2", + "model.visual.blocks.25.attn.proj", + "model.visual.blocks.25.attn.qkv", + "model.visual.blocks.25.mlp.linear_fc1", + "model.visual.blocks.25.mlp.linear_fc2", + "model.visual.blocks.26.attn.proj", + "model.visual.blocks.26.attn.qkv", + "model.visual.blocks.26.mlp.linear_fc1", + "model.visual.blocks.26.mlp.linear_fc2", + "model.visual.blocks.3.attn.proj", + "model.visual.blocks.3.attn.qkv", + "model.visual.blocks.3.mlp.linear_fc1", + "model.visual.blocks.3.mlp.linear_fc2", + "model.visual.blocks.4.attn.proj", + "model.visual.blocks.4.attn.qkv", + "model.visual.blocks.4.mlp.linear_fc1", + "model.visual.blocks.4.mlp.linear_fc2", + "model.visual.blocks.5.attn.proj", + "model.visual.blocks.5.attn.qkv", + "model.visual.blocks.5.mlp.linear_fc1", + "model.visual.blocks.5.mlp.linear_fc2", + "model.visual.blocks.6.attn.proj", + "model.visual.blocks.6.attn.qkv", + "model.visual.blocks.6.mlp.linear_fc1", + "model.visual.blocks.6.mlp.linear_fc2", + "model.visual.blocks.7.attn.proj", + "model.visual.blocks.7.attn.qkv", + "model.visual.blocks.7.mlp.linear_fc1", + "model.visual.blocks.7.mlp.linear_fc2", + "model.visual.blocks.8.attn.proj", + "model.visual.blocks.8.attn.qkv", + "model.visual.blocks.8.mlp.linear_fc1", + "model.visual.blocks.8.mlp.linear_fc2", + "model.visual.blocks.9.attn.proj", + "model.visual.blocks.9.attn.qkv", + "model.visual.blocks.9.mlp.linear_fc1", + "model.visual.blocks.9.mlp.linear_fc2", + "model.visual.merger.linear_fc1", + "model.visual.merger.linear_fc2", + "model.visual.patch_embed.proj", + "model.visual.pos_embed", + "mtp.fc", + "mtp.layers.0.mlp.gate", + "mtp.layers.0.mlp.shared_expert_gate" + ], + "quant_method": "fp8", + "weight_block_size": [ + 128, + 128 + ], + "weight_per_tensor": false + }, + "repo": "Qwen/Qwen3.5-397B-A17B-FP8" + } + ] + }, + "qwen4_exp": { + "config": { + "architectures": [ + "Qwen4ExpForConditionalGeneration" + ], + "image_token_id": 248056, + "language_model_only": false, + "model_type": "qwen4_exp", + "text_config": { + "attention_bias": false, + "attention_dropout": 0.0, + "bos_token_id": 248044, + "dtype": "bfloat16", + "eos_token_id": 248044, + "full_attention_interval": 4, + "hc_count": 4, + "hc_lowrank": 320, + "head_dim": 256, + "heads_per_ngram": 8, + "hidden_act": "silu", + "hidden_size": 2560, + "indexer_budget": 2048, + "indexer_compress_ratio": 4, + "indexer_head_dim": 128, + "indexer_kv_heads": 1, + "indexer_n_heads": 4, + "initializer_range": 0.02, + "layer_types": [ + "linear_attention", + "linear_attention", + "linear_attention", + "full_attention", + "linear_attention", + "linear_attention", + "linear_attention", + "full_attention", + "linear_attention", + "linear_attention", + "linear_attention", + "full_attention", + "linear_attention", + "linear_attention", + "linear_attention", + "full_attention", + "linear_attention", + "linear_attention", + "linear_attention", + "full_attention", + "linear_attention", + "linear_attention", + "linear_attention", + "full_attention", + "linear_attention", + "linear_attention", + "linear_attention", + "full_attention", + "linear_attention", + "linear_attention", + "linear_attention", + "full_attention", + "linear_attention", + "linear_attention", + "linear_attention", + "full_attention", + "linear_attention", + "linear_attention", + "linear_attention", + "full_attention", + "linear_attention", + "linear_attention", + "linear_attention", + "full_attention", + "linear_attention", + "linear_attention", + "linear_attention", + "full_attention" + ], + "linear_conv_kernel_dim": 4, + "linear_key_head_dim": 128, + "linear_num_key_heads": 16, + "linear_num_value_heads": 48, + "linear_value_head_dim": 128, + "make_ngram_vocab_size_divisible_by": 128, + "mamba_ssm_dtype": "float32", + "max_position_embeddings": 262144, + "model_type": "qwen4_exp_text", + "moe_intermediate_size": 640, + "mtp": { + "hybrid": true, + "layer_types": [ + "full_attention" + ], + "mtp_use_hidden_state_from_layer": null, + "num_hidden_layers": 1, + "rope_theta": 10000000 + }, + "mtp_num_hidden_layers": 1, + "mtp_use_dedicated_embeddings": false, + "ngram_size": 3, + "ngram_vocab_size_base": 20000000, + "num_attention_heads": 24, + "num_experts": 512, + "num_experts_per_tok": 10, + "num_hidden_layers": 48, + "num_key_value_heads": 2, + "output_gate_type": "sigmoid", + "output_router_logits": false, + "pad_token_id": null, + "partial_rotary_factor": 0.25, + "ple_conv_kernel_size": 4, + "ple_embed_dim": 2560, + "ple_layer_ids": [ + 2 + ], + "rms_norm_eps": 1e-06, + "rope_parameters": { + "mrope_interleaved": true, + "mrope_section": [ + 11, + 11, + 10 + ], + "partial_rotary_factor": 0.25, + "rope_theta": 10000000, + "rope_type": "default" + }, + "router_aux_loss_coef": 0.001, + "shared_expert_intermediate_size": 640, + "split_ngram_parts": 128, + "tie_word_embeddings": false, + "use_cache": true, + "vocab_size": 248320 + }, + "tie_word_embeddings": false, + "video_token_id": 248057, + "vision_config": { + "deepstack_visual_indexes": [], + "depth": 27, + "hidden_act": "gelu_pytorch_tanh", + "hidden_size": 1152, + "in_channels": 3, + "initializer_range": 0.02, + "intermediate_size": 4304, + "model_type": "qwen4_exp", + "num_heads": 16, + "num_position_embeddings": 2304, + "out_hidden_size": 2560, + "patch_size": 16, + "spatial_merge_size": 2, + "temporal_patch_size": 2 + }, + "vision_end_token_id": 248054, + "vision_start_token_id": 248053 + }, + "repo": "Qwen/Qwen3.8-Flash-Next-FP8", + "variants": [] + } +} \ No newline at end of file diff --git a/operatorx/runners/common/vllm/engine.py b/operatorx/runners/common/vllm/engine.py new file mode 100644 index 0000000000..5f14fc6029 --- /dev/null +++ b/operatorx/runners/common/vllm/engine.py @@ -0,0 +1,138 @@ +"""The vLLM engine a case runs under, configured as `vllm serve` configures it. + +engine_args(): the InferenceX recipe's serve arguments (OPERATORX_ENGINE_ARGS, less those +recipes.serve_argv does not apply) through vLLM's own parser, the op's parallel split and +the backend's overrides, with the external_launcher executor (one srun task per rank, +joined through env://). Layer ops enter that config once per process (context()); +attention builds an LLM from it. +""" +from __future__ import annotations + +import dataclasses +import json +import os +import tempfile + +import torch + +from operatorx.core import UnsupportedOpError, parallel + +# for layer cases without a checkpoint config; MoE, so vLLM builds the expert-parallel group +_STAND_IN = {"architectures": ["MixtralForCausalLM"], "model_type": "mixtral", "hidden_size": 4096, + "intermediate_size": 512, "num_attention_heads": 64, "num_key_value_heads": 8, + "num_hidden_layers": 1, "num_local_experts": 8, "num_experts_per_tok": 2, + "vocab_size": 1024, "max_position_embeddings": 65536, "torch_dtype": "bfloat16"} + +_STATE: dict | None = None +_META: dict = {} + + +def launch() -> None: + """The env:// rendezvous external_launcher reads (one device by default).""" + os.environ.setdefault("RANK", "0") + os.environ.setdefault("LOCAL_RANK", "0") + os.environ.setdefault("WORLD_SIZE", "1") + os.environ.setdefault("MASTER_ADDR", "127.0.0.1") + os.environ.setdefault("MASTER_PORT", str(29500 + os.getpid() % 1000)) + torch.cuda.set_device(int(os.environ["LOCAL_RANK"])) + + +def recipe_kwargs(without: tuple = ()) -> tuple[dict, dict]: + """The recipe's server arguments as EngineArgs kwargs (non-default fields only), and the + attention and compilation configs apart from them: attention is merged with the op's, and + an eager engine applies only the compilation config's custom-op selection.""" + from vllm.engine.arg_utils import EngineArgs + from vllm.utils.argparse_utils import FlexibleArgumentParser + + from operatorx.recipes import serve_argv + args = {k: v for k, v in json.loads(os.environ.get("OPERATORX_ENGINE_ARGS") or "{}").items() + if k not in without} + attention = json.loads(args.pop("attention-config", None) or "{}") + compilation = json.loads(args.pop("compilation-config", None) or "{}") + if args.get("max-cudagraph-capture-size"): # moved: vLLM rejects it in both places + compilation.setdefault("max_cudagraph_capture_size", int(args.pop("max-cudagraph-capture-size"))) + info = {"attention_config": attention, "compilation_config": compilation} + if not args: + return {}, info + parser = EngineArgs.add_cli_args(FlexibleArgumentParser()) + base = vars(parser.parse_known_args(["--model", "m"])[0]) + ns, frontend_only = parser.parse_known_args(["--model", "m", *serve_argv(args)]) + fields = {f.name for f in dataclasses.fields(EngineArgs)} - {"model"} + kwargs = {k: v for k, v in vars(ns).items() if k in fields and v != base.get(k)} + info["ignored_args"] = frontend_only + return kwargs, info + + +def engine_args(split: dict | None, without: tuple = (), eager: bool = False, defaults: dict | None = None, + axes: tuple = parallel.AXES, **overrides) -> tuple[dict, dict]: + """EngineArgs kwargs for a case: defaults, the recipe's, the split, the overrides (an + attention_config override merges into the recipe's); plus the recipe's attention and + compilation configs. Expert parallelism stays the recipe's when "ep" is not among the + op's axes (it does not change the world size).""" + kw, info = recipe_kwargs(without) + kw = {**(defaults or {}), **kw} + compilation = info["compilation_config"] + if not eager and compilation: + kw["compilation_config"] = compilation + elif compilation.get("custom_ops"): # custom-op selection holds without compilation + kw["compilation_config"] = {"custom_ops": compilation["custom_ops"]} + key = parallel.normalize(split) + recipe = json.loads(os.environ.get("OPERATORX_ENGINE_ARGS") or "{}") + ep = key["ep"] > 1 if "ep" in axes else bool(recipe.get("enable-expert-parallel")) + kw.update(tensor_parallel_size=key["tp"], data_parallel_size=key["dp"], + decode_context_parallel_size=key["dcp"], enable_expert_parallel=ep, + distributed_executor_backend="external_launcher") + attention = {**info["attention_config"], **overrides.pop("attention_config", {})} + kw.update(overrides) + if attention: + kw["attention_config"] = attention + _META.update(engine_args=kw, ignored_args=info.get("ignored_args", [])) + return kw, info + + +def full_graph_sizes(kw: dict, compilation: dict) -> list[int]: + """The sizes vLLM captures full CUDA graphs at for these arguments and the recipe's + compilation config, as a non-eager engine would; none when it captures no full graphs.""" + from vllm.engine.arg_utils import EngineArgs + cfg = EngineArgs(**{**kw, "enforce_eager": False, "compilation_config": compilation}).create_engine_config() + cc = cfg.compilation_config + return sorted(cc.cudagraph_capture_sizes or []) if cc.cudagraph_mode.has_full_cudagraphs() else [] + + +def _model_dir() -> str: + path = os.environ.get("OPERATORX_MODEL_CONFIG") + if path: + return path + path = tempfile.mkdtemp(prefix="opx_vllm_cfg_") + with open(os.path.join(path, "config.json"), "w") as f: + json.dump(_STAND_IN, f) + return path + + +def context(split: dict | None = None): + """Enter vLLM's config and the worker's distributed environment, once per process.""" + global _STATE + key = parallel.normalize(split) + if _STATE is not None: + if _STATE["split"] != key: + raise UnsupportedOpError(f"this process runs parallel={_STATE['split']}, not {key}") + return _STATE["config"] + from vllm.config import set_current_vllm_config + from vllm.engine.arg_utils import EngineArgs + from vllm.v1.worker.gpu_worker import init_worker_distributed_environment + from vllm.v1.worker.workspace import init_workspace_manager + + launch() + kw, _ = engine_args(split, model=_model_dir(), skip_tokenizer_init=True, load_format="dummy") + cfg = EngineArgs(**kw).create_engine_config() + ctx = set_current_vllm_config(cfg) + ctx.__enter__() + local_rank = int(os.environ["LOCAL_RANK"]) + init_worker_distributed_environment(cfg, int(os.environ["RANK"]), "env://", local_rank) + init_workspace_manager(torch.device("cuda", local_rank)) + _STATE = {"split": key, "config": cfg, "ctx": ctx} + return cfg + + +def meta() -> dict: + return {k: json.loads(json.dumps(v, default=str)) for k, v in _META.items()} diff --git a/operatorx/runners/common/vllm/linear.py b/operatorx/runners/common/vllm/linear.py new file mode 100644 index 0000000000..c8369b5a43 --- /dev/null +++ b/operatorx/runners/common/vllm/linear.py @@ -0,0 +1,325 @@ +"""Dense GEMM as a vLLM RowParallelLinear (K split over tp, all-reduced), so vLLM picks the kernel. + +Built under the quantization config a checkpoint with the operands' scheme would declare, with +synthetic checkpoint-format weights and process_weights_after_loading; layer(x) on a bf16 +activation, as a CUDA graph where vLLM would capture one. The chosen kernel is recorded. +""" +from __future__ import annotations + +import os +import sys + +import torch + +from operatorx.core import BackendImpl, Op, UnsupportedOpError, lookup_versions +from operatorx.runners.common import ranks +from operatorx.runners.common.vllm import engine + +PER_TENSOR, PER_TOKEN, PER_CHANNEL = [-1, -1], [1, -1], [-1, 1] + + +def device() -> "torch.device": + """cuda on NVIDIA and ROCm.""" + from vllm.platforms import current_platform + return torch.device(current_platform.device_type) + + +def sync() -> None: + getattr(torch, device().type).synchronize() + +_MX_FP4 = {"dtype": "fp4", "qscheme": "per_group", "ch_axis": -1, "group_size": 32, "symmetric": None, + "round_method": "half_even", "scale_type": "float", "scale_format": "e8m0", + "scale_calculation_mode": "even", "mx_element_dtype": None, "observer_cls": "PerBlockMXObserver", + "is_scale_quant": False} +# the quantization_config AMD's MXFP4 checkpoints ship (e.g. amd/Kimi-K2.5-MXFP4), exclusions dropped +_QUARK_MXFP4 = { + "quant_method": "quark", "quant_mode": "eager_mode", "exclude": [], "algo_config": None, + "global_quant_config": {"input_tensors": {**_MX_FP4, "is_dynamic": True}, + "weight": {**_MX_FP4, "is_dynamic": False}, + "output_tensors": None, "bias": None, "target_device": None}, + "layer_type_quant_config": {}, "layer_quant_config": {}, "kv_cache_quant_config": {}, + "export": {"kv_cache_group": [], "min_kv_scale": 0.0, "pack_method": "reorder", + "weight_format": "real_quantized", "weight_merge_groups": None}, +} + + +# Kimi-K3's routed experts: MXFP4 weights, bf16 activations (compressed-tensors mxfp4-pack) +_CT_MXFP4_W4A16 = { + "quant_method": "compressed-tensors", "format": "mxfp4-pack-quantized", "ignore": [], + "config_groups": {"group_0": { + "format": "mxfp4-pack-quantized", "targets": ["Linear"], "input_activations": None, + "output_activations": None, + "weights": {"actorder": None, "block_structure": None, "dynamic": False, "group_size": 32, "num_bits": 4, + "observer": "minmax", "observer_kwargs": {}, "scale_dtype": "torch.uint8", + "strategy": "group", "symmetric": True, "type": "float", "zp_dtype": None}}}} + + +def _scheme(qa: dict, qb: dict) -> tuple[str, dict] | None: + """Operands -> (quant method, checkpoint quantization_config); None: unquantized; (): unsupported.""" + if "scale" not in qa and "scale" not in qb: + return None if qa["dtype"] == qb["dtype"] == "bf16" else () + sa, sb = qa.get("scale"), qb.get("scale") + if (qa == {"dtype": "bf16"} and qb["dtype"] == "e2m1" and sb == {"dtype": "ue8m0", "static": True, "group": [1, 32]} + and "scale2" not in qb): + return "compressed-tensors", _CT_MXFP4_W4A16 + if sa is None or sb is None or not sb["static"] or not qa.get("symmetric", True) or not qb.get("symmetric", True): + return () + if qa["dtype"] == qb["dtype"] == "e4m3" and "scale2" not in qa and "scale2" not in qb: + ga, gb = sa["group"], sb["group"] + dyn = "static" if sa["static"] else "dynamic" + if ga == gb == PER_TENSOR and sa["dtype"] == sb["dtype"] == "fp32": + return "fp8", {"quant_method": "fp8", "activation_scheme": dyn} + if ga == gb == [1, 32] and sa["dtype"] == sb["dtype"] == "ue8m0" and not sa["static"]: + return "mxfp8", {"quant_method": "mxfp8", "activation_scheme": "dynamic", "weight_block_size": [1, 32], + "ignored_layers": []} + if gb[0] >= 1 and gb[1] > 1 and ga == [1, gb[1]] and not sa["static"] and ( + sa["dtype"] == sb["dtype"] or (sa["dtype"] == "fp32" and sb["dtype"] == "bf16")): + cfg = {"quant_method": "fp8", "activation_scheme": "dynamic", "fmt": "e4m3", "weight_block_size": gb} + if sb["dtype"] in ("fp32", "bf16"): # bf16 block scales load like fp32 ones + return "fp8", cfg + if sb["dtype"] == "ue8m0": + # DeepSeek-V4: vLLM remaps the checkpoint's fp8 config to deepseek_v4_fp8 + return "deepseek_v4_fp8", {**cfg, "scale_fmt": "ue8m0"} + if ga == PER_TOKEN and gb == PER_CHANNEL and not sa["static"]: + return "compressed-tensors", { + "quant_method": "compressed-tensors", "format": "float-quantized", "ignore": [], + "config_groups": {"group_0": { + "targets": ["Linear"], + "weights": {"num_bits": 8, "type": "float", "strategy": "channel", "dynamic": False, + "symmetric": True}, + "input_activations": {"num_bits": 8, "type": "float", "strategy": "token", "dynamic": True, + "symmetric": True}}}} + if (qa["dtype"] == qb["dtype"] == "e2m1" and sa["group"] == sb["group"] == [1, 32] and not sa["static"] + and sa["dtype"] == sb["dtype"] == "ue8m0" and "scale2" not in qa and "scale2" not in qb): + return "quark", _QUARK_MXFP4 + if (qa["dtype"] == qb["dtype"] == "e2m1" and sa["group"] == sb["group"] == [1, 16] and not sa["static"] + and sa["dtype"] == sb["dtype"] == "e4m3" and qa.get("scale2") and qb.get("scale2")): + return "modelopt_fp4", {"quantization": { + "quant_algo": "NVFP4", "group_size": 16, "kv_cache_quant_algo": None, "exclude_modules": []}} + return () + + +# env that steers vLLM's linear-kernel choice; recorded with every result +_ENV_KEYS = ("VLLM_USE_DEEP_GEMM", "VLLM_USE_DEEP_GEMM_E8M0", "VLLM_BLOCKSCALE_FP8_GEMM_FLASHINFER", + "VLLM_ROCM_USE_AITER", "VLLM_ROCM_USE_AITER_LINEAR") + +def versions() -> dict[str, str]: + return lookup_versions("vllm", "torch") + + +def _quant_config(args): + sch = _scheme(args["a"], args["b"]) + if sch is None: + return None + if not sch: + raise UnsupportedOpError(f"no vLLM quantization config for a={args['a']} b={args['b']}") + return quant_config(*sch) + + +def quant_config(method: str, cfg: dict): + from vllm.model_executor.layers.quantization import QUANTIZATION_METHODS, get_quantization_config + if method not in QUANTIZATION_METHODS: + raise UnsupportedOpError(f"vLLM has no {method!r} quantization method") + return get_quantization_config(method).from_config(cfg) + + +def _reject_fallback(qc, method) -> None: + """An unquantized fallback method would record a bf16 matmul under the quantized row + (vLLM's mxfp4 does this for linear layers; only its MoE experts have a kernel).""" + if qc is not None and "Unquantized" in type(method).__name__: + raise UnsupportedOpError( + f"{type(qc).__name__} has no quantized linear method here; the layer fell back " + f"to {type(method).__name__}") + + +def _set_quant_fp8_op(qc) -> None: + """VllmConfig enables +quant_fp8 for blocked-weight checkpoints; mirror it per layer + (CustomOp reads it at build).""" + from vllm.config import get_current_vllm_config + ops = get_current_vllm_config().compilation_config.custom_ops + blocked = getattr(qc, "weight_block_size", None) is not None + if blocked and "+quant_fp8" not in ops: + ops.append("+quant_fp8") + elif not blocked and "+quant_fp8" in ops: + ops.remove("+quant_fp8") + + +def _is_fault(e: BaseException) -> bool: + """Allocation failures and device faults are errors, never 'unsupported'.""" + msg = str(e) + return isinstance(e, torch.OutOfMemoryError) or any( + s in msg for s in ("CUDA error", "HIP error", "illegal memory", "out of memory", "device-side assert")) + + +def _fill(layer: torch.nn.Module) -> None: + for name, p in layer.named_parameters(recurse=False): + with torch.no_grad(): + if p.dtype in (torch.float8_e4m3fn, torch.float8_e5m2, torch.float8_e4m3fnuz): + p.copy_((torch.randn(p.shape, device=p.device) * 0.5).to(p.dtype)) + elif p.dtype in (torch.uint8, torch.int8, torch.int32): + p.copy_(torch.randint(0, 127, p.shape, device=p.device, dtype=p.dtype)) + elif "scale" in name: + p.fill_(0.01) + else: + p.copy_((torch.randn(p.shape, device=p.device) * 0.02).to(p.dtype)) + + +def _kernel_names(layer) -> dict[str, str]: + qm = layer.quant_method + out = {} + for owner in (qm, getattr(qm, "scheme", None), getattr(layer, "scheme", None)): + if owner is None: + continue + if owner is not qm: + out["scheme"] = type(owner).__name__ + for k, v in vars(owner).items(): + t = type(v) + if t.__module__.startswith("vllm.model_executor.kernels") or t.__name__.endswith("Kernel"): + out[k] = t.__name__ + return out + + +def _param_dtypes(layer) -> dict[str, str]: + return {n: str(p.dtype).removeprefix("torch.") for n, p in layer.named_parameters(recurse=False)} + + +def _prepare_gemm(op: Op) -> dict: + a = op.args + if a.get("activation") is not None: + raise UnsupportedOpError(f"vllm linear has no fused activation; got {a['activation']!r}") + m, n, k = a["m"], a["n"], a["k"] + if min(m, n, k) <= 0: + raise UnsupportedOpError(f"degenerate gemm shape m={m} n={n} k={k}") + if a["a"].get("input", "bf16") != "bf16": + raise UnsupportedOpError("vLLM linear layers take a bf16 activation") + if a.get("out", "bf16") == "fp32" or a["b"]["dtype"] == "fp32": + return _prepare_fp32_out(op) + if a.get("out", "bf16") != "bf16": + raise UnsupportedOpError(f"vLLM linear layers return bf16 or fp32; got {a['out']!r}") + tp = (a.get("parallel") or {}).get("tp", 1) + if k % tp: + raise UnsupportedOpError(f"k={k} does not split {tp} ways") + engine.context(a.get("parallel")) + import vllm.envs as envs + from vllm.model_executor.layers.linear import RowParallelLinear + with ranks.together("build this layer"): + qc = _quant_config(a) + _set_quant_fp8_op(qc) + prev = torch.get_default_dtype() + torch.set_default_dtype(torch.bfloat16) # as vLLM's model loader does while building layers + try: + layer = RowParallelLinear(k, n, bias=bool(a.get("bias")), quant_config=qc, params_dtype=torch.bfloat16, + reduce_results=True, prefix="model.layers.0.mlp.down_proj").to(device()) + _fill(layer) + loaded = _param_dtypes(layer) + qm = layer.quant_method + _reject_fallback(qc, qm) + if hasattr(qm, "process_weights_after_loading"): + qm.process_weights_after_loading(layer) + except (NotImplementedError, AssertionError, ValueError, RuntimeError) as e: + if _is_fault(e): + raise + raise UnsupportedOpError(f"vLLM rejected {a}: {type(e).__name__}: {e}"[:400]) from e + finally: + torch.set_default_dtype(prev) + kernels = _kernel_names(layer) + if any(v.startswith("Emulation") for v in kernels.values()): + raise UnsupportedOpError(f"vLLM has only an emulation kernel for {a} here: {kernels}") + x = torch.randn(m, layer.input_size_per_partition, device=device(), dtype=torch.bfloat16) + ctx = {"layer": layer, "x": x, + "meta": {"vllm_quant_method": type(qm).__name__, "vllm_kernels": kernels, + "param_dtypes_loaded": loaded, "param_dtypes": _param_dtypes(layer), + "vllm_env": {k: getattr(envs, k) for k in _ENV_KEYS if hasattr(envs, k)}, + "vllm_engine": engine.meta()}} + with ranks.together("run this layer"): + try: + _kernel_gemm(ctx) + sync() + except (NotImplementedError, AssertionError, RuntimeError, ValueError) as e: + if _is_fault(e): + raise + raise UnsupportedOpError(f"vLLM kernel failed for {a}: {type(e).__name__}: {e}"[:400]) from e + return ctx + + +class _TorchMM(torch.nn.Module): + """vLLM's fp32-output gate/router matmul (GateLinear's cuBLAS tier).""" + + def __init__(self, k: int, n: int, dtype: torch.dtype, bias: bool): + super().__init__() + self.weight = torch.nn.Parameter(torch.randn(n, k, device=device()).mul_(0.02).to(dtype), + requires_grad=False) + self.bias = torch.nn.Parameter(torch.zeros(n, device=device(), dtype=dtype), + requires_grad=False) if bias else None + + def forward(self, x: torch.Tensor) -> torch.Tensor: + if self.weight.dtype == torch.bfloat16 and self.bias is None: + return torch.mm(x, self.weight.T, out_dtype=torch.float32) + return torch.nn.functional.linear(x.to(self.weight.dtype), self.weight, self.bias).float() + + +def _prepare_fp32_out(op: Op) -> dict: + a = op.args + qa, qb = a["a"], a["b"] + if "scale" in qa or "scale" in qb or qa["dtype"] != "bf16" or qb["dtype"] not in ("bf16", "fp32") \ + or a.get("out", "bf16") != "fp32": + raise UnsupportedOpError(f"no fp32-output vLLM matmul for a={qa} b={qb} out={a.get('out')}") + if (a.get("parallel") or {}).get("tp", 1) > 1: + raise UnsupportedOpError("vLLM's fp32-output gates and routers are replicated, never split") + engine.context(a.get("parallel")) + layer = _TorchMM(a["k"], a["n"], getattr(torch, {"bf16": "bfloat16", "fp32": "float32"}[qb["dtype"]]), + bool(a.get("bias"))) + ctx = {"layer": layer, "x": torch.randn(a["m"], a["k"], device=device(), dtype=torch.bfloat16), + "meta": {"vllm_quant_method": "torch.mm", "vllm_kernels": {}, + "param_dtypes_loaded": _param_dtypes(layer), "param_dtypes": _param_dtypes(layer)}} + _kernel_gemm(ctx) + sync() + return ctx + + +def _kernel_gemm(ctx: dict) -> None: + ctx["out"] = ctx["layer"](ctx["x"]) + + +def _capture_sizes() -> list[int]: + from vllm.config import get_current_vllm_config + return sorted(get_current_vllm_config().compilation_config.cudagraph_capture_sizes or []) + + +def _launcher(ctx: dict): + """(callable, cuda_graph): as served, a batch up to the largest capture size replays the next + size's graph, larger ones run eagerly. OPERATORX_CUDA_GRAPH_MAX_TOKENS caps it (0 = never).""" + from vllm.distributed.parallel_state import graph_capture + x = ctx["x"] + eager = (lambda: _kernel_gemm(ctx)), False # noqa: E731 + sizes = _capture_sizes() + cap = int(os.environ.get("OPERATORX_CUDA_GRAPH_MAX_TOKENS", sizes[-1] if sizes else 0)) + size = next((s for s in sizes if s >= x.shape[0]), None) + if cap <= 0 or size is None or size > cap: + return eager + xp = torch.randn(size, *x.shape[1:], device=x.device, dtype=x.dtype) + err = None + try: + for _ in range(2): + ctx["layer"](xp) + torch.cuda.synchronize() + g = torch.cuda.CUDAGraph() + with graph_capture(x.device) as gc, torch.cuda.graph(g, pool=torch.cuda.graph_pool_handle(), + stream=gc.stream): + ctx["graph_out"] = ctx["layer"](xp) + torch.cuda.synchronize() + except Exception as e: + if _is_fault(e): + raise + torch.cuda.synchronize() + err = e + if not ranks.agree(err is None): # every rank replays, or every rank runs eagerly + print(f"[vllm.linear] CUDA-graph capture failed, timing eagerly: {type(err).__name__}: {err}"[:300] + if err else "[vllm.linear] another rank's CUDA-graph capture failed, timing eagerly", + file=sys.stderr) + return eager + ctx["graph"], ctx["x_padded"] = g, xp + return g.replay, True + + +IMPLS = [BackendImpl(op_type="gemm", prepare=_prepare_gemm, kernel=_kernel_gemm, launcher=_launcher)] diff --git a/operatorx/runners/common/vllm/modules.py b/operatorx/runners/common/vllm/modules.py new file mode 100644 index 0000000000..b3326a9678 --- /dev/null +++ b/operatorx/runners/common/vllm/modules.py @@ -0,0 +1,72 @@ +"""A checkpoint's own vLLM MoE module, built as its decoder layer builds it, with dummy +weights and post-load processing (opt-in: OPERATORX_MODEL_MODULES=1).""" +from __future__ import annotations + +import importlib + +import torch + + +def _layer(pkg: str): + from vllm.platforms import current_platform + return importlib.import_module(f"vllm.models.{pkg}.{'amd' if current_platform.is_rocm() else 'nvidia'}.model") + + +def _deepseek_v4(pkg): + def build(cfg, hf, op_args): + m = _layer(pkg) + hash_layers = getattr(hf, "num_hash_layers", None) or 0 + i = 0 if op_args["router"]["select"]["kind"] == "hash" else hash_layers + kw = {"num_hash_layers": hash_layers} if pkg == "deepseek_v4" else {} + return m.DeepseekV4MoE(cfg, prefix=f"model.layers.{i}.ffn", + use_sequence_parallel=m._use_sequence_parallel(cfg), **kw) + return build + + +def _minimax_m3(cfg, hf, op_args): + m = _layer("minimax_m3") + i = next(i for i in range(hf.num_hidden_layers) if m._is_moe_layer(hf, i)) + # reduce here: the op includes the reduction the model fuses into the next RMSNorm + return m.MiniMaxM3MoE(config=hf, layer_id=i, quant_config=cfg.quant_config, reduce_results=True, + prefix=f"model.layers.{i}.block_sparse_moe") + + +def _kimi_k3(cfg, hf, op_args): + m = _layer("kimi_k3") + i = hf.first_k_dense_replace + p = cfg.parallel_config + use_sp = (p.pipeline_parallel_size == 1 and p.enable_expert_parallel and p.tensor_parallel_size > 1 + and (cfg.kernel_config.moe_backend == "deep_gemm_mega_moe" or p.data_parallel_size > 1)) + return m.KimiMoE(config=hf, vllm_config=cfg, quant_config=cfg.quant_config, + prefix=f"model.layers.{i}.block_sparse_moe", layer_idx=i, use_sequence_parallel=use_sp) + + +BUILDERS = { + "DeepseekV4ForCausalLM": _deepseek_v4("deepseek_v4"), + "DeepseekV41ForCausalLM": _deepseek_v4("deepseek_v41"), + "MiniMaxM3SparseForConditionalGeneration": _minimax_m3, + "KimiK3ForConditionalGeneration": _kimi_k3, +} + + +def build(op_args: dict) -> torch.nn.Module | None: + """None when the engine's model is a stand-in or has no entry here.""" + from vllm.config import get_current_vllm_config + from vllm.model_executor.model_loader import get_model_loader + from vllm.model_executor.model_loader.utils import process_weights_after_loading + + cfg = get_current_vllm_config() + arch = next((a for a in (cfg.model_config.architectures or ()) if a in BUILDERS), None) + if arch is None: + return None + device = torch.device("cuda", torch.cuda.current_device()) + prev = torch.get_default_dtype() + torch.set_default_dtype(cfg.model_config.dtype) + try: + with device: + module = BUILDERS[arch](cfg, cfg.model_config.hf_text_config, op_args) + finally: + torch.set_default_dtype(prev) + get_model_loader(cfg.load_config).load_weights(module, cfg.model_config) + process_weights_after_loading(module, cfg.model_config, device) + return module diff --git a/operatorx/runners/common/vllm/moe.py b/operatorx/runners/common/vllm/moe.py new file mode 100644 index 0000000000..e134c685bd --- /dev/null +++ b/operatorx/runners/common/vllm/moe.py @@ -0,0 +1,450 @@ +"""MoE layer through vLLM's MoE pipeline, so vLLM picks router, expert and shared-expert kernels. + +A GateLinear router, FusedMoEFactory experts and a shared-expert MLP under the quant configs the +descriptors imply, synthetic weights + process_weights_after_loading, timed on bf16 x (CUDA graph +where vLLM would capture one). Splits shard as vLLM's parallel config does; shared experts and +sequence-parallel chunking follow DeepseekV2MLP / DeepseekV2MoE. +""" +from __future__ import annotations + +import os + +import torch + +from operatorx.core import BackendImpl, Op, UnsupportedOpError +from operatorx.runners.common import ranks +from operatorx.runners.common.vllm import engine, modules +from operatorx.runners.common.vllm import linear as vllm_linear +from operatorx.runners.common.vllm.linear import (_ENV_KEYS, _fill, _is_fault, _launcher, device, sync, + versions) + +__all__ = ["IMPLS", "versions"] + +_DTYPES = {"bf16": torch.bfloat16, "fp32": torch.float32} +_MAX_TOKENS = 65536 # the largest token count in the testlists +_LAYER = 0 + + +def _quant(x: dict, w: dict, where: str): + """(activation, weight) descriptors -> vLLM quantization config, or None for bf16.""" + if x.get("input", "bf16") != "bf16": + raise UnsupportedOpError(f"{where}: vLLM MoE layers take a bf16 input") + sch = vllm_linear._scheme(x, w) + if sch is None: + return None + if not sch: + raise UnsupportedOpError(f"{where}: no vLLM quantization config for x={x} w={w}") + return vllm_linear.quant_config(*sch) + + +def _act_kwargs(act: dict) -> dict: + kind = act["kind"] + if not act.get("gated", True): + raise UnsupportedOpError("only gated expert activations are wired") + if kind == "silu": + return {"activation": "silu", "swiglu_limit": act.get("limit")} + if kind == "swigluoai": + return {"activation": "swigluoai" if act.get("interleaved") else "swigluoai_uninterleave", + "swiglu_limit": act.get("limit", 7.0), "swiglu_alpha": act.get("alpha", 1.702), + "swiglu_beta": act.get("beta")} + if kind == "situ": + return {"activation": "situ", "activation_situ_beta": act.get("alpha", 1.0), + "activation_situ_linear_beta": act.get("beta")} + raise UnsupportedOpError(f"activation {kind!r} is not wired for vLLM MoE") + + +def _act_fn(act: dict): + from vllm.model_executor.layers import activation as A + kind = act["kind"] + if kind == "silu": + return A.SiluAndMulWithClamp(act["limit"]) if act.get("limit") else A.SiluAndMul() + if kind == "swigluoai": + if act.get("interleaved"): + return A.SwigluOAIAndMul(alpha=act.get("alpha", 1.702), limit=act.get("limit", 7.0)) + return A.SiluAndMulWithClamp(act.get("limit", 7.0), alpha=act.get("alpha", 1.702), + beta=act.get("beta") or 0.0) + if kind == "situ": + return A.SituAndMul(beta=act.get("alpha", 1.0), linear_beta=act.get("beta")) + raise UnsupportedOpError(f"shared-expert activation {kind!r} is not wired") + + +def _dp_tokens(cache: dict, T: int): + """DP ranks' token counts, coordinated as the model runner does per step (untimed).""" + if T not in cache: + from vllm.config import get_current_vllm_config + from vllm.v1.worker.dp_utils import coordinate_batch_across_dp + cfg = get_current_vllm_config() + graph = int(bool(cfg.compilation_config.cudagraph_capture_sizes)) + cache[T] = coordinate_batch_across_dp(T, False, cfg.parallel_config, cudagraph_mode=graph)[1] + return cache[T] + + +class _SharedMLP(torch.nn.Module): + + def __init__(self, hidden: int, inter: int, act: dict, qc, prefix: str, expert_gate=None, + sequence_parallel: bool = False): + super().__init__() + self.expert_gate = expert_gate # Qwen: sigmoid(gate(x)) scales the shared output + from vllm.model_executor.layers.linear import MergedColumnParallelLinear, RowParallelLinear + self.gate_up_proj = MergedColumnParallelLinear(hidden, [inter] * 2, bias=False, quant_config=qc, + disable_tp=sequence_parallel, + prefix=f"{prefix}.gate_up_proj") + self.down_proj = RowParallelLinear(inter, hidden, bias=False, quant_config=qc, reduce_results=False, + disable_tp=sequence_parallel, prefix=f"{prefix}.down_proj") + self.act_fn = _act_fn(act) + + def forward(self, x): + h, _ = self.gate_up_proj(x) + h, _ = self.down_proj(self.act_fn(h)) + if self.expert_gate is not None: + h = torch.sigmoid(self.expert_gate(x)[0]) * h + return h + + +# DeepSeek-V4 routed experts: MXFP4 weights under the checkpoint's fp8 (ue8m0) config, which vLLM +# routes to its MXFP4 MoE method (expert_dtype "fp4") +_DSV4_FP4_EXPERTS = ("deepseek_v4_fp8", {"quant_method": "fp8", "activation_scheme": "dynamic", "fmt": "e4m3", + "scale_fmt": "ue8m0", "weight_block_size": [128, 128]}) + + +def _expert_quant(x: dict, w: dict): + if (x.get("dtype") == "e4m3" and x.get("scale", {}).get("dtype") == "ue8m0" and w["dtype"] == "e2m1" + and w.get("scale") == {"dtype": "ue8m0", "static": True, "group": [1, 32]} and "scale2" not in w): + return vllm_linear.quant_config(*_DSV4_FP4_EXPERTS) + return _quant(x, w, "experts") + + +def _scores(logits: torch.Tensor, scoring: str) -> torch.Tensor: + logits = logits.float() + if scoring == "softmax": + return torch.softmax(logits, dim=-1) + if scoring == "sigmoid": + return torch.sigmoid(logits) + return torch.sqrt(torch.nn.functional.softplus(logits)) # sqrtsoftplus + + +def _forced_ids(dist, tokens: int, experts: int, top_k: int, seed: int) -> torch.Tensor: + """[tokens, top_k] expert ids with the requested expert-load distribution.""" + g = torch.Generator().manual_seed(seed) + if dist == "balanced": # round-robin: every expert gets tokens*top_k/experts slots + base = torch.arange(tokens * top_k) % experts + ids = base.view(tokens, top_k) + return ids[:, torch.randperm(top_k, generator=g)] + if dist == "single_hot": + rest = torch.stack([torch.randperm(experts - 1, generator=g)[: top_k - 1] + 1 for _ in range(tokens)]) + return torch.cat([torch.zeros(tokens, 1, dtype=torch.long), rest], dim=1) + p = 1.0 / torch.arange(1, experts + 1, dtype=torch.float64) ** dist["s"] # zipf over expert rank + ranked = torch.randperm(experts, generator=g) + return ranked[torch.multinomial(p.expand(tokens, -1), top_k, replacement=False, generator=g)] + + +def _kimi_latent(): + """Kimi-K3's latent-MoE runner and output transform (platform-specific modules).""" + from vllm.platforms import current_platform + if current_platform.is_rocm(): + from vllm.models.kimi_k3.amd.latent_moe_runner import ROCmLatentMoERunner as LatentMoERunner + from vllm.models.kimi_k3.amd.linear import KimiRoutedOutputTransform + return LatentMoERunner, KimiRoutedOutputTransform, 256 + from vllm.models.kimi_k3.nvidia.latent_moe_runner import LatentMoERunner + from vllm.models.kimi_k3.nvidia.model import (_ROUTED_DOWN_PROJ_STREAM_TOKEN_THRESHOLD, + KimiRoutedOutputTransform) + return LatentMoERunner, KimiRoutedOutputTransform, _ROUTED_DOWN_PROJ_STREAM_TOKEN_THRESHOLD + + +class _ForcedRouting: + """vLLM custom_routing_function: fixed expert choice, the router's weights for it.""" + + def __init__(self, owner: torch.nn.Module, scoring: str): + self.owner, self.scoring = owner, scoring + + def __call__(self, hidden_states, gating_output, topk, renormalize): + ids = self.owner.forced_ids[: gating_output.shape[0]] + w = _scores(gating_output, self.scoring).gather(1, ids.long()) + if renormalize: + w = w / w.sum(dim=-1, keepdim=True) + return w, ids + + +class _MoeBlock(torch.nn.Module): + """Router + routed experts (+ shared experts), wired as vLLM's model blocks wire them.""" + + def __init__(self, a: dict, prefix: str): + super().__init__() + from vllm.config import get_current_vllm_config + from vllm.model_executor.layers.fused_moe.layer import FusedMoEFactory + from vllm.model_executor.layers.fused_moe.router.gate_linear import GateLinear + from vllm.model_executor.layers.fused_moe.utils import resolve_layer_fused_shared_expert + ex, rt, sh, act = a["experts"], a["router"], a.get("shared"), a["activation"] + if ex.get("zero"): + raise UnsupportedOpError("zero experts are not wired for vLLM MoE yet") + if rt.get("weight_on_input"): + raise UnsupportedOpError("router weight on the expert input is not wired") + if ex["w1"] != ex["w2"] or ex["a2"] != ex["a1"]: + raise UnsupportedOpError("vLLM MoE takes one scheme for w1/w2 and for a1/a2") + qc = _expert_quant(ex["a1"], ex["w1"]) + vllm_linear._set_quant_fp8_op(qc) + H, E, K = a["hidden"], ex["num"], ex["top_k"] + self.sequence_parallel = get_current_vllm_config().parallel_config.use_sequence_parallel_moe + self._dp_tokens: dict[int, torch.Tensor | None] = {} + L = ex.get("latent") or H + sel = rt["select"] + routing = a.get("routing") or {"distribution": "natural", "seed": 0} + logits = _DTYPES[rt["gate"].get("logits", rt["gate"]["dtype"])] + self.gate = GateLinear(H, E, params_dtype=_DTYPES[rt["gate"]["dtype"]], + out_dtype=None if logits == torch.bfloat16 else logits, prefix=f"{prefix}.gate") + self.gate.e_score_correction_bias = ( + torch.nn.Parameter(torch.zeros(E, dtype=torch.float32)) if rt.get("bias") else None) + self.gate.tid2eid = None + self.register_buffer("input_ids", None) + if sel["kind"] == "hash": # DeepSeek-V4 hash layers: token id -> experts table + g = torch.Generator().manual_seed(routing["seed"]) + self.gate.tid2eid = torch.nn.Parameter( + torch.randint(0, E, (sel["vocab"], K), generator=g, dtype=torch.int32), requires_grad=False) + self.register_buffer("input_ids", torch.randint(0, sel["vocab"], (_MAX_TOKENS,), generator=g)) + custom = None + if routing["distribution"] != "natural": + if sel["kind"] == "hash": + raise UnsupportedOpError("a forced expert-load distribution conflicts with hash routing") + ids = _forced_ids(routing["distribution"], _MAX_TOKENS, E, K, routing["seed"]) + self.register_buffer("forced_ids", ids.to(torch.int32)) + custom = _ForcedRouting(self, rt["scoring"]) + shared, shared_gate, fused_shared = None, None, False + if sh is not None: + if sh["w1"] != sh["w2"]: + raise UnsupportedOpError("shared experts take one weight scheme for w1/w2") + sqc = _quant(sh["a1"], sh["w1"], "shared") + if all(sh[k] == ex[k] for k in ("a1", "w1", "w2")) and not ex.get("latent"): + fused_shared = resolve_layer_fused_shared_expert(qc, prefix) + expert_gate = None + if sh.get("gate") == "sigmoid": + from vllm.model_executor.layers.linear import ReplicatedLinear + expert_gate = ReplicatedLinear(H, 1, bias=False, quant_config=None, disable_tp=True, + prefix=f"{prefix}.shared_expert_gate") + if fused_shared: # as Qwen's block: the runner takes the gate only with fused shared experts + shared_gate = expert_gate + else: + shared = _SharedMLP(H, sh["inter"] * sh["count"], act, sqc, f"{prefix}.shared_experts", + expert_gate=expert_gate, sequence_parallel=self.sequence_parallel) + latent = {} + self.down_proj = None + if ex.get("latent"): # Kimi-K3: routed experts run at width L between bf16 projections + from vllm.model_executor.layers.layernorm import RMSNorm + from vllm.model_executor.layers.linear import ReplicatedLinear + LatentMoERunner, KimiRoutedOutputTransform, self._stream_tokens = _kimi_latent() + self.down_proj = ReplicatedLinear(H, L, bias=False, quant_config=None, + prefix=f"{prefix}.routed_expert_down_proj") + norm = RMSNorm(L) if ex.get("latent_norm") else None + up = ReplicatedLinear(L, H, bias=False, quant_config=None, prefix=f"{prefix}.routed_expert_up_proj") + self.up_transform = KimiRoutedOutputTransform(norm, up) + latent = {"routed_output_transform": self.up_transform, "runner_cls": LatentMoERunner} + # a forced expert choice replaces selection, so vLLM's grouped/bias selection is off + grouped = sel["kind"] == "grouped_topk" and custom is None + bias = None if custom is not None else self.gate.e_score_correction_bias + self.experts_quant_config = qc + self.experts = FusedMoEFactory( + num_experts=E, top_k=K, hidden_size=L, intermediate_size=ex["inter"], + renormalize=bool(rt.get("renormalize")), quant_config=qc, use_grouped_topk=grouped, + num_expert_group=sel["groups"] if grouped else None, + topk_group=sel["topk_groups"] if grouped else None, + prefix=f"{prefix}.experts", scoring_func=rt["scoring"], + routed_scaling_factor=rt.get("scale") or 1.0, + e_score_correction_bias=bias, + hash_indices_table=self.gate.tid2eid, custom_routing_function=custom, + has_bias=bool(ex.get("bias")), reduce_results=True, is_sequence_parallel=self.sequence_parallel, + n_shared_experts=sh["count"] if fused_shared else None, fuse_shared_experts=fused_shared, + router_logits_dtype=self.gate.out_dtype, gate=None if ex.get("latent") else self.gate, + shared_experts=shared, shared_expert_gate=shared_gate, **latent, **_act_kwargs(act)) + self.shared = shared + self.fused_shared = fused_shared + if ex.get("latent") and device().type == "cuda": + from vllm.utils.torch_utils import aux_stream + self._aux = aux_stream() + self._events = (torch.cuda.Event(), torch.cuda.Event()) + + def dp_tokens(self, T: int): + return _dp_tokens(self._dp_tokens, T) + + def forward(self, x): + from vllm.config import get_current_vllm_config + from vllm.forward_context import set_forward_context + T = x.shape[0] + with set_forward_context(None, get_current_vllm_config(), num_tokens=T, + num_tokens_across_dp=self.dp_tokens(T)): + if not self.sequence_parallel: + return self._experts(x) + from vllm.distributed import tensor_model_parallel_all_gather + from vllm.model_executor.models.utils import sequence_parallel_chunk + return tensor_model_parallel_all_gather(self._experts(sequence_parallel_chunk(x)), 0)[:T] + + def _experts(self, x): + T = x.shape[0] + if self.down_proj is None or not hasattr(self, "_aux"): + if self.down_proj is not None: # latent projection without an aux stream + lat, _ = self.down_proj(x) + return self.experts(hidden_states=lat, router_logits=self.gate(x)[0], + shared_experts_input=x) + ids = None if self.input_ids is None else self.input_ids[:T] + return self.experts(hidden_states=x, router_logits=x, input_ids=ids) + # as Kimi-K3's block: router and latent down projection on two streams at decode sizes + from vllm.utils.multi_stream_utils import maybe_execute_in_parallel + logits, (lat, _) = maybe_execute_in_parallel( + lambda: self.gate(x)[0], lambda: self.down_proj(x), self._events[0], self._events[1], + self._aux if T <= self._stream_tokens else None) + return self.experts(hidden_states=lat, router_logits=logits, shared_experts_input=x) + + +class _ModelBlock(torch.nn.Module): + """A checkpoint's own MoE module (modules.py), called as its decoder layer calls it.""" + + def __init__(self, module: torch.nn.Module): + super().__init__() + import inspect + + from vllm.config import get_current_vllm_config + self.module = module + self.sequence_parallel = False # handled inside the module + self.fused_shared = None + self._dp_tokens: dict[int, torch.Tensor | None] = {} + self.takes_ids = "input_ids" in inspect.signature(module.forward).parameters + vocab = get_current_vllm_config().model_config.hf_text_config.vocab_size + self.register_buffer("input_ids", torch.randint(0, vocab, (_MAX_TOKENS,), device=device())) + + def dp_tokens(self, T: int): + return _dp_tokens(self._dp_tokens, T) + + def forward(self, x): + from vllm.config import get_current_vllm_config + from vllm.forward_context import set_forward_context + T = x.shape[0] + with set_forward_context(None, get_current_vllm_config(), num_tokens=T, + num_tokens_across_dp=self.dp_tokens(T)): + if self.takes_ids: + return self.module(x, input_ids=self.input_ids[:T]) + return self.module(x) + + +def _describe(block) -> dict: + """Quant method and kernel objects vLLM attached to every quantized submodule.""" + out = {} + for name, m in block.named_modules(): + qm = getattr(m, "quant_method", None) + if qm is None or not hasattr(qm, "process_weights_after_loading"): + continue + entry = {"method": type(qm).__name__} + for owner in (qm, getattr(qm, "scheme", None), getattr(m, "scheme", None)): + for k, v in (vars(owner).items() if owner is not None else ()): + t = type(v).__name__ + if any(t.endswith(s) for s in ("Kernel", "Experts", "PrepareAndFinalize", "Impl")): + entry[k] = t + impl = getattr(v, "impl", None) # FusedMoEKernel: the experts and dispatch it runs + for part in ("fused_experts", "prepare_finalize"): + if getattr(impl, part, None) is not None: + entry[f"{k}.{part}"] = type(getattr(impl, part)).__name__ + out[name or "."] = entry + return out + + +def _release_previous() -> None: + """Drop the previous op's layers from vLLM's forward context so their memory is freed.""" + import gc + + from vllm.config import get_current_vllm_config + ctx = get_current_vllm_config().compilation_config.static_forward_context + for name in [k for k in ctx if k.startswith("model.layers.")]: + del ctx[name] + gc.collect() + torch.cuda.empty_cache() + + +def _missing_op(e: BaseException) -> bool: + """A vLLM custom op this build does not ship (e.g. Marlin repack on ROCm).""" + return isinstance(e, AttributeError) and "_OpNamespace" in str(e) + + +def _generic_block(a: dict, prefix: str) -> _MoeBlock: + from vllm.model_executor.layers.quantization.base_config import QuantizeMethodBase + block = _MoeBlock(a, prefix).to(device()) + for m in block.modules(): + _fill(m) + if block.gate.e_score_correction_bias is not None: + block.gate.e_score_correction_bias.data.zero_() + if block.gate.tid2eid is not None: # the filler knows nothing of expert ids + block.gate.tid2eid.data.random_(0, a["experts"]["num"]) + methods = [] + for m in block.modules(): + qm = getattr(m, "quant_method", None) + if isinstance(qm, QuantizeMethodBase): + methods.append(type(qm).__name__) + qm.process_weights_after_loading(m) + # an all-unquantized block computed bf16 under a quantized row; a bf16 shared expert alone is normal + if block.experts_quant_config is not None and methods and all("Unquantized" in n for n in methods): + raise UnsupportedOpError( + f"{type(block.experts_quant_config).__name__} has no quantized MoE method here; " + f"the layer fell back to {sorted(set(methods))}") + return block + + +def _prepare_moe(op: Op) -> dict: + global _LAYER + a = op.args + if a.get("out", "bf16") != "bf16": + raise UnsupportedOpError("vLLM MoE layers return bf16") + if a["tokens"] > _MAX_TOKENS: + raise UnsupportedOpError(f"tokens > {_MAX_TOKENS} is not wired") + routing = a.get("routing") or {"distribution": "natural", "seed": 0} + engine.context(a.get("parallel")) + _release_previous() + import vllm.envs as envs + torch.manual_seed(routing["seed"]) + _LAYER += 1 # vLLM registers layers by name; each op builds a fresh one + prefix = f"model.layers.{_LAYER}.mlp" + with ranks.together("build this layer"): + prev = torch.get_default_dtype() + torch.set_default_dtype(torch.bfloat16) + try: + own = modules.build(a) if os.environ.get("OPERATORX_MODEL_MODULES") == "1" else None + block = _ModelBlock(own) if own is not None else _generic_block(a, prefix) + except ImportError as e: # an image whose vLLM predates the MoE API this block uses + raise UnsupportedOpError(f"this image's vLLM has no {e.name or e}"[:400]) from e + except (NotImplementedError, AssertionError, ValueError, RuntimeError, TypeError, KeyError, + AttributeError) as e: + if _is_fault(e) or (isinstance(e, AttributeError) and not _missing_op(e)): + raise + raise UnsupportedOpError(f"vLLM rejected this MoE layer: {type(e).__name__}: {e}"[:400]) from e + finally: + torch.set_default_dtype(prev) + kernels = _describe(block) + if any("Emulation" in str(v) for e in kernels.values() for v in e.values()): + raise UnsupportedOpError(f"vLLM has only an emulation kernel for this MoE layer here: {kernels}") + # the tokens of one data-parallel group, identical on its tensor-parallel ranks + from vllm.distributed import get_dp_group + g = torch.Generator(device=device()).manual_seed(routing["seed"] * 1000 + get_dp_group().rank_in_group) + x = torch.randn(a["tokens"], a["hidden"], device=device(), dtype=torch.bfloat16, generator=g) + ctx = {"layer": block, "x": x, + "meta": {"vllm_modules": kernels, "fused_shared_experts": block.fused_shared, + "sequence_parallel_moe": block.sequence_parallel, + "model_module": type(block.module).__name__ if isinstance(block, _ModelBlock) else None, + "vllm_env": {k: getattr(envs, k) for k in (*_ENV_KEYS, *_MOE_ENV_KEYS) if hasattr(envs, k)}, + "vllm_engine": engine.meta()}} + with ranks.together("run this layer"): + try: + _kernel_moe(ctx) + sync() + except (NotImplementedError, AssertionError, RuntimeError, ValueError, TypeError, AttributeError) as e: + if _is_fault(e) or (isinstance(e, AttributeError) and not _missing_op(e)): + raise + raise UnsupportedOpError(f"vLLM MoE kernel failed: {type(e).__name__}: {e}"[:400]) from e + return ctx + + +# env that steers vLLM's MoE kernel and stream choice; recorded with every result +_MOE_ENV_KEYS = ("VLLM_ROCM_USE_AITER_MOE", "VLLM_ROCM_USE_AITER_FUSION_SHARED_EXPERTS", + "VLLM_USE_FLASHINFER_MOE_FP8", "VLLM_USE_FLASHINFER_MOE_FP4", "VLLM_FLASHINFER_MOE_BACKEND", + "VLLM_DISABLE_SHARED_EXPERTS_STREAM", "VLLM_SHARED_EXPERTS_STREAM_TOKEN_THRESHOLD") + + +def _kernel_moe(ctx: dict) -> None: + ctx["out"] = ctx["layer"](ctx["x"]) + + +IMPLS = [BackendImpl(op_type="moe", prepare=_prepare_moe, kernel=_kernel_moe, launcher=_launcher)] diff --git a/operatorx/runners/moe.py b/operatorx/runners/moe.py deleted file mode 100644 index 6f99636903..0000000000 --- a/operatorx/runners/moe.py +++ /dev/null @@ -1,89 +0,0 @@ -"""Single-GPU routed expert kernels matching vLLM's generic MoE benchmark shapes. - -Routing is prepared before timing. EP/TP describe a local kernel shape, not a -live distributed group. No router, shared experts or communication is timed. -""" - -from __future__ import annotations - -from importlib import import_module -from typing import Any, Mapping - -import torch - -from operatorx.core import BackendImpl, Op, UnsupportedOpError, lookup_versions - - -def versions() -> dict[str, str]: - return lookup_versions("torch", "vllm", "triton") - - -def local_dimensions(args: Mapping[str, Any]) -> tuple[int, int]: - """Validate a routed-only request and derive the per-rank weight dimensions.""" - for key in ("num_tokens", "hidden", "intermediate", "num_experts", "top_k"): - if type(args[key]) is not int or args[key] <= 0: - raise UnsupportedOpError(f"MoE requires positive integer {key}") - ep = args.get("expert_parallel_size", 1) - tp = args.get("routed_tensor_parallel_size", 1) - if type(ep) is not int or ep <= 0 or type(tp) is not int or tp <= 0: - raise UnsupportedOpError("MoE shard factors must be positive integers") - if args["num_experts"] % ep or args["intermediate"] % tp: - raise UnsupportedOpError("MoE dimensions must divide evenly into EP/TP shards") - experts, intermediate = args["num_experts"] // ep, args["intermediate"] // tp - if args["top_k"] > experts: - raise UnsupportedOpError("MoE top_k exceeds the local expert count") - if args.get("n_shared_experts", 0) != 0: - raise UnsupportedOpError("vllm moe_gemm measures routed experts only") - if args.get("expert_distribution", "uniform") != "uniform": - raise UnsupportedOpError("vllm moe_gemm supports uniform-random routing only") - if ( - args["dtype_act"] not in {"bf16", "fp16"} - or args["dtype_weight"] != args["dtype_act"] - ): - raise UnsupportedOpError( - "vllm moe_gemm requires matching BF16/FP16 activations and weights" - ) - return experts, intermediate - - -def prepare(op: Op) -> dict: - a = op.args - experts, intermediate = local_dimensions(a) - fused_experts = import_module( - "vllm.model_executor.layers.fused_moe.fused_moe" - ).fused_experts - dtype = {"bf16": torch.bfloat16, "fp16": torch.float16}[a["dtype_act"]] - tokens, hidden, top_k = a["num_tokens"], a["hidden"], a["top_k"] - x = torch.empty(tokens, hidden, device="cuda", dtype=dtype).normal_(std=0.1) - w1 = torch.empty( - experts, 2 * intermediate, hidden, device="cuda", dtype=dtype - ).normal_(std=0.02) - w2 = torch.empty(experts, hidden, intermediate, device="cuda", dtype=dtype).normal_( - std=0.02 - ) - logits = torch.randn(tokens, experts, device="cuda", dtype=torch.float32) - weights, ids = torch.topk(torch.softmax(logits, dim=-1), top_k, dim=-1) - weights = (weights / weights.sum(dim=-1, keepdim=True)).contiguous() - return { - "x": x, - "w1": w1, - "w2": w2, - "topk_weights": weights, - "topk_ids": ids.to(torch.int32), - "fused_experts": fused_experts, - } - - -def kernel(ctx: dict) -> None: - # vLLM's default activation is SiLU-and-multiply, as in benchmark_moe.py. - ctx["out"] = ctx["fused_experts"]( - ctx["x"], - ctx["w1"], - ctx["w2"], - ctx["topk_weights"], - ctx["topk_ids"], - inplace=False, - ) - - -IMPLS = [BackendImpl(op_type="moe_gemm", prepare=prepare, kernel=kernel)] diff --git a/operatorx/runners/nvidia/backends/deepep.py b/operatorx/runners/nvidia/backends/deepep.py deleted file mode 100644 index 1d5ffaa1a6..0000000000 --- a/operatorx/runners/nvidia/backends/deepep.py +++ /dev/null @@ -1,293 +0,0 @@ -from __future__ import annotations - -import deep_ep -import torch -import torch.distributed as dist - -from operatorx.core import BackendImpl, Op, lookup_versions - - -def versions() -> dict[str, str]: - return lookup_versions("deep_ep", "torch") - -_DTYPES = {"bf16": torch.bfloat16, "fp16": torch.float16, "fp32": torch.float32} -_BUFFERS: dict[tuple[int, str], deep_ep.Buffer] = {} - - -def _ensure_dist() -> None: - import os - - if not dist.is_initialized(): - local_rank = int(os.environ.get("LOCAL_RANK", "0")) - torch.cuda.set_device(local_rank) - dist.init_process_group("nccl", device_id=torch.device(f"cuda:{local_rank}")) - - -def _get_buffer( - world_size: int, mode: str, num_experts: int, hidden: int, - max_tpr: int | None = None, -) -> deep_ep.Buffer: - key = (world_size, mode, max_tpr) if mode == "low_latency" else (world_size, mode) - if key in _BUFFERS: - return _BUFFERS[key] - deep_ep.Buffer.set_num_sms(24) - needs_rdma = world_size > 8 - if mode == "low_latency": - rdma_hint = deep_ep.Buffer.get_low_latency_rdma_size_hint( - max_tpr, hidden, world_size, num_experts - ) - buf = deep_ep.Buffer( - dist.group.WORLD, - num_nvl_bytes=int(2e9), - num_rdma_bytes=rdma_hint, - low_latency_mode=True, - num_qps_per_rank=num_experts // world_size, - ) - else: - buf = deep_ep.Buffer( - dist.group.WORLD, - num_nvl_bytes=int(2e9), - num_rdma_bytes=int(2e9) if needs_rdma else 0, - low_latency_mode=False, - ) - _BUFFERS[key] = buf - return buf - - -def _make_inputs(op: Op): - from operatorx.core import UnsupportedOpError - a = op.args - if a["dtype"] not in _DTYPES: - raise UnsupportedOpError(f"deepep doesn't support dtype={a['dtype']!r}") - nt, ne, k, h = a["num_tokens"], a["num_experts"], a["top_k"], a["hidden"] - dt = _DTYPES[a["dtype"]] - # adjust experts to be divisible by world_size - ws = a["world_size"] - if ne % ws != 0: - ne = ((ne + ws - 1) // ws) * ws - x = torch.randn(nt, h, dtype=dt, device="cuda") - topk_idx = torch.randint(0, ne, (nt, k), device="cuda", dtype=torch.int64) - topk_weights = torch.randn(nt, k, device="cuda", dtype=torch.float32).softmax(dim=-1) - return x, topk_idx, topk_weights, ne, h - - -def _build_dispatch_call_normal(op: Op): - a = op.args - ws = a["world_size"] - x, topk_idx, topk_weights, ne, h = _make_inputs(op) - buf = _get_buffer(ws, "normal", ne, h) - ntpr, ntprr, ntpe, itir, _ = buf.get_dispatch_layout(topk_idx, ne) - torch.cuda.synchronize() - cfg = deep_ep.Buffer.get_dispatch_config(ws) - pinned = [x, topk_idx, topk_weights, ntpr, ntprr, ntpe, itir] - - def _call(_pin=pinned): - return buf.dispatch( - x, topk_idx=topk_idx, topk_weights=topk_weights, - num_tokens_per_rank=ntpr, num_tokens_per_rdma_rank=ntprr, - is_token_in_rank=itir, num_tokens_per_expert=ntpe, - config=cfg, async_finish=False, - ) - return _call - - -def _build_dispatch_call_ll(op: Op): - a = op.args - ws = a["world_size"] - x, topk_idx, _, ne, h = _make_inputs(op) - max_tpr = max(1, (a["num_tokens"] * a["top_k"] + ws - 1) // ws * 2) - buf = _get_buffer(ws, "low_latency", ne, h, max_tpr=max_tpr) - pinned = [x, topk_idx] - - def _call(_pin=pinned): - return buf.low_latency_dispatch( - x, topk_idx, max_tpr, ne, use_fp8=True, async_finish=False, - ) - return _call - - -def _pick_fastest(builders: dict, label: str): - """Build each candidate, time 3+5 iters, return the closure of the winner. - Builders that throw at construction or call are silently dropped.""" - candidates = {} - for name, build in builders.items(): - try: - c = build() - for _ in range(3): - c() - torch.cuda.synchronize() - start = torch.cuda.Event(enable_timing=True) - end = torch.cuda.Event(enable_timing=True) - start.record() - for _ in range(5): - c() - end.record() - torch.cuda.synchronize() - candidates[name] = (c, start.elapsed_time(end) / 5.0) - except Exception: - continue - if not candidates: - from operatorx.core import UnsupportedOpError - raise UnsupportedOpError(f"deepep {label}: no variant supports this shape") - winner = min(candidates, key=lambda k: candidates[k][1]) - return candidates[winner][0] - - -def _try_ll() -> bool: - # DeepEP's low_latency path requires NVSHMEM + IBGDA. On clusters where - # IBGDA DCT setup fails, nvshmem_init aborts the process — we can't - # catch it from Python, so we gate the LL probe behind an opt-in env var. - import os - return os.environ.get("OPERATORX_DEEPEP_TRY_LL", "0") == "1" - - -def _prepare_dispatch(op: Op) -> dict: - _ensure_dist() - builders = {"normal": lambda: _build_dispatch_call_normal(op)} - if _try_ll(): - builders["low_latency"] = lambda: _build_dispatch_call_ll(op) - call = _pick_fastest(builders, "dispatch") - return {"_call": call} - - -def _kernel_dispatch(ctx: dict) -> None: - ctx["_call"]() - - -def _build_combine_call_normal(op: Op): - a = op.args - ws = a["world_size"] - x, topk_idx, topk_weights, ne, h = _make_inputs(op) - buf = _get_buffer(ws, "normal", ne, h) - ntpr, ntprr, ntpe, itir, _ = buf.get_dispatch_layout(topk_idx, ne) - torch.cuda.synchronize() - d_cfg = deep_ep.Buffer.get_dispatch_config(ws) - c_cfg = deep_ep.Buffer.get_combine_config(ws) - recv_x, _, recv_tw, _, handle, _ = buf.dispatch( - x, topk_idx=topk_idx, topk_weights=topk_weights, - num_tokens_per_rank=ntpr, num_tokens_per_rdma_rank=ntprr, - is_token_in_rank=itir, num_tokens_per_expert=ntpe, - config=d_cfg, async_finish=False, - ) - torch.cuda.synchronize() - expert_out = torch.randn_like(recv_x) - pinned = [x, topk_idx, topk_weights, recv_x, recv_tw, expert_out, handle] - - def _call(_pin=pinned): - return buf.combine( - x=expert_out, handle=handle, topk_weights=recv_tw, - config=c_cfg, async_finish=False, - ) - return _call - - -def _build_combine_call_ll(op: Op): - a = op.args - ws = a["world_size"] - x, topk_idx, topk_weights, ne, h = _make_inputs(op) - max_tpr = max(1, (a["num_tokens"] * a["top_k"] + ws - 1) // ws * 2) - buf = _get_buffer(ws, "low_latency", ne, h, max_tpr=max_tpr) - packed, _expert_count, handle, _, _ = buf.low_latency_dispatch( - x, topk_idx, max_tpr, ne, use_fp8=True, async_finish=False, - ) - torch.cuda.synchronize() - recv_tensor = packed[0] if isinstance(packed, tuple) else packed - expert_out = torch.randn(*recv_tensor.shape, dtype=torch.bfloat16, device="cuda").contiguous() - pinned = [x, topk_idx, topk_weights, expert_out, handle] - - def _call(_pin=pinned): - return buf.low_latency_combine( - x=expert_out, topk_idx=topk_idx, topk_weights=topk_weights, - handle=handle, async_finish=False, - ) - return _call - - -def _prepare_combine(op: Op) -> dict: - _ensure_dist() - builders = {"normal": lambda: _build_combine_call_normal(op)} - if _try_ll(): - builders["low_latency"] = lambda: _build_combine_call_ll(op) - call = _pick_fastest(builders, "combine") - return {"_call": call} - - -def _kernel_combine(ctx: dict) -> None: - ctx["_call"]() - - -def _prepare_moe_forward(op: Op) -> dict: - _ensure_dist() - a = op.args - nt, h, im = a["num_tokens"], a["hidden"], a["intermediate"] - ne, k = a["num_experts"], a["top_k"] - ws = a["world_size"] - from operatorx.core import UnsupportedOpError - if a["dtype_act"] not in _DTYPES: - raise UnsupportedOpError(f"deepep doesn't support dtype_act={a['dtype_act']!r}") - if a["dtype_weight"] not in _DTYPES: - raise UnsupportedOpError(f"deepep doesn't support dtype_weight={a['dtype_weight']!r}") - dt_act = _DTYPES[a["dtype_act"]] - dt_w = _DTYPES[a["dtype_weight"]] - - if ne % ws != 0: - ne = ((ne + ws - 1) // ws) * ws - num_local_experts = ne // ws - - x = torch.randn(nt, h, dtype=dt_act, device="cuda") - gate_w = torch.randn(h, ne, dtype=dt_w, device="cuda") - gate_proj_w = torch.randn(num_local_experts, h, im, dtype=dt_w, device="cuda") - up_w = torch.randn(num_local_experts, h, im, dtype=dt_w, device="cuda") - down_w = torch.randn(num_local_experts, im, h, dtype=dt_w, device="cuda") - - buf = _get_buffer(ws, "normal", ne, h) - cfg_d = deep_ep.Buffer.get_dispatch_config(ws) - cfg_c = deep_ep.Buffer.get_combine_config(ws) - - return { - "x": x, "gate_w": gate_w, - "gate_proj_w": gate_proj_w, "up_w": up_w, "down_w": down_w, - "buffer": buf, "k": k, "ne": ne, - "cfg_d": cfg_d, "cfg_c": cfg_c, - } - - -def _kernel_moe_forward(ctx: dict) -> None: - x = ctx["x"] - logits = x @ ctx["gate_w"] - weights, idx = torch.topk(torch.softmax(logits.float(), dim=-1), ctx["k"]) - # DeepEP combine requires fp32 topk_weights; keep weights as float32. - - ntpr, ntprr, ntpe, itir, _ = ctx["buffer"].get_dispatch_layout(idx, ctx["ne"]) - recv_x, _, recv_w, num_recv_per_expert, handle, _ = ctx["buffer"].dispatch( - x, topk_idx=idx, topk_weights=weights, - num_tokens_per_rank=ntpr, num_tokens_per_rdma_rank=ntprr, - is_token_in_rank=itir, num_tokens_per_expert=ntpe, - config=ctx["cfg_d"], async_finish=False, - ) - - counts = num_recv_per_expert.tolist() if isinstance(num_recv_per_expert, torch.Tensor) else list(num_recv_per_expert) - expert_out = torch.empty_like(recv_x) - offset = 0 - for e, n in enumerate(counts): - if n == 0: - continue - x_e = recv_x[offset:offset + n] - gate_h = x_e @ ctx["gate_proj_w"][e] - up_h = x_e @ ctx["up_w"][e] - h_act = torch.nn.functional.silu(gate_h) * up_h - expert_out[offset:offset + n] = h_act @ ctx["down_w"][e] - offset += n - - ctx["buffer"].combine( - x=expert_out, handle=handle, - topk_weights=recv_w, config=ctx["cfg_c"], - async_finish=False, - ) - - -IMPLS = [ - BackendImpl(op_type="dispatch", prepare=_prepare_dispatch, kernel=_kernel_dispatch), - BackendImpl(op_type="combine", prepare=_prepare_combine, kernel=_kernel_combine), - BackendImpl(op_type="moe_forward", prepare=_prepare_moe_forward, kernel=_kernel_moe_forward), -] diff --git a/operatorx/runners/nvidia/backends/deepgemm.py b/operatorx/runners/nvidia/backends/deepgemm.py deleted file mode 100644 index cee65eb24a..0000000000 --- a/operatorx/runners/nvidia/backends/deepgemm.py +++ /dev/null @@ -1,45 +0,0 @@ -from __future__ import annotations - -import deep_gemm -import torch - -from operatorx.core import BackendImpl, Op, UnsupportedOpError, lookup_versions - - -def versions() -> dict[str, str]: - return lookup_versions("deep_gemm", "torch") - - -def _prepare_gemm(op: Op) -> dict: - a = op.args - if a["dtype_a"] != "fp8" or a["dtype_b"] != "fp8": - raise UnsupportedOpError(f"deepgemm requires fp8 inputs (got {a['dtype_a']}/{a['dtype_b']})") - if a.get("bias"): - raise UnsupportedOpError("deepgemm fp8_gemm_nt does not expose a bias parameter") - if a.get("activation") is not None: - raise UnsupportedOpError( - f"deepgemm fp8_gemm_nt has no fused activation; got activation={a['activation']!r}" - ) - if a["m"] <= 0 or a["n"] <= 0 or a["k"] <= 0: - raise UnsupportedOpError(f"degenerate gemm shape m={a['m']} n={a['n']} k={a['k']}") - # Pad K up to a multiple of 128 (block size). - K_pad = ((a["k"] + 127) // 128) * 128 - M, N, K = a["m"], a["n"], K_pad - A_bf16 = torch.randn(M, K, dtype=torch.bfloat16, device="cuda") - B_bf16 = torch.randn(N, K, dtype=torch.bfloat16, device="cuda") - - A_fp8, A_scales = deep_gemm.per_token_cast_to_fp8(A_bf16, use_ue8m0=True) - B_fp8, B_scales = deep_gemm.per_block_cast_to_fp8(B_bf16, use_ue8m0=True) - - out = torch.empty(M, N, dtype=torch.bfloat16, device="cuda") - return {"lhs": (A_fp8, A_scales), "rhs": (B_fp8, B_scales), "out": out} - - -def _kernel_gemm(ctx: dict) -> None: - # FIXME: hardcoded layout - deep_gemm.fp8_gemm_nt(ctx["lhs"], ctx["rhs"], ctx["out"]) - - -IMPLS = [ - BackendImpl(op_type="gemm", prepare=_prepare_gemm, kernel=_kernel_gemm), -] diff --git a/operatorx/runners/nvidia/backends/flashinfer.py b/operatorx/runners/nvidia/backends/flashinfer.py deleted file mode 100644 index 9db912a1fc..0000000000 --- a/operatorx/runners/nvidia/backends/flashinfer.py +++ /dev/null @@ -1,704 +0,0 @@ -from __future__ import annotations - -import flashinfer -import torch - -from operatorx.core import BackendImpl, Op, UnsupportedOpError, lookup_versions - - -def versions() -> dict[str, str]: - return lookup_versions("flashinfer", "torch") - -_DTYPES = { - "bf16": torch.bfloat16, - "fp16": torch.float16, -} - -# nvfp4 vs mxfp4 differ only in block size: 16 vs 32. mm_fp4 selects via flags. -_FP4_FAMILIES = {"nvfp4": (16, True), "mxfp4": (32, False)} - - -def _prepare_gemm(op: Op) -> dict: - """flashinfer GEMMs by dtype: - - bf16/bf16 -> flashinfer.mm_bf16 (native bias supported) - - fp8/fp8 -> flashinfer.mm_fp8 (no bias param) - - nvfp4/nvfp4 -> flashinfer.mm_fp4 (no bias param) - - mxfp4/mxfp4 -> flashinfer.mm_fp4 (no bias param) - - Mixed-precision (e.g. bf16 act × mxfp4/int4 weight) and activation fusion - are NOT exposed by any flashinfer API at this layer — those land in - UnsupportedOpError so the dashboard reflects real backend coverage. - Quantization happens in prepare (outside the timed window). - """ - a = op.args - if a.get("activation") is not None: - raise UnsupportedOpError( - f"flashinfer gemm: no native activation fusion in mm_bf16/mm_fp8/mm_fp4; " - f"got activation={a['activation']!r}" - ) - if a["dtype_a"] != a["dtype_b"]: - raise UnsupportedOpError( - f"flashinfer gemm: no native mixed-precision GEMM (a={a['dtype_a']} b={a['dtype_b']})" - ) - if a["dtype_out"] not in _DTYPES: - raise UnsupportedOpError( - f"flashinfer gemm out_dtype must be bf16/fp16; got {a['dtype_out']!r}" - ) - out_dt = _DTYPES[a["dtype_out"]] - m, n, k = a["m"], a["n"], a["k"] - has_bias = bool(a.get("bias")) - family = a["dtype_a"] - - if family == "bf16": - A = torch.randn(m, k, dtype=torch.bfloat16, device="cuda") - B = torch.randn(k, n, dtype=torch.bfloat16, device="cuda") - bias = torch.randn(n, dtype=out_dt, device="cuda") if has_bias else None - return {"kind": "bf16", "A": A, "B": B, "bias": bias, "out_dt": out_dt} - - if family == "fp8": - # flashinfer.mm_fp8 is trtllm_low_latency which expects a block-scaled - # 3D weight layout — not a plain MxK @ KxN matmul. flashinfer's only - # plain-FP8 path is gemm_fp8_nt_groupwise which also wants block scales. - # Plain FP8 GEMM is covered by torch._scaled_mm / deepgemm / sglang. - raise UnsupportedOpError( - "flashinfer has no plain FP8 GEMM (mm_fp8 needs block-scaled 3D B); " - "use the torch / deepgemm / sglang backends for FP8 GEMM" - ) - - if family not in _FP4_FAMILIES: - raise UnsupportedOpError( - f"flashinfer gemm: unsupported dtype family {family!r} " - f"(supported: bf16, fp8, nvfp4, mxfp4)" - ) - if has_bias: - raise UnsupportedOpError( - f"flashinfer mm_fp4 does not expose a bias parameter; got bias=True for {family!r}" - ) - block_size, use_nvfp4 = _FP4_FAMILIES[family] - # Pad K and N up to a multiple of block_size; the kernel requires it for - # the block-scaled FP4 layout. - k = ((k + block_size - 1) // block_size) * block_size - n = ((n + block_size - 1) // block_size) * block_size - a_hi = torch.randn(m, k, dtype=torch.bfloat16, device="cuda") - b_hi = torch.randn(n, k, dtype=torch.bfloat16, device="cuda") - g_a = (448 * 6) / a_hi.float().abs().nan_to_num().max() - g_b = (448 * 6) / b_hi.float().abs().nan_to_num().max() - alpha = (1.0 / (g_a * g_b)).reshape(1).contiguous() - if use_nvfp4: - a_q, a_sf = flashinfer.nvfp4_quantize(a_hi, g_a, sf_vec_size=block_size, do_shuffle=False) - b_q, b_sf = flashinfer.nvfp4_quantize(b_hi, g_b, sf_vec_size=block_size, do_shuffle=False) - backend = "cute-dsl" - else: - if not hasattr(flashinfer, "mxfp4_quantize"): - raise UnsupportedOpError("flashinfer has no mxfp4_quantize in this build") - a_q, a_sf = flashinfer.mxfp4_quantize(a_hi) - b_q, b_sf = flashinfer.mxfp4_quantize(b_hi) - backend = "cudnn" - with flashinfer.autotune(tune_mode=True): - flashinfer.mm_fp4( - a_q, b_q.T, a_sf, b_sf.T, alpha=alpha, - block_size=block_size, use_nvfp4=use_nvfp4, - out_dtype=out_dt, backend=backend, - ) - return { - "kind": "fp4", - "a": a_q, "b": b_q.T, "a_sf": a_sf, "b_sf": b_sf.T, - "alpha": alpha, "block_size": block_size, "use_nvfp4": use_nvfp4, - "out_dt": out_dt, "backend": backend, - } - - -def _kernel_gemm(ctx: dict) -> None: - kind = ctx["kind"] - if kind == "bf16": - ctx["out"] = flashinfer.mm_bf16( - ctx["A"], ctx["B"], bias=ctx["bias"], out_dtype=ctx["out_dt"], - ) - else: # fp4 - ctx["out"] = flashinfer.mm_fp4( - ctx["a"], ctx["b"], ctx["a_sf"], ctx["b_sf"], alpha=ctx["alpha"], - block_size=ctx["block_size"], use_nvfp4=ctx["use_nvfp4"], - out_dtype=ctx["out_dt"], backend=ctx["backend"], - ) - - -def _routing_logits(nt: int, ne: int, distribution: str, device: str = "cuda", - dtype: torch.dtype = torch.bfloat16) -> torch.Tensor: - """Generate routing logits matching the requested expert distribution. - - The kernel's downstream topk picks the dominant experts; biasing the logits - here is equivalent to biasing the router's output. - """ - base = torch.randn(nt, ne, dtype=dtype, device=device) - if distribution == "uniform": - return base - if distribution == "zipf": - # Boost the first few experts. log(1/(rank+1)) for s=1. - rank = torch.arange(1, ne + 1, dtype=torch.float32, device=device) - bias = -torch.log(rank).to(dtype) - return base + bias # broadcast over tokens - if distribution == "single_hot": - base[:, 0] += 10.0 - return base - raise UnsupportedOpError( - f"unknown expert_distribution={distribution!r} " - f"(expected 'uniform' | 'zipf' | 'single_hot')" - ) - - -def _prepare_moe_gemm(op: Op) -> dict: - """Per-rank routed-expert GEMM via flashinfer's fp4 MoE kernels. - - Routes by weight dtype: - - mxfp4 weights + bf16 acts -> trtllm_fp4_block_scale_moe (bf16 acts work here) - - nvfp4 weights + bf16/nvfp4 acts -> cute_dsl_fused_moe_nvfp4 - - fp8 weights + fp8 acts -> trtllm_fp8_block_scale_moe (DeepSeek block-FP8) - """ - a = op.args - # trtllm fp4/fp8 block-scale kernels and cute_dsl nvfp4 all use 128-wide - # K-tiles; flashinfer's tactic table has no entry when hidden % 128 != 0 - # (e.g. gptoss h=2880 → NO_VALID_CONFIG on mxfp4, illegal address on nvfp4). - if a["hidden"] % 128 != 0: - raise UnsupportedOpError( - f"flashinfer moe_gemm kernels require hidden % 128 == 0; got hidden={a['hidden']}" - ) - if a["num_tokens"] == 0: - raise UnsupportedOpError("flashinfer moe_gemm requires num_tokens > 0") - if a["dtype_weight"] == "fp8" and a["dtype_act"] == "fp8": - return _prepare_moe_gemm_fp8(op) - if a["dtype_weight"] not in ("nvfp4", "mxfp4"): - raise UnsupportedOpError( - f"flashinfer moe_gemm only implements nvfp4/mxfp4/fp8 weights; " - f"got dtype_weight={a['dtype_weight']!r}" - ) - # cute_dsl path takes pre-quantized nvfp4 acts and we generate them in - # prepare, so dtype_act in {bf16, nvfp4} both route there. trtllm path - # (mxfp4 weights) only accepts bf16 acts. - if a["dtype_weight"] == "mxfp4" and a["dtype_act"] != "bf16": - raise UnsupportedOpError( - f"flashinfer mxfp4 path requires bf16 acts; got dtype_act={a['dtype_act']!r}" - ) - if a["dtype_act"] not in ("bf16", "nvfp4"): - raise UnsupportedOpError( - f"flashinfer moe_gemm only supports bf16/nvfp4 acts; got dtype_act={a['dtype_act']!r}" - ) - nt, h = a["num_tokens"], a["hidden"] - ne, k = a["num_experts"], a["top_k"] - ep = a.get("expert_parallel_size", 1) - routed_tp = a.get("routed_tensor_parallel_size", 1) - shared_tp = a.get("shared_tensor_parallel_size", 1) - n_shared = a.get("n_shared_experts", 0) - # Single-rank kernel: each rank's slice is (intermediate / routed_tp) - # along intermediate, (num_experts / ep) along routed experts, plus - # n_shared always-on shared experts appended as extra groups. - if a["intermediate"] % max(1, routed_tp) != 0: - raise UnsupportedOpError( - f"intermediate={a['intermediate']} not divisible by routed_tp={routed_tp}" - ) - im = a["intermediate"] // max(1, routed_tp) - local_routed = ne // max(1, ep) - # Shared experts run as a separate dense FP4 MLP after the routed kernel - # (matches real production: trtllm routed kernel + dense shared MLP + - # accumulate). Fusing via routing biases interacts badly with trtllm's - # softmax-then-topk and suppresses routed compute. - local_total = local_routed - use_nvfp4 = (a["dtype_weight"] == "nvfp4") - block_size = 16 if use_nvfp4 else 32 - - for name, dim in (("hidden", h), ("intermediate", im)): - if dim % block_size != 0: - raise UnsupportedOpError( - f"flashinfer fp4 MoE needs {name}={dim} divisible by block_size={block_size}" - ) - if local_routed < 1: - raise UnsupportedOpError(f"local_routed={local_routed} < 1 (num_experts={ne} ep={ep})") - - g_w = torch.tensor([1.0], dtype=torch.float32, device="cuda") - if use_nvfp4: - def quant(x): - return flashinfer.nvfp4_quantize(x, g_w, sf_vec_size=block_size, do_shuffle=False) - else: - def quant(x): - return flashinfer.mxfp4_quantize(x) - # gate_up: (local_total, 2*im, h); down: (local_total, h, im). - w1_hi = torch.randn(local_total * 2 * im, h, dtype=torch.bfloat16, device="cuda") - w2_hi = torch.randn(local_total * h, im, dtype=torch.bfloat16, device="cuda") - w1_q, w1_sf = quant(w1_hi) - w2_q, w2_sf = quant(w2_hi) - # trtllm_fp4_block_scale_moe wants scales as float8_e4m3fn. Both mxfp4 and - # nvfp4 quantizers return uint8 — reinterpret-view to fp8 (same bytes). - if not use_nvfp4: - w1_sf = w1_sf.view(torch.float8_e4m3fn) - w2_sf = w2_sf.view(torch.float8_e4m3fn) - # The swizzled 128x4 scale layout pads BOTH dimensions: rows to multiples - # of 128 and columns to multiples of 4. Reshape to 3D using the actual - # column count returned by the quantizer (which may be > h//block_size or - # im//block_size after padding). - w1_q = w1_q.view(local_total, 2 * im, w1_q.size(-1)) - w1_sf = w1_sf.view(local_total, 2 * im, w1_sf.size(-1)) - w2_q = w2_q.view(local_total, h, w2_q.size(-1)) - w2_sf = w2_sf.view(local_total, h, w2_sf.size(-1)) - - distribution = a.get("expert_distribution", "uniform") - - # Shared experts: per-expert FP4 dense MLP weights sized for the shared TP - # slice (intermediate / shared_tp). Run as plain mm_fp4 calls after the - # routed kernel and accumulate. Per-rank: n_shared two-GEMM MLPs. - shared_pack: dict | None = None - if n_shared > 0: - if a["intermediate"] % max(1, shared_tp) != 0: - raise UnsupportedOpError( - f"intermediate={a['intermediate']} not divisible by shared_tp={shared_tp}" - ) - im_s = a["intermediate"] // max(1, shared_tp) - if im_s % block_size != 0: - raise UnsupportedOpError( - f"shared MLP intermediate={im_s} not divisible by block_size={block_size}" - ) - shared_experts = [] - for _ in range(n_shared): - sw1_hi = torch.randn(2 * im_s, h, dtype=torch.bfloat16, device="cuda") - sw2_hi = torch.randn(h, im_s, dtype=torch.bfloat16, device="cuda") - sw1_q, sw1_sf = quant(sw1_hi) - sw2_q, sw2_sf = quant(sw2_hi) - if not use_nvfp4: - sw1_sf = sw1_sf.view(torch.float8_e4m3fn) - sw2_sf = sw2_sf.view(torch.float8_e4m3fn) - shared_experts.append({"w1": sw1_q, "w1_sf": sw1_sf, "w2": sw2_q, "w2_sf": sw2_sf}) - shared_pack = { - "experts": shared_experts, - "use_nvfp4": use_nvfp4, - "block_size": block_size, - "alpha": torch.tensor(1.0, dtype=torch.float32, device="cuda"), - } - - if not use_nvfp4: - hidden_states = torch.randn(nt, h, dtype=torch.bfloat16, device="cuda") - routing_logits = _routing_logits(nt, ne, distribution) - return { - "path": "trtllm", - "routing_logits": routing_logits, "hidden_states": hidden_states, - "gemm1_weights": w1_q, "gemm1_weights_scale": w1_sf, - "gemm2_weights": w2_q, "gemm2_weights_scale": w2_sf, - "num_experts": ne, "top_k": k, "intermediate_size": im, - "local_expert_offset": 0, "local_num_experts": local_total, - "shared": shared_pack, - "num_tokens_for_tune": nt, - } - - x_hi = torch.randn(nt, h, dtype=torch.bfloat16, device="cuda") - # nvfp4_quantize returns scale factors in the swizzled 128x4 layout (rows - # padded to multiples of 128). The kernel expects that native layout — do - # not .view() it back to the dense (nt, k//block_size) shape. - x_q, x_sf = flashinfer.nvfp4_quantize(x_hi, g_w, sf_vec_size=block_size, do_shuffle=False) - router_logits = _routing_logits(nt, ne, distribution) - token_final_scales, token_selected_experts = torch.topk(router_logits.float(), k, dim=-1) - token_selected_experts = token_selected_experts.to(torch.int32) - token_final_scales = torch.softmax(token_final_scales, dim=-1).to(torch.bfloat16) - w1_alpha = torch.ones(local_total, dtype=torch.float32, device="cuda") - w2_alpha = torch.ones(local_total, dtype=torch.float32, device="cuda") - fc2_input_scale = torch.ones(1, dtype=torch.float32, device="cuda") - return { - "path": "cute_dsl", - "x": x_q, "x_sf": x_sf, - "x_hi": x_hi, - "token_selected_experts": token_selected_experts, - "token_final_scales": token_final_scales, - "w1_weight": w1_q, "w1_weight_sf": w1_sf, "w1_alpha": w1_alpha, - "w2_weight": w2_q, "w2_weight_sf": w2_sf, "w2_alpha": w2_alpha, - "fc2_input_scale": fc2_input_scale, - "num_experts": ne, "top_k": k, - "num_local_experts": local_total, "local_expert_offset": 0, - "shared": shared_pack, - } - - -def _prepare_moe_gemm_fp8(op: Op) -> dict: - """fp8/fp8 path via DeepSeek block-FP8 (128x128 weight blocks, per-token-128 act scale). - - Both the routed kernel and the shared MLP are dispatched through a single - `flashinfer.trtllm_fp8_block_scale_moe` call. For the shared expert we pad - `num_experts` to 4 (the kernel routing checks `num_experts % 4 == 0`), - route all tokens to expert 0 with top_k=1, and ignore the other 3 dummy - slots — that gives us the fused gate_up + silu + down kernel in a single - launch instead of chaining gemm_fp8_nt_groupwise → silu → quant → groupwise. - - Returns `_call` (routed) and `_call_shared` (shared, when n_shared>0) closures - that capture all prep temporaries. Without the pinning, intermediate bf16 - tensors (x_bf16 + per-iteration randn fed to per_block_cast_to_fp8) get freed - when this function returns; the kernel then allocates its workspace into - those freed blocks at addresses that fail the kernel's launch-param check - (cudaLaunchKernelEx → "operation not supported on global/shared address - space" at trtllm_fused_moe_dev_kernel.cu:376). - """ - from sglang.srt.layers.quantization.fp8_kernel import per_token_group_quant_fp8 - from sglang.srt.layers.quantization.fp8_utils import per_block_cast_to_fp8 - a = op.args - nt, h = a["num_tokens"], a["hidden"] - intermediate = a["intermediate"] - ne, k = a["num_experts"], a["top_k"] - ep = a.get("expert_parallel_size", 1) - routed_tp = a.get("routed_tensor_parallel_size", 1) - shared_tp = a.get("shared_tensor_parallel_size", 1) - n_shared = a.get("n_shared_experts", 0) - distribution = a.get("expert_distribution", "uniform") - - BLOCK = 128 - if intermediate % max(1, routed_tp) != 0: - raise UnsupportedOpError(f"intermediate={intermediate} not divisible by routed_tp={routed_tp}") - im = intermediate // max(1, routed_tp) - local_routed = ne // max(1, ep) - for name, dim in (("hidden", h), ("intermediate/routed_tp", im)): - if dim % BLOCK != 0: - raise UnsupportedOpError(f"fp8 path requires {name}={dim} divisible by {BLOCK}") - if local_routed < 1: - raise UnsupportedOpError(f"local_routed={local_routed} < 1 (num_experts={ne} ep={ep})") - - pinned: list = [] # keep every intermediate bf16 tensor alive across kernel calls - x_bf16 = torch.randn(nt, h, dtype=torch.bfloat16, device="cuda") - pinned.append(x_bf16) - # per_token_group_quant_fp8 returns (M, K//128); trtllm_fp8_block_scale_moe - # expects (K//128, M) — transpose + .contiguous() to materialize the layout. - x_fp8, x_scale_mk = per_token_group_quant_fp8(x_bf16, BLOCK) - x_scale = x_scale_mk.T.contiguous() - - w1_q = torch.empty(local_routed, 2 * im, h, dtype=torch.float8_e4m3fn, device="cuda") - w1_sf = torch.empty(local_routed, 2 * im // BLOCK, h // BLOCK, dtype=torch.float32, device="cuda") - w2_q = torch.empty(local_routed, h, im, dtype=torch.float8_e4m3fn, device="cuda") - w2_sf = torch.empty(local_routed, h // BLOCK, im // BLOCK, dtype=torch.float32, device="cuda") - for i in range(local_routed): - tmp1 = torch.randn(2 * im, h, dtype=torch.bfloat16, device="cuda") - pinned.append(tmp1) - w1_q[i], w1_sf[i] = per_block_cast_to_fp8(tmp1) - tmp2 = torch.randn(h, im, dtype=torch.bfloat16, device="cuda") - pinned.append(tmp2) - w2_q[i], w2_sf[i] = per_block_cast_to_fp8(tmp2) - - routing_logits = _routing_logits(nt, ne, distribution) - nt_for_tune = 1 - while nt_for_tune < max(1, nt): - nt_for_tune <<= 1 - - # Shared MLP via fused trtllm_fp8_block_scale_moe with num_experts padded to - # 4 (kernel asserts num_experts % 4 == 0), top_k=1, all tokens routed to - # expert 0. The other 3 expert slots are dummy weights that never get touched. - call_shared = None - if n_shared > 0: - if intermediate % max(1, shared_tp) != 0: - raise UnsupportedOpError(f"intermediate={intermediate} not divisible by shared_tp={shared_tp}") - im_s = intermediate // max(1, shared_tp) - if im_s % BLOCK != 0: - raise UnsupportedOpError(f"fp8 shared MLP requires intermediate/shared_tp={im_s} divisible by {BLOCK}") - pad_ne = max(4, ((n_shared + 3) // 4) * 4) - sw1 = torch.empty(pad_ne, 2 * im_s, h, dtype=torch.float8_e4m3fn, device="cuda") - sw1_sf = torch.ones(pad_ne, 2 * im_s // BLOCK, h // BLOCK, dtype=torch.float32, device="cuda") - sw2 = torch.empty(pad_ne, h, im_s, dtype=torch.float8_e4m3fn, device="cuda") - sw2_sf = torch.ones(pad_ne, h // BLOCK, im_s // BLOCK, dtype=torch.float32, device="cuda") - # Routing: large logit on expert 0 forces every token through it - shared_logits = torch.full((nt, pad_ne), -100.0, dtype=torch.bfloat16, device="cuda") - shared_logits[:, 0] = 100.0 - - def _call_shared(_pin=pinned): - return flashinfer.trtllm_fp8_block_scale_moe( - routing_logits=shared_logits, routing_bias=None, - hidden_states=x_fp8, hidden_states_scale=x_scale, - gemm1_weights=sw1, gemm1_weights_scale=sw1_sf, - gemm2_weights=sw2, gemm2_weights_scale=sw2_sf, - num_experts=pad_ne, top_k=1, n_group=None, topk_group=None, - intermediate_size=im_s, - local_expert_offset=0, local_num_experts=pad_ne, - routed_scaling_factor=None, - routing_method_type=1, - tune_max_num_tokens=nt_for_tune, - ) - call_shared = _call_shared - - def _call_routed(): - # `pinned` must be referenced here so the closure cell keeps it alive - # (and with it, x_bf16 + per-iteration randn temps). Without this - # reference the kernel reuses freed allocator blocks and fails. - _ = pinned - return flashinfer.trtllm_fp8_block_scale_moe( - routing_logits=routing_logits, routing_bias=None, - hidden_states=x_fp8, hidden_states_scale=x_scale, - gemm1_weights=w1_q, gemm1_weights_scale=w1_sf, - gemm2_weights=w2_q, gemm2_weights_scale=w2_sf, - num_experts=ne, top_k=k, n_group=None, topk_group=None, - intermediate_size=im, - local_expert_offset=0, local_num_experts=local_routed, - routed_scaling_factor=None, - # routing_method_type=1 (Renormalize) picks a robust tactic across - # all (local_num_experts, num_tokens) we benchmark. The default - # rmt=0 (Default/TopK-Softmax) crashes for ne_local=48 at nt=1 - # (cudaLaunchKernelEx launch reject) and for ne_local∈{256,512} - # at nt=256 (trtllm_batched_gemm_runner runtime error). We're - # benchmarking the routed-GEMM cost, not the routing softmax - # itself, so the choice of routing post-process is incidental. - routing_method_type=1, - tune_max_num_tokens=nt_for_tune, - ) - - return { - "path": "trtllm_fp8", - "_call": _call_routed, - "_call_shared": call_shared, - } - - -def _shared_mlp_fp4(x_bf16: torch.Tensor, shared_pack: dict) -> torch.Tensor: - """Run the shared expert(s) as plain FP4 GEMMs and return the bf16 sum.""" - from flashinfer.gemm import mm_fp4 - block_size = shared_pack["block_size"] - use_nvfp4 = shared_pack["use_nvfp4"] - alpha = shared_pack["alpha"] - g_w = torch.tensor([1.0], dtype=torch.float32, device=x_bf16.device) - - if use_nvfp4: - x_q, x_sf = flashinfer.nvfp4_quantize(x_bf16, g_w, sf_vec_size=block_size, do_shuffle=False) - else: - x_q, x_sf = flashinfer.mxfp4_quantize(x_bf16) - x_sf = x_sf.view(torch.float8_e4m3fn) - - out = None - for expert in shared_pack["experts"]: - gate_up = mm_fp4( - x_q, expert["w1"].T, x_sf, expert["w1_sf"].T, alpha, - out_dtype=torch.bfloat16, block_size=block_size, use_nvfp4=use_nvfp4, - ) - gate, up_proj = gate_up.chunk(2, dim=-1) - mid = torch.nn.functional.silu(gate) * up_proj - if use_nvfp4: - mid_q, mid_sf = flashinfer.nvfp4_quantize(mid, g_w, sf_vec_size=block_size, do_shuffle=False) - else: - mid_q, mid_sf = flashinfer.mxfp4_quantize(mid) - mid_sf = mid_sf.view(torch.float8_e4m3fn) - down = mm_fp4( - mid_q, expert["w2"].T, mid_sf, expert["w2_sf"].T, alpha, - out_dtype=torch.bfloat16, block_size=block_size, use_nvfp4=use_nvfp4, - ) - out = down if out is None else out + down - return out - - -def _kernel_moe_gemm(ctx: dict) -> None: - path = ctx["path"] - # Match sglang's pattern: lock the autotuner's tuning window to this - # shape's M via tune_max_num_tokens=next_power_of_2(num_tokens). Without - # this, the kernel keeps the default tune_max_num_tokens=8192 and the - # autotuner cache is keyed for a different M range than the runtime tactic - # selector picks, producing CUDA illegal memory access for shapes outside - # the default tuning window. - nt_for_tune = 1 - while nt_for_tune < max(1, ctx.get("num_tokens_for_tune", 1)): - nt_for_tune <<= 1 - if path == "trtllm": - out = flashinfer.trtllm_fp4_block_scale_moe( - routing_logits=ctx["routing_logits"], routing_bias=None, - hidden_states=ctx["hidden_states"], hidden_states_scale=None, - gemm1_weights=ctx["gemm1_weights"], gemm1_weights_scale=ctx["gemm1_weights_scale"], - gemm1_bias=None, gemm1_alpha=None, gemm1_beta=None, gemm1_clamp_limit=None, - gemm2_weights=ctx["gemm2_weights"], gemm2_weights_scale=ctx["gemm2_weights_scale"], - gemm2_bias=None, - output1_scale_scalar=None, output1_scale_gate_scalar=None, output2_scale_scalar=None, - num_experts=ctx["num_experts"], top_k=ctx["top_k"], - n_group=None, topk_group=None, - intermediate_size=ctx["intermediate_size"], - local_expert_offset=ctx["local_expert_offset"], local_num_experts=ctx["local_num_experts"], - routed_scaling_factor=None, - tune_max_num_tokens=nt_for_tune, - ) - elif path == "trtllm_fp8": - # Closure call: keeps prep-time temporaries alive across the kernel - # launch so the workspace allocator doesn't reuse freed blocks. - out = ctx["_call"]() - else: # cute_dsl - out = flashinfer.cute_dsl_fused_moe_nvfp4( - x=ctx["x"], x_sf=ctx["x_sf"], - token_selected_experts=ctx["token_selected_experts"], - token_final_scales=ctx["token_final_scales"], - w1_weight=ctx["w1_weight"], w1_weight_sf=ctx["w1_weight_sf"], w1_alpha=ctx["w1_alpha"], - fc2_input_scale=ctx["fc2_input_scale"], - w2_weight=ctx["w2_weight"], w2_weight_sf=ctx["w2_weight_sf"], w2_alpha=ctx["w2_alpha"], - num_experts=ctx["num_experts"], top_k=ctx["top_k"], - num_local_experts=ctx["num_local_experts"], local_expert_offset=ctx["local_expert_offset"], - ) - out = out[0] if isinstance(out, (list, tuple)) else out - if path == "trtllm_fp8": - cs = ctx.get("_call_shared") - if cs is not None: - shared_out = cs() - shared_out = shared_out[0] if isinstance(shared_out, (list, tuple)) else shared_out - out = out + shared_out - elif ctx.get("shared") is not None: - x_for_shared = ctx["hidden_states"] if path == "trtllm" else ctx["x_hi"] - out = out + _shared_mlp_fp4(x_for_shared, ctx["shared"]) - ctx["out"] = out - - -def _prepare_attention_mha(op: Op) -> dict: - a = op.args - if a["batch_size"] != 1: - raise NotImplementedError("flashinfer attention_mha currently supports batch_size=1") - if a.get("kv_layout", "contig") != "contig": - raise NotImplementedError("flashinfer attention_mha here only handles contiguous KV") - - S_q, S_kv = a["seq_len_q"], a["seq_len_kv"] - H, H_kv, D = a["num_heads"], a["num_heads_kv"], a["head_dim"] - if a["dtype_q"] not in _DTYPES: - raise UnsupportedOpError(f"flashinfer doesn't support dtype_q={a['dtype_q']!r}") - dt = _DTYPES[a["dtype_q"]] - causal = a.get("causal", True) - - q = torch.randn(S_q, H, D, dtype=dt, device="cuda") - k = torch.randn(S_kv, H_kv, D, dtype=dt, device="cuda") - v = torch.randn(S_kv, H_kv, D, dtype=dt, device="cuda") - - if S_q == 1: - def fn(q_=q, k_=k, v_=v): - return flashinfer.single_decode_with_kv_cache(q_[0], k_, v_) - else: - def fn(q_=q, k_=k, v_=v, causal_=causal): - return flashinfer.single_prefill_with_kv_cache(q_, k_, v_, causal=causal_) - - return {"fn": fn} - - -def _kernel_attention_mha(ctx: dict) -> None: - ctx["out"] = ctx["fn"]() - - -# ---------- attention_mla ---------- -# -# flashinfer ships two distinct MLA kernels (per the project's tracing issue -# https://github.com/flashinfer-ai/flashinfer/issues/792): -# -# 1. Decode -> BatchMLAPagedAttentionWrapper (matrix-absorbed, paged KV) -# q_nope lives in kv_lora_rank space, ckv/kpe are the compressed cache. -# Loads O(S_kv * (R + D_pe)) bytes — dominates over S_kv. -# -# 2. Prefill -> BatchPrefillWithRaggedKVCacheWrapper (no absorption, ragged) -# Plain self-attention with head_dim_qk = nope+rope = 192, -# head_dim_vo = D_v = 128. Loads O(S_kv * H * D) bytes but the inner -# attention dim stays small (192 vs the absorbed 512), so prefill is -# faster here than through the absorbed wrapper. -# -# We dispatch on S_q == 1: pure decode goes via path 1, anything else uses -# path 2. (Chunked prefill / incremental prefill that mixes both kernels is -# out of scope — neither shape in our testlist hits that regime.) -# -# Either path: workspace 128 MiB; only wrapper.run() is timed. Up-projection -# gemms (c_kv -> K_nope/V) are scored separately via the gemm op. - -_MLA_WORKSPACE_BYTES = 128 * 1024 * 1024 # 128 MiB, recommended by flashinfer docs - - -def _prepare_mla_decode(a: dict, dt: torch.dtype) -> dict: - B = a["batch_size"] - S_q = a["seq_len_q"] - S_kv = a["seq_len_kv"] - H = a["num_heads"] - R = a["kv_lora_rank"] - D_pe = a["head_dim_qk_rope"] - D_nope = a["head_dim_qk_nope"] - causal = a.get("causal", True) - page_size = 1 - - q_indptr = torch.arange(0, B + 1, dtype=torch.int32, device="cuda") * S_q - kv_indptr = torch.arange(0, B + 1, dtype=torch.int32, device="cuda") * S_kv - kv_indices = torch.arange(0, B * S_kv, dtype=torch.int32, device="cuda") - kv_lens = torch.full((B,), S_kv, dtype=torch.int32, device="cuda") - - q_nope = torch.randn(B * S_q, H, R, dtype=dt, device="cuda") - q_pe = torch.randn(B * S_q, H, D_pe, dtype=dt, device="cuda") - ckv = torch.randn(B * S_kv, page_size, R, dtype=dt, device="cuda") - kpe = torch.randn(B * S_kv, page_size, D_pe, dtype=dt, device="cuda") - - workspace = torch.empty(_MLA_WORKSPACE_BYTES, dtype=torch.int8, device="cuda") - wrapper = flashinfer.mla.BatchMLAPagedAttentionWrapper(workspace, backend="auto") - - # Per flashinfer docs: "use head dimension before matrix absorption" for sm_scale. - sm_scale = 1.0 / ((D_nope + D_pe) ** 0.5) - wrapper.plan( - q_indptr, kv_indptr, kv_indices, kv_lens, - H, R, D_pe, page_size, - causal, sm_scale, - dt, dt, - ) - - def _run(): - return wrapper.run(q_nope, q_pe, ckv, kpe, return_lse=False) - - return {"fn": _run} - - -def _prepare_mla_prefill(a: dict, dt: torch.dtype) -> dict: - B = a["batch_size"] - S_q = a["seq_len_q"] - S_kv = a["seq_len_kv"] - H = a["num_heads"] - qk_dim = a["head_dim_qk_nope"] + a["head_dim_qk_rope"] - D_v = a["head_dim_v"] - causal = a.get("causal", True) - - qo_indptr = torch.arange(0, B + 1, dtype=torch.int32, device="cuda") * S_q - kv_indptr = torch.arange(0, B + 1, dtype=torch.int32, device="cuda") * S_kv - - # Ragged layout NHD: [nnz, num_heads, head_dim]. MLA has num_kv_heads == num_heads. - q = torch.randn(B * S_q, H, qk_dim, dtype=dt, device="cuda") - k = torch.randn(B * S_kv, H, qk_dim, dtype=dt, device="cuda") - v = torch.randn(B * S_kv, H, D_v, dtype=dt, device="cuda") - - workspace = torch.empty(_MLA_WORKSPACE_BYTES, dtype=torch.int8, device="cuda") - wrapper = flashinfer.BatchPrefillWithRaggedKVCacheWrapper(workspace, kv_layout="NHD") - - sm_scale = 1.0 / (qk_dim ** 0.5) - wrapper.plan( - qo_indptr, kv_indptr, - num_qo_heads=H, num_kv_heads=H, - head_dim_qk=qk_dim, head_dim_vo=D_v, - causal=causal, sm_scale=sm_scale, - q_data_type=dt, kv_data_type=dt, - ) - - def _run(): - return wrapper.run(q, k, v) - - return {"fn": _run} - - -def _prepare_attention_mla(op: Op) -> dict: - a = op.args - if a["dtype_q"] not in _DTYPES: - raise UnsupportedOpError( - f"flashinfer MLA dtype_q={a['dtype_q']!r} not supported (need bf16/fp16)" - ) - if a["dtype_kv"] not in _DTYPES: - raise UnsupportedOpError( - f"flashinfer MLA dtype_kv={a['dtype_kv']!r} not supported (need bf16/fp16); " - "neither MLA paged wrapper nor ragged prefill wrapper has a native fp8 path" - ) - if a["dtype_q"] != a["dtype_kv"]: - raise UnsupportedOpError( - f"flashinfer MLA needs dtype_q == dtype_kv " - f"(got {a['dtype_q']} vs {a['dtype_kv']})" - ) - - dt = _DTYPES[a["dtype_q"]] - # Pure decode (S_q == 1) goes through the matrix-absorbed paged MLA wrapper. - # Anything with S_q > 1 (self-attention prefill) goes through the ragged - # prefill wrapper with head_dim_qk = nope+rope, head_dim_vo = D_v. - if a["seq_len_q"] == 1: - return _prepare_mla_decode(a, dt) - return _prepare_mla_prefill(a, dt) - - -def _kernel_attention_mla(ctx: dict) -> None: - ctx["out"] = ctx["fn"]() - - -IMPLS = [ - BackendImpl(op_type="gemm", prepare=_prepare_gemm, kernel=_kernel_gemm), - BackendImpl(op_type="moe_gemm", prepare=_prepare_moe_gemm, kernel=_kernel_moe_gemm), - BackendImpl(op_type="attention_mha", prepare=_prepare_attention_mha, kernel=_kernel_attention_mha), - BackendImpl(op_type="attention_mla", prepare=_prepare_attention_mla, kernel=_kernel_attention_mla), -] diff --git a/operatorx/runners/nvidia/backends/flashinfer_comm.py b/operatorx/runners/nvidia/backends/flashinfer_comm.py deleted file mode 100644 index 189faf370d..0000000000 --- a/operatorx/runners/nvidia/backends/flashinfer_comm.py +++ /dev/null @@ -1,104 +0,0 @@ -"""flashinfer collective backend: TRT-LLM custom_all_reduce via -flashinfer.comm.allreduce_fusion with backend='auto' (AUTO internally picks -between Lamport one-shot and two-shot ring per shape). - -We cache the IPC workspace per (tokens, hidden, dtype) — workspace creation -does a cross-rank IPC handle exchange that costs tens of seconds, and the -workspace itself owns shared-memory buffers that we don't want to multiply. -""" -from __future__ import annotations - -import os - -import torch -import torch.distributed as dist - -from operatorx.core import BackendImpl, Op, UnsupportedOpError, lookup_versions - - -def versions() -> dict[str, str]: - return lookup_versions("flashinfer", "torch") - - -_DTYPES = {"bf16": torch.bfloat16, "fp16": torch.float16, "fp32": torch.float32} - - -def _ensure_dist() -> None: - if not dist.is_initialized(): - local_rank = int(os.environ.get("LOCAL_RANK", "0")) - torch.cuda.set_device(local_rank) - dist.init_process_group("nccl", device_id=torch.device(f"cuda:{local_rank}")) - - -# Cache workspaces by (tokens, hidden, dtype) — IPC setup is expensive. -_WS: dict[tuple, object] = {} - - -def _factor(n: int, max_hidden: int = 8192, align: int = 128) -> tuple[int, int]: - """Factor n into (tokens, hidden) with hidden ≤ max_hidden and hidden % align == 0.""" - if n % align != 0: - raise UnsupportedOpError( - f"flashinfer_comm allreduce: num_elements={n} must be a multiple of {align}" - ) - if n <= max_hidden: - return 1, n - hidden = max_hidden - while n % hidden != 0 and hidden > align: - hidden -= align - if n % hidden != 0: - raise UnsupportedOpError( - f"flashinfer_comm allreduce: cannot factor n={n} with hidden ≤ {max_hidden}" - ) - return n // hidden, hidden - - -def _get_workspace(tokens: int, hidden: int, dt: torch.dtype, ws: int, rank: int): - key = (tokens, hidden, dt, ws) - if key in _WS: - return _WS[key] - from flashinfer.comm import create_allreduce_fusion_workspace - # backend="trtllm" avoids the "auto" probe path that lazy-imports mpi4py - # (which we don't have); trtllm IPC works on standard NVLink/NVSwitch. - workspace = create_allreduce_fusion_workspace( - backend="trtllm", world_size=ws, rank=rank, - max_token_num=tokens, hidden_dim=hidden, dtype=dt, - ) - _WS[key] = workspace - return workspace - - -def _prepare_allreduce(op: Op) -> dict: - a = op.args - if a["dtype"] not in _DTYPES: - raise UnsupportedOpError(f"flashinfer_comm allreduce: dtype={a['dtype']!r} not supported") - _ensure_dist() - dt = _DTYPES[a["dtype"]] - ws = int(a["world_size"]) - rank = dist.get_rank() - n = int(a["num_elements"]) - tokens, hidden = _factor(n) - - from flashinfer.comm import AllReduceFusionPattern, allreduce_fusion - - workspace = _get_workspace(tokens, hidden, dt, ws, rank) - x = torch.randn(tokens, hidden, dtype=dt, device="cuda") - y = torch.empty_like(x) - - def _call(_pin=(x, y, workspace)): - return allreduce_fusion( - input=x, workspace=workspace, - pattern=AllReduceFusionPattern.kAllReduce, - output=y, use_oneshot=None, # let AUTO pick oneshot/twoshot - launch_with_pdl=False, - ) - - return {"_call": _call} - - -def _kernel_allreduce(ctx: dict) -> None: - ctx["_call"]() - - -IMPLS = [ - BackendImpl(op_type="allreduce", prepare=_prepare_allreduce, kernel=_kernel_allreduce), -] diff --git a/operatorx/runners/nvidia/backends/sglang.py b/operatorx/runners/nvidia/backends/sglang.py deleted file mode 100644 index ff9733bab2..0000000000 --- a/operatorx/runners/nvidia/backends/sglang.py +++ /dev/null @@ -1,333 +0,0 @@ -from __future__ import annotations - -import os -from types import SimpleNamespace - -import torch -import torch.distributed as dist - -from operatorx.core import BackendImpl, Op, lookup_versions - - -def versions() -> dict[str, str]: - return lookup_versions("sglang", "torch") - -_DTYPES = {"bf16": torch.bfloat16, "fp16": torch.float16, "fp32": torch.float32} -_MOE_CACHE: dict[tuple, object] = {} - - -def _ensure_dist() -> None: - if not dist.is_initialized(): - torch.cuda.set_device(int(os.environ.get("LOCAL_RANK", "0"))) - dist.init_process_group("nccl") - - -_SGLANG_READY = False -_MP_STATE: dict[str, int | None] = {"tp": None, "ep": None} - - -def _ensure_sglang(tp: int, ep: int) -> None: - # sglang's parallel-group state is process-global and can't be safely - # re-initialized while cached MoE modules still reference the old groups. - # First (tp, ep) wins for this process; mismatched later shapes are rejected. - global _SGLANG_READY - if _SGLANG_READY: - if _MP_STATE["tp"] != tp or _MP_STATE["ep"] != ep: - from operatorx.core import UnsupportedOpError - raise UnsupportedOpError( - f"sglang already initialised with tp={_MP_STATE['tp']} ep={_MP_STATE['ep']}; " - f"this op needs tp={tp} ep={ep}" - ) - return - _ensure_dist() - - from sglang.srt.distributed.parallel_state import ( - init_distributed_environment, - initialize_model_parallel, - model_parallel_is_initialized, - ) - import sglang.srt.layers.moe.utils as moe_utils - import sglang.srt.layers.dp_attention as dp_attn - import sglang.srt.server_args as sa_module - - rank = dist.get_rank() - world_size = dist.get_world_size() - local_rank = int(os.environ.get("LOCAL_RANK", "0")) - - if not model_parallel_is_initialized(): - init_distributed_environment( - backend="nccl", - world_size=world_size, rank=rank, local_rank=local_rank, - distributed_init_method="env://", - ) - initialize_model_parallel( - tensor_model_parallel_size=tp, - pipeline_model_parallel_size=1, - expert_model_parallel_size=ep, - ) - - class _FakeServerArgs: - disable_shared_experts_fusion = True - ep_num_redundant_experts = 0 - enable_eplb = False - enable_deterministic_inference = False - enable_symm_mem = False - kt_weight_path = None - kt_num_gpu_experts = None - kt_cpuinfer = None - kt_threadpool_count = None - dp_size = 1 - chunked_prefill_size = 8192 - - def __getattr__(self, name): - if name.startswith("enable_") or name.startswith("disable_"): - return False - if name.endswith("_size"): - return 1 - return None - - sa_module._global_server_args = _FakeServerArgs() - moe_utils.MOE_A2A_BACKEND = moe_utils.MoeA2ABackend.NONE - dp_attn._DP_ATTENTION_ENABLED = False - - _MP_STATE["tp"] = tp - _MP_STATE["ep"] = ep - _SGLANG_READY = True - - -def _prepare_gemm(op: Op) -> dict: - from operatorx.core import UnsupportedOpError - a = op.args - if a["dtype_a"] != "fp8" or a["dtype_b"] != "fp8": - raise UnsupportedOpError(f"sglang requires fp8 inputs (got {a['dtype_a']}/{a['dtype_b']})") - if a.get("bias"): - raise UnsupportedOpError("sglang w8a8_block_fp8_matmul does not expose a bias parameter") - if a.get("activation") is not None: - raise UnsupportedOpError( - f"sglang w8a8_block_fp8_matmul has no fused activation; got activation={a['activation']!r}" - ) - if a["m"] <= 0 or a["n"] <= 0 or a["k"] <= 0: - raise UnsupportedOpError(f"degenerate gemm shape m={a['m']} n={a['n']} k={a['k']}") - from sglang.srt.layers.quantization.fp8_kernel import ( - per_token_group_quant_fp8, - w8a8_block_fp8_matmul, - ) - from sglang.srt.layers.quantization.fp8_utils import per_block_cast_to_fp8 - - # Pad K up to a multiple of 128 (block size); the kernel's grid wants this. - K_pad = ((a["k"] + 127) // 128) * 128 - M, N, K = a["m"], a["n"], K_pad - A_bf16 = torch.randn(M, K, dtype=torch.bfloat16, device="cuda") - B_bf16 = torch.randn(N, K, dtype=torch.bfloat16, device="cuda") - block_size = [128, 128] - A_fp8, A_scales = per_token_group_quant_fp8(A_bf16, block_size[1]) - B_fp8, B_scales = per_block_cast_to_fp8(B_bf16) - return { - "matmul_fn": w8a8_block_fp8_matmul, - "a": A_fp8, "b": B_fp8, - "a_scales": A_scales, "b_scales": B_scales, - "block_size": block_size, - } - - -def _kernel_gemm(ctx: dict) -> None: - ctx["out"] = ctx["matmul_fn"]( - ctx["a"], ctx["b"], ctx["a_scales"], ctx["b_scales"], - ctx["block_size"], torch.bfloat16, - ) - - -def _build_moe( - num_experts: int, hidden: int, intermediate: int, top_k: int, - dtype: torch.dtype, weight_dtype: str, - expert_distribution: str = "uniform", - n_shared_experts: int = 1, -): - """Mirror master's bench_deepseek_moe_standalone.create_moe_layer. - - ``weight_dtype`` selects the expert weight precision: - - 'fp8' -> Fp8Config (DeepSeek block-FP8) - - 'nvfp4' -> ModelOptFp4Config (NVFP4). Routes through MOE_RUNNER_BACKEND - (scoped to FLASHINFER_CUTEDSL during construction). - - 'bf16'/'fp16' -> unquantized - - ``intermediate`` is the *global* per-expert intermediate dim. sglang's - FusedMoE and DeepseekV2MLP shard it internally based on the TP/EP groups - set up by ``initialize_model_parallel`` — don't pre-divide here. - """ - from sglang.srt.models.deepseek_v2 import DeepseekV2MoE - - if weight_dtype == "fp8": - quant_config_dict = {"quant_method": "fp8", "weight_block_size": [128, 128]} - elif weight_dtype == "nvfp4": - quant_config_dict = {"quant_method": "modelopt_fp4"} - else: - quant_config_dict = None - - config = SimpleNamespace( - n_routed_experts=num_experts, - num_experts_per_tok=top_k, - n_group=8, - topk_group=4, - topk_method="noaux_tc", - norm_topk_prob=True, - routed_scaling_factor=2.5, - hidden_size=hidden, - moe_intermediate_size=intermediate, - hidden_act="silu", - n_shared_experts=n_shared_experts, - first_k_dense_replace=3, - num_hidden_layers=61, - quantization_config=quant_config_dict, - ) - - if weight_dtype == "fp8": - from sglang.srt.layers.quantization.fp8 import Fp8Config - quant_config = Fp8Config( - is_checkpoint_fp8_serialized=True, - weight_block_size=[128, 128], - ) - elif weight_dtype == "nvfp4": - from sglang.srt.layers.quantization.modelopt_quant import ModelOptFp4Config - # group_size=16 (NVFP4 block size). The router gate and shared-experts - # are bf16 Linear layers; exclude them so apply() doesn't look for - # checkpoint-only `input_scale_inv` attrs we don't generate. - quant_config = ModelOptFp4Config( - is_checkpoint_nvfp4_serialized=True, group_size=16, - exclude_modules=["gate", "shared_experts", "lm_head", "embed"], - packed_modules_mapping={}, - ) - else: - quant_config = None - - torch.set_default_dtype(dtype) - try: - moe = DeepseekV2MoE( - config=config, layer_id=3, quant_config=quant_config, - prefix="model.layers.3.mlp", alt_stream=None, is_nextn=False, - ) - finally: - torch.set_default_dtype(torch.float32) - - moe = moe.to(device="cuda") - with torch.no_grad(): - for name, param in moe.named_parameters(): - if param.dtype == torch.uint8: - # nvfp4-packed weight tensors; any byte is a valid fp4 pair. - param.random_(0, 256) - elif param.dtype == torch.float8_e4m3fn: - # block scales — positive fp8 values around 1.0 via bf16. - param.copy_(torch.rand_like(param, dtype=dtype).add_(0.5).to(param.dtype)) - elif param.dtype.is_floating_point: - param.normal_(0, 0.02) - if "correction_bias" in name: - param.abs_() - if "scale" in name and param.dtype == torch.float32: - param.fill_(1.0) - - # Bias the router's correction_bias to shape the expert distribution - # observed by the kernel. We patch correction_bias (added directly to - # the routing scores) rather than the gate's weight matrix so the bias - # is independent of the input activations. - if expert_distribution != "uniform": - for name, param in moe.named_parameters(): - if "correction_bias" in name and param.shape[-1] == num_experts: - if expert_distribution == "zipf": - rank = torch.arange(1, num_experts + 1, - dtype=torch.float32, device=param.device) - bias = -torch.log(rank).to(param.dtype) - elif expert_distribution == "single_hot": - bias = torch.zeros(num_experts, dtype=param.dtype, device=param.device) - bias[0] = 10.0 - else: - raise ValueError(f"unknown expert_distribution={expert_distribution!r}") - param.copy_(bias) - - # NVFP4 needs `process_weights_after_loading` to consolidate per-expert - # scales into the `*_scale_quant` tensors the kernel reads. The relevant - # `enable_flashinfer_*_moe` flags are @property methods that read the - # global MOE_RUNNER_BACKEND at call time, so the override must already be - # active when this is called — _prepare_moe_forward sets it before invoking. - if weight_dtype == "nvfp4": - from sglang.srt.layers.moe.fused_moe_triton import FusedMoE - for sub in moe.modules(): - if isinstance(sub, FusedMoE) and hasattr(sub, "quant_method"): - sub.quant_method.process_weights_after_loading(sub) - return moe - - -def _prepare_moe_forward(op: Op) -> dict: - from operatorx.core import UnsupportedOpError - a = op.args - if a["dtype_weight"] not in ("fp8", "nvfp4", "bf16", "fp16"): - raise UnsupportedOpError( - f"sglang moe_forward expects fp8/nvfp4/bf16/fp16 weights (got dtype_weight={a['dtype_weight']!r})" - ) - if a["dtype_act"] not in _DTYPES: - raise UnsupportedOpError(f"sglang moe_forward doesn't support dtype_act={a['dtype_act']!r}") - ne, h, im = a["num_experts"], a["hidden"], a["intermediate"] - k, ws = a["top_k"], a["world_size"] - ep = a.get("expert_parallel_size", 1) - tp = a.get("routed_tensor_parallel_size", 1) - s_tp = a.get("shared_tensor_parallel_size", 1) - if ep * tp != ws: - raise UnsupportedOpError( - f"expert_parallel_size * routed_tensor_parallel_size must equal world_size " - f"(got {ep} * {tp} != {ws})" - ) - if s_tp not in (1, tp): - raise UnsupportedOpError( - f"shared_tensor_parallel_size must be 1 or routed_tensor_parallel_size " - f"(got {s_tp}, routed_tp={tp})" - ) - # SGLANG_SHARED_EXPERT_TP1=1 forces DeepseekV2MoE to build the shared expert - # with tp_size=1 (replicated within the TP group). Must be set before - # construction since it's read at __init__ time. - if s_tp == 1 and tp > 1: - os.environ["SGLANG_SHARED_EXPERT_TP1"] = "1" - else: - os.environ.pop("SGLANG_SHARED_EXPERT_TP1", None) - _ensure_sglang(tp=tp, ep=ep) - # NVFP4: pin MOE_RUNNER_BACKEND to a flashinfer variant for the lifetime of - # this op. `enable_flashinfer_*_moe` are @property methods that re-read the - # global at every call, so we need this set during build, weight-process, - # and the timed kernel iterations. AUTO has no nvfp4 dispatch branch. - if a["dtype_weight"] == "nvfp4": - import sglang.srt.layers.moe.utils as _moe_utils - runner_name = os.environ.get("OPERATORX_SGLANG_MOE_RUNNER", "FLASHINFER_CUTLASS") - _moe_utils.MOE_RUNNER_BACKEND = _moe_utils.MoeRunnerBackend[runner_name] - # sglang's fp8 MoE config (weight_block_size=[128,128]) requires the per-rank - # expert count to be a multiple of block_n=128, else weight loading aborts. - if a["dtype_weight"] == "fp8": - experts_per_rank = ne // max(1, ep) - if experts_per_rank % 128 != 0: - raise UnsupportedOpError( - f"sglang fp8 MoE requires num_experts/ep to be a multiple of 128 " - f"(weight block_n); got num_experts={ne} ep={ep} -> {experts_per_rank}" - ) - dt = _DTYPES[a["dtype_act"]] - nt = a["num_tokens"] - - distribution = a.get("expert_distribution", "uniform") - n_shared = a.get("n_shared_experts", 0) - cache_key = (ne, h, im, k, ws, tp, ep, s_tp, dt, a["dtype_weight"], distribution, n_shared) - if cache_key not in _MOE_CACHE: - _MOE_CACHE[cache_key] = _build_moe( - ne, h, im, k, dt, a["dtype_weight"], - expert_distribution=distribution, n_shared_experts=n_shared, - ) - moe = _MOE_CACHE[cache_key] - - x = torch.randn(nt, h, dtype=dt, device="cuda") - return {"moe": moe, "x": x} - - -def _kernel_moe_forward(ctx: dict) -> None: - with torch.no_grad(): - ctx["out"] = ctx["moe"].forward_normal(ctx["x"]) - - -IMPLS = [ - BackendImpl(op_type="gemm", prepare=_prepare_gemm, kernel=_kernel_gemm), - BackendImpl(op_type="moe_forward", prepare=_prepare_moe_forward, kernel=_kernel_moe_forward), -] diff --git a/operatorx/runners/nvidia/backends/sglang_comm.py b/operatorx/runners/nvidia/backends/sglang_comm.py deleted file mode 100644 index 674baeb6d8..0000000000 --- a/operatorx/runners/nvidia/backends/sglang_comm.py +++ /dev/null @@ -1,232 +0,0 @@ -"""sglang collective backend: tries vLLM CustomAllreduce, MSCCL++, and torch -symmetric memory for allreduce, then picks the fastest per shape. - -For non-AR collectives sglang doesn't ship its own kernels — it falls back to -torch.distributed/NCCL, which is already covered by the `torch` backend. -""" -from __future__ import annotations - -import os - -import torch -import torch.distributed as dist - -from operatorx.core import BackendImpl, Op, UnsupportedOpError, lookup_versions - - -def versions() -> dict[str, str]: - return lookup_versions("sglang", "torch") - - -_DTYPES = {"bf16": torch.bfloat16, "fp16": torch.float16, "fp32": torch.float32} - - -def _ensure_dist() -> None: - if not dist.is_initialized(): - local_rank = int(os.environ.get("LOCAL_RANK", "0")) - torch.cuda.set_device(local_rank) - dist.init_process_group("nccl", device_id=torch.device(f"cuda:{local_rank}")) - - -# Cache communicator instances per process — each holds IPC handles and pinned -# scratch buffers; constructing them is expensive (registers buffers in NCCL). -_COMMS: dict[str, object] = {} - - -def _get_communicators() -> dict[str, object]: - """Construct one of each available all-reduce communicator. Returns whatever - succeeded; failures (e.g. missing peer access, unsupported topology) are - silently dropped so we benchmark only what's actually usable.""" - if _COMMS: - return _COMMS - group = dist.group.WORLD - device = torch.cuda.current_device() - - try: - from sglang.srt.distributed.device_communicators.custom_all_reduce import ( - CustomAllreduce, - ) - # 256 MB cap covers the largest testlist shapes (64M elems × 4 bytes). - _COMMS["custom_ar"] = CustomAllreduce(group, device, max_size=256 * 1024 * 1024) - except Exception: - pass - - try: - from sglang.srt.distributed.device_communicators.pymscclpp import ( - PyMscclppCommunicator, - ) - _COMMS["mscclpp"] = PyMscclppCommunicator(group, device, max_bytes=256 * 1024 * 1024) - except Exception: - pass - - try: - from sglang.srt.distributed.device_communicators.torch_symm_mem import ( - TorchSymmMemCommunicator, - ) - _COMMS["symm_mem"] = TorchSymmMemCommunicator(group, device) - except Exception: - pass - - return _COMMS - - -def _supports(name: str, comm, t: torch.Tensor) -> bool: - """Each communicator has a size/topology gate; respect it instead of - blindly invoking (which would return None or fall back to NCCL).""" - if name == "custom_ar": - return bool(comm.should_custom_ar(t)) - if name == "mscclpp": - return bool(comm.should_mscclpp_allreduce(t)) - if name == "symm_mem": - return bool(comm.should_torch_symm_mem_allreduce(t)) - return False - - -def _allreduce_with(name: str, comm, t: torch.Tensor) -> torch.Tensor: - if name == "custom_ar": - return comm.custom_all_reduce(t) - if name == "mscclpp": - return comm.all_reduce(t) - if name == "symm_mem": - return comm.all_reduce(t) - raise UnsupportedOpError(f"unknown comm={name!r}") - - -def _prepare_allreduce(op: Op) -> dict: - a = op.args - if a["dtype"] not in _DTYPES: - raise UnsupportedOpError(f"sglang_comm allreduce: dtype={a['dtype']!r} not supported") - _ensure_dist() - dt = _DTYPES[a["dtype"]] - t = torch.randn(a["num_elements"], dtype=dt, device="cuda") - - comms = _get_communicators() - eligible = [(name, c) for name, c in comms.items() if _supports(name, c, t)] - if not eligible: - raise UnsupportedOpError( - f"sglang_comm allreduce: no variant supports this tensor " - f"(n={a['num_elements']}, dtype={a['dtype']!r}, world_size={a['world_size']})" - ) - - # Mini-benchmark to pick the fastest variant for this exact shape; runs - # entirely outside the timed window. CUDA events on the current stream - # give sub-microsecond resolution and account for cross-rank sync. - pinned: list = [] # keep the tensor alive across the closure call below - pinned.append(t) - times = {} - for name, c in eligible: - try: - for _ in range(3): - _allreduce_with(name, c, t) - torch.cuda.synchronize() - start = torch.cuda.Event(enable_timing=True) - end = torch.cuda.Event(enable_timing=True) - start.record() - for _ in range(5): - _allreduce_with(name, c, t) - end.record() - torch.cuda.synchronize() - times[name] = start.elapsed_time(end) / 5.0 - except Exception: - continue - - if not times: - raise UnsupportedOpError( - "sglang_comm allreduce: all variants failed during selection" - ) - - winner_name = min(times, key=times.get) - winner_comm = dict(eligible)[winner_name] - - def _call(_pin=pinned): - return _allreduce_with(winner_name, winner_comm, t) - - return {"_call": _call, "_winner": winner_name, "_times_us": times} - - -def _kernel_allreduce(ctx: dict) -> None: - ctx["_call"]() - - -def _symm_mem_ag_rs_available() -> bool: - try: - import torch.distributed._symmetric_memory as ts # noqa: F401 - return True - except Exception: - return False - - -def _ensure_symm_mem_group(): - """Symmetric memory needs the group to be opted in. Idempotent.""" - import torch.distributed._symmetric_memory as ts - group = dist.group.WORLD - name = group.group_name - if not ts.is_symm_mem_enabled_for_group(name): - ts.enable_symm_mem_for_group(name) - return name - - -def _prepare_allgather(op: Op) -> dict: - a = op.args - if a["dtype"] not in _DTYPES: - raise UnsupportedOpError(f"sglang_comm allgather: dtype={a['dtype']!r} not supported") - if not _symm_mem_ag_rs_available(): - raise UnsupportedOpError("sglang_comm allgather: torch symm_mem not available") - _ensure_dist() - import torch.distributed._symmetric_memory as ts - group_name = _ensure_symm_mem_group() - dt = _DTYPES[a["dtype"]] - n_per = int(a["num_elements_per_rank"]) - - # symm_mem path: input must be allocated in symm_mem. Allocate via - # ts.empty + rendezvous so peer pointers exist. - inp = ts.empty(n_per, dtype=dt, device="cuda") - ts.rendezvous(inp, group_name) - pinned = [inp] - - def _call(_pin=pinned): - return ts._low_contention_all_gather(inp, group_name) - - return {"_call": _call} - - -def _kernel_allgather(ctx: dict) -> None: - ctx["_call"]() - - -def _prepare_reduce_scatter(op: Op) -> dict: - a = op.args - if a["dtype"] not in _DTYPES: - raise UnsupportedOpError(f"sglang_comm reduce_scatter: dtype={a['dtype']!r} not supported") - if not _symm_mem_ag_rs_available(): - raise UnsupportedOpError("sglang_comm reduce_scatter: torch symm_mem not available") - _ensure_dist() - import torch.distributed._symmetric_memory as ts - group_name = _ensure_symm_mem_group() - dt = _DTYPES[a["dtype"]] - n = int(a["num_elements"]) - ws = int(a["world_size"]) - if n % ws != 0: - raise UnsupportedOpError( - f"sglang_comm reduce_scatter: num_elements={n} not divisible by world_size={ws}" - ) - - inp = ts.empty(n, dtype=dt, device="cuda") - ts.rendezvous(inp, group_name) - pinned = [inp] - - def _call(_pin=pinned): - return ts._low_contention_reduce_scatter(inp, "sum", group_name) - - return {"_call": _call} - - -def _kernel_reduce_scatter(ctx: dict) -> None: - ctx["_call"]() - - -IMPLS = [ - BackendImpl(op_type="allreduce", prepare=_prepare_allreduce, kernel=_kernel_allreduce), - BackendImpl(op_type="allgather", prepare=_prepare_allgather, kernel=_kernel_allgather), - BackendImpl(op_type="reduce_scatter", prepare=_prepare_reduce_scatter, kernel=_kernel_reduce_scatter), -] diff --git a/operatorx/runners/nvidia/backends/torch.py b/operatorx/runners/nvidia/backends/torch.py deleted file mode 100644 index e129d3b2b8..0000000000 --- a/operatorx/runners/nvidia/backends/torch.py +++ /dev/null @@ -1,210 +0,0 @@ -from __future__ import annotations - -import torch -from operatorx.runners import attention - -from operatorx.core import BackendImpl, Op, UnsupportedOpError, lookup_versions - - -def versions() -> dict[str, str]: - return lookup_versions("torch") - -_DTYPES = { - "bf16": torch.bfloat16, - "fp16": torch.float16, - "fp32": torch.float32, - # FP8 dtypes exist in torch but most kernels (torch.matmul, SDPA, NCCL collectives) - # don't accept them directly. We let them through here for tensor construction; ops - # that can't actually run on fp8 will fail at the kernel and be recorded as errors. - "fp8": getattr(torch, "float8_e4m3fn", None), -} - - -def _resolve(dtype: str) -> torch.dtype: - if dtype not in _DTYPES or _DTYPES[dtype] is None: - raise UnsupportedOpError(f"torch backend doesn't support dtype={dtype!r}") - return _DTYPES[dtype] - - -def _prepare_gemm(op: Op) -> dict: - a = op.args - if a.get("activation") is not None: - raise UnsupportedOpError( - f"torch gemm has no native activation fusion; got activation={a['activation']!r}" - ) - if a["dtype_b"] != a["dtype_a"]: - raise UnsupportedOpError( - f"torch gemm requires dtype_a == dtype_b; got {a['dtype_a']}/{a['dtype_b']}" - ) - dt = _resolve(a["dtype_a"]) - M, N, K = a["m"], a["n"], a["k"] - has_bias = bool(a.get("bias")) - out_dt = _DTYPES.get(a.get("dtype_out") or "bf16") or torch.bfloat16 - - if dt == _DTYPES.get("fp8"): - # B200 fp8 path: torch._scaled_mm. Expects A:(M,K) row-major, B:(K,N) column-major, - # so build B as (N,K) and transpose. Scales = 1.0 (no quantization simulated here). - A = torch.randn(M, K, dtype=torch.bfloat16, device="cuda").to(dt) - B = torch.randn(N, K, dtype=torch.bfloat16, device="cuda").to(dt).t() - scale_a = torch.tensor(1.0, device="cuda") - scale_b = torch.tensor(1.0, device="cuda") - bias = torch.randn(N, dtype=out_dt, device="cuda") if has_bias else None - return {"A": A, "B": B, "scale_a": scale_a, "scale_b": scale_b, - "out_dtype": out_dt, "bias": bias, "fp8": True} - - A = torch.randn(M, K, dtype=dt, device="cuda") - B = torch.randn(K, N, dtype=dt, device="cuda") - bias = torch.randn(N, dtype=dt, device="cuda") if has_bias else None - return {"A": A, "B": B, "bias": bias, "fp8": False} - - -def _kernel_gemm(ctx: dict) -> None: - if ctx["fp8"]: - # torch._scaled_mm accepts bias natively as a kwarg. - ctx["C"] = torch._scaled_mm( - ctx["A"], ctx["B"], ctx["scale_a"], ctx["scale_b"], - out_dtype=ctx["out_dtype"], bias=ctx["bias"], - ) - elif ctx["bias"] is not None: - # torch.addmm(bias, A, B) → bias[None,:] + A @ B, native fused bias. - ctx["C"] = torch.addmm(ctx["bias"], ctx["A"], ctx["B"]) - else: - ctx["C"] = torch.matmul(ctx["A"], ctx["B"]) - - -def _ensure_dist() -> None: - import os - - import torch.distributed as dist - if not dist.is_initialized(): - local_rank = int(os.environ.get("LOCAL_RANK", "0")) - torch.cuda.set_device(local_rank) - dist.init_process_group("nccl", device_id=torch.device(f"cuda:{local_rank}")) - - -def _reject_fp8_collective(op: Op, name: str) -> None: - if op.args["dtype"] == "fp8": - raise UnsupportedOpError( - f"torch.distributed/{name} (NCCL) doesn't support fp8 reductions" - ) - - -def _prepare_allreduce(op: Op) -> dict: - _reject_fp8_collective(op, "allreduce") - _ensure_dist() - dt = _resolve(op.args["dtype"]) - t = torch.randn(op.args["num_elements"], dtype=dt, device="cuda") - return {"t": t} - - -def _kernel_allreduce(ctx: dict) -> None: - import torch.distributed as dist - dist.all_reduce(ctx["t"]) - - -def _prepare_allgather(op: Op) -> dict: - _reject_fp8_collective(op, "allgather") - _ensure_dist() - dt = _resolve(op.args["dtype"]) - n_per = op.args["num_elements_per_rank"] - ws = op.args["world_size"] - in_t = torch.randn(n_per, dtype=dt, device="cuda") - out_t = torch.empty(n_per * ws, dtype=dt, device="cuda") - return {"in": in_t, "out": out_t} - - -def _kernel_allgather(ctx: dict) -> None: - import torch.distributed as dist - dist.all_gather_into_tensor(ctx["out"], ctx["in"]) - - -def _prepare_reduce_scatter(op: Op) -> dict: - _reject_fp8_collective(op, "reduce_scatter") - _ensure_dist() - dt = _resolve(op.args["dtype"]) - n = op.args["num_elements"] - ws = op.args["world_size"] - if n % ws != 0: - raise UnsupportedOpError( - f"reduce_scatter requires num_elements divisible by world_size ({n} % {ws} != 0)" - ) - in_t = torch.randn(n, dtype=dt, device="cuda") - out_t = torch.empty(n // ws, dtype=dt, device="cuda") - return {"in": in_t, "out": out_t} - - -def _kernel_reduce_scatter(ctx: dict) -> None: - import torch.distributed as dist - dist.reduce_scatter_tensor(ctx["out"], ctx["in"]) - - -def _prepare_alltoall(op: Op) -> dict: - # NCCL all_to_all_single technically accepts fp8 byte buffers (no reduction), but - # torch's interface still rejects the dtype on most builds. Mark unsupported. - _reject_fp8_collective(op, "alltoall") - _ensure_dist() - dt = _resolve(op.args["dtype"]) - n_per = op.args["num_elements_per_rank"] - ws = op.args["world_size"] - if n_per % ws != 0: - raise UnsupportedOpError( - f"alltoall requires num_elements_per_rank divisible by world_size ({n_per} % {ws} != 0)" - ) - in_t = torch.randn(n_per, dtype=dt, device="cuda") - out_t = torch.empty(n_per, dtype=dt, device="cuda") - return {"in": in_t, "out": out_t} - - -def _kernel_alltoall(ctx: dict) -> None: - import torch.distributed as dist - dist.all_to_all_single(ctx["out"], ctx["in"]) - - -# --- MoE collectives (dispatch/combine) --- -# Reference torch baseline: token routing modeled as plain all_to_all_single. -# Each rank's hidden-state batch is shuffled across the world; volume per rank is -# num_tokens * top_k * hidden / world_size. Matches the bus-bytes formula. - -def _moe_routed_buffer(op: Op) -> torch.Tensor: - _ensure_dist() - a = op.args - dt = _resolve(a["dtype"]) - nt, k, h = a["num_tokens"], a["top_k"], a["hidden"] - # Total per-rank send/recv volume = nt * k * h elements (split evenly across ws). - return torch.randn(nt * k * h, dtype=dt, device="cuda") - - -def _prepare_dispatch(op: Op) -> dict: - buf_in = _moe_routed_buffer(op) - buf_out = torch.empty_like(buf_in) - return {"in": buf_in, "out": buf_out} - - -def _kernel_dispatch(ctx: dict) -> None: - import torch.distributed as dist - dist.all_to_all_single(ctx["out"], ctx["in"]) - - -def _prepare_combine(op: Op) -> dict: - return _prepare_dispatch(op) # symmetric - - -def _kernel_combine(ctx: dict) -> None: - _kernel_dispatch(ctx) - - -# moe_forward is owned by the sglang and flashinfer backends (production MoE -# kernels). The torch backend doesn't expose its own MoE module. - - -IMPLS = [ - BackendImpl(op_type="gemm", prepare=_prepare_gemm, kernel=_kernel_gemm), - BackendImpl(op_type="attention_mha", prepare=attention.prepare, kernel=attention.kernel), - BackendImpl(op_type="attention_mla", prepare=attention.prepare, kernel=attention.kernel), - BackendImpl(op_type="allreduce", prepare=_prepare_allreduce, kernel=_kernel_allreduce), - BackendImpl(op_type="allgather", prepare=_prepare_allgather, kernel=_kernel_allgather), - BackendImpl(op_type="reduce_scatter", prepare=_prepare_reduce_scatter, kernel=_kernel_reduce_scatter), - BackendImpl(op_type="alltoall", prepare=_prepare_alltoall, kernel=_kernel_alltoall), - BackendImpl(op_type="dispatch", prepare=_prepare_dispatch, kernel=_kernel_dispatch), - BackendImpl(op_type="combine", prepare=_prepare_combine, kernel=_kernel_combine), -] diff --git a/operatorx/runners/nvidia/backends/vllm.py b/operatorx/runners/nvidia/backends/vllm.py index d1d5ec1271..1e4bdf3da8 100644 --- a/operatorx/runners/nvidia/backends/vllm.py +++ b/operatorx/runners/nvidia/backends/vllm.py @@ -1,5 +1,7 @@ -"""vLLM routed expert kernels on CUDA.""" +"""GEMM, MoE and attention through vLLM's layers and kernel selection.""" +from operatorx.runners.common.vllm import attention, linear, moe +from operatorx.runners.common.vllm.linear import versions -from operatorx.runners.moe import IMPLS, versions +IMPLS = [*linear.IMPLS, *moe.IMPLS, *attention.IMPLS] __all__ = ["IMPLS", "versions"] diff --git a/operatorx/runners/nvidia/runner.py b/operatorx/runners/nvidia/runner.py index a6155522d5..db3af54fcc 100644 --- a/operatorx/runners/nvidia/runner.py +++ b/operatorx/runners/nvidia/runner.py @@ -1,12 +1,18 @@ from __future__ import annotations +import os +import pkgutil +import time from importlib import import_module import torch from operatorx.core import BackendImpl, Op, Result, UnsupportedOpError +from operatorx.runners.common import profiling, ranks, telemetry, timing + +_COOLDOWN_RATIO = float(os.environ.get("OPERATORX_COOLDOWN_RATIO", "4")) +_COOLDOWN_MAX_S = float(os.environ.get("OPERATORX_COOLDOWN_MAX_S", "1.0")) -_BACKENDS = ["torch", "deepgemm", "flashinfer", "deepep", "sglang", "flashinfer_comm", "sglang_comm", "vllm"] _DISPATCH: dict[tuple[str, str], BackendImpl] = {} _L2_BUF: dict[int, torch.Tensor] = {} @@ -14,53 +20,49 @@ def _load() -> None: if _DISPATCH: return - for name in _BACKENDS: + from operatorx.runners.nvidia import backends + for info in pkgutil.iter_modules(backends.__path__): + if info.name.startswith("_"): + continue try: - mod = import_module(f"operatorx.runners.nvidia.backends.{name}") + mod = import_module(f"operatorx.runners.nvidia.backends.{info.name}") except ImportError: continue for impl in getattr(mod, "IMPLS", []): - _DISPATCH[(impl.op_type, name)] = impl + _DISPATCH[(impl.op_type, info.name)] = impl def _clear_l2() -> None: - """Flush L2 by writing zeros to a buffer sized to the device's L2 cache.""" dev = torch.cuda.current_device() - buf = _L2_BUF.get(dev) - if buf is None: - l2 = torch.cuda.get_device_properties(dev).L2_cache_size - buf = torch.empty(l2, dtype=torch.int8, device=dev) - _L2_BUF[dev] = buf - buf.zero_() + if dev not in _L2_BUF: + size = torch.cuda.get_device_properties(dev).L2_cache_size + _L2_BUF[dev] = torch.empty(size, dtype=torch.int8, device=dev) + _L2_BUF[dev].zero_() -_WARMUP = 5 -_ITERS = 10 -_NUM_BUFFER_SETS = 1 - def run(op: Op) -> Result: _load() impl = _DISPATCH.get((op.type, op.backend)) if impl is None: - raise UnsupportedOpError( - f"nvidia/{op.backend} has no impl for op_type={op.type!r}" - ) - - ctxs = [impl.prepare(op) for _ in range(_NUM_BUFFER_SETS)] + raise UnsupportedOpError(f"nvidia/{op.backend} has no impl for op_type={op.type!r}") + ctx = impl.prepare(op) - for i in range(_WARMUP): - impl.kernel(ctxs[i % _NUM_BUFFER_SETS]) - torch.cuda.synchronize() + fn, cuda_graph = impl.launcher(ctx) if impl.launcher else ((lambda: impl.kernel(ctx)), False) + median_us, telem = telemetry.measure(op, lambda sleep_s: timing.time_op(fn, _clear_l2, sleep_s)) - starts = [torch.cuda.Event(enable_timing=True) for _ in range(_ITERS)] - ends = [torch.cuda.Event(enable_timing=True) for _ in range(_ITERS)] - for i in range(_ITERS): - _clear_l2() - starts[i].record() - impl.kernel(ctxs[i % _NUM_BUFFER_SETS]) - ends[i].record() - torch.cuda.synchronize() + if _COOLDOWN_RATIO > 0.0: + busy_s = median_us * 1e-6 * (timing.ITERS + timing.WARMUP) + time.sleep(min(busy_s * _COOLDOWN_RATIO, _COOLDOWN_MAX_S)) - times = sorted(starts[i].elapsed_time(ends[i]) * 1000.0 for i in range(_ITERS)) - median_us = times[_ITERS // 2] - return Result(op=op, metrics={"latency_us": median_us}) + metrics = {"latency_us": median_us, "cuda_graph": cuda_graph, "telemetry": telem} + if isinstance(ctx, dict) and ctx.get("meta"): + metrics["backend_meta"] = ctx["meta"] + prof = profiling.profile_op(fn) + if prof is not None: + metrics["profile"] = prof + per_rank = ranks.summary(metrics) # every rank's timeline and telemetry, on rank 0 + if per_rank is not None: + metrics["ranks"] = per_rank + if per_rank["capped_ranks"] and metrics.get("telemetry"): + metrics["telemetry"]["capped"] = True # any rank capped caps the op + return Result(op=op, metrics=metrics) diff --git a/operatorx/runners/nvidia/runtime.py b/operatorx/runners/nvidia/runtime.py index b3acc7e68f..d1ce815892 100644 --- a/operatorx/runners/nvidia/runtime.py +++ b/operatorx/runners/nvidia/runtime.py @@ -1,6 +1,4 @@ -"""NVIDIA-specific runtime probes. ``collect()`` returns software/driver -versions to merge into RunInfo.software. Empty dict when not on NVIDIA or -when probes fail.""" +"""NVIDIA driver version for RunInfo.software; empty when the probe fails.""" from __future__ import annotations diff --git a/operatorx/runners/tpu/__init__.py b/operatorx/runners/tpu/__init__.py deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/operatorx/runners/tpu/backends/__init__.py b/operatorx/runners/tpu/backends/__init__.py deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/operatorx/runners/tpu/backends/jax.py b/operatorx/runners/tpu/backends/jax.py deleted file mode 100644 index 595212f595..0000000000 --- a/operatorx/runners/tpu/backends/jax.py +++ /dev/null @@ -1,245 +0,0 @@ -from __future__ import annotations - -import functools - -import jax -import jax.numpy as jnp -from jax.experimental.shard_map import shard_map -from jax.sharding import Mesh, PartitionSpec as P - -from operatorx.core import BackendImpl, Op, UnsupportedOpError, lookup_versions - - -def versions() -> dict[str, str]: - return lookup_versions("jax", "jaxlib") - -_DTYPES = { - "bf16": jnp.bfloat16, - "fp16": jnp.float16, - "fp32": jnp.float32, - # jax has fp8 dtypes; on TPU v6e there's no FP8 MXU so the runtime upcasts - # to bf16 — kernel still runs, timing reflects the bf16 path. - "fp8": jnp.float8_e4m3fn, -} - -_AOT_CACHE: dict[tuple, object] = {} - - -def _aot(fn, *args): - """Lower + compile once per (fn, arg shapes), then reuse the Executable.""" - key = (id(fn),) + tuple( - ("arr", a.shape, str(a.dtype)) if hasattr(a, "shape") else ("id", id(a)) - for a in args - ) - exe = _AOT_CACHE.get(key) - if exe is None: - exe = fn.lower(*args).compile() - _AOT_CACHE[key] = exe - return exe - - -def _resolve(dtype: str) -> jnp.dtype: - if dtype not in _DTYPES: - raise UnsupportedOpError(f"jax backend doesn't support dtype={dtype!r}") - return _DTYPES[dtype] - - -_gemm_jit = jax.jit(jnp.dot) - - -def _prepare_gemm(op: Op) -> dict: - # TPU v6e MXU is natively bf16; fp8 has no hardware path on Trillium and would - # silently upcast to bf16, so we don't expose an fp8 gemm here. - a = op.args - if a["dtype_b"] != a["dtype_a"]: - raise UnsupportedOpError("jax gemm requires dtype_a == dtype_b") - dt = _resolve(a["dtype_a"]) - key = jax.random.PRNGKey(0) - A = jax.random.normal(key, (a["m"], a["k"]), dtype=dt) - B = jax.random.normal(key, (a["k"], a["n"]), dtype=dt) - return {"A": A, "B": B, "fn": _aot(_gemm_jit, A, B)} - - -def _kernel_gemm(ctx: dict) -> None: - ctx["out"] = ctx["fn"](ctx["A"], ctx["B"]) - - - - - - -@functools.lru_cache(maxsize=8) -def _allreduce_fn(world_size: int): - devices = jax.devices()[:world_size] - - def _reduce(x): - return jax.lax.psum(x, "i") - - return jax.pmap(_reduce, axis_name="i", devices=devices) - - -def _prepare_allreduce(op: Op) -> dict: - a = op.args - ws = a["world_size"] - if ws > jax.device_count(): - raise NotImplementedError( - f"world_size={ws} exceeds available TPU devices ({jax.device_count()})" - ) - dt = _resolve(a["dtype"]) - # pmap convention: leading dim = world_size; each rank sees its own row. - x = jax.random.normal(jax.random.PRNGKey(0), (ws, a["num_elements"]), dtype=dt) - fn = _allreduce_fn(ws) - return {"x": x, "fn": _aot(fn, x)} - - -def _kernel_allreduce(ctx: dict) -> None: - ctx["out"] = ctx["fn"](ctx["x"]) - - -@functools.lru_cache(maxsize=8) -def _allgather_fn(world_size: int): - devices = jax.devices()[:world_size] - - def _gather(x): - return jax.lax.all_gather(x, "i", tiled=True) - - return jax.pmap(_gather, axis_name="i", devices=devices) - - -def _prepare_allgather(op: Op) -> dict: - a = op.args - ws = a["world_size"] - if ws > jax.device_count(): - raise NotImplementedError(f"world_size={ws} > available {jax.device_count()}") - dt = _resolve(a["dtype"]) - # pmap convention: leading dim = world_size; each rank sees its own row of (per_rank,). - x = jax.random.normal(jax.random.PRNGKey(0), (ws, a["num_elements_per_rank"]), dtype=dt) - fn = _allgather_fn(ws) - return {"x": x, "fn": _aot(fn, x)} - - -def _kernel_allgather(ctx: dict) -> None: - ctx["out"] = ctx["fn"](ctx["x"]) - - -@functools.lru_cache(maxsize=8) -def _reduce_scatter_fn(world_size: int): - devices = jax.devices()[:world_size] - mesh = Mesh(devices, ("i",)) - - def _rs(x): - return jax.lax.psum_scatter(x, "i", tiled=True) - - return jax.jit(shard_map(_rs, mesh=mesh, in_specs=P(), out_specs=P("i"))) - - -def _prepare_reduce_scatter(op: Op) -> dict: - a = op.args - ws = a["world_size"] - if ws > jax.device_count(): - raise NotImplementedError(f"world_size={ws} > available {jax.device_count()}") - if a["num_elements"] % ws != 0: - raise ValueError("num_elements must be divisible by world_size for reduce_scatter") - dt = _resolve(a["dtype"]) - x = jax.random.normal(jax.random.PRNGKey(0), (a["num_elements"],), dtype=dt) - fn = _reduce_scatter_fn(ws) - return {"x": x, "fn": _aot(fn, x)} - - -def _kernel_reduce_scatter(ctx: dict) -> None: - ctx["out"] = ctx["fn"](ctx["x"]) - - -@functools.lru_cache(maxsize=8) -def _alltoall_fn(world_size: int): - devices = jax.devices()[:world_size] - mesh = Mesh(devices, ("i",)) - - def _ata(x): - return jax.lax.all_to_all(x, "i", split_axis=0, concat_axis=0, tiled=True) - - return jax.jit(shard_map(_ata, mesh=mesh, in_specs=P("i"), out_specs=P("i"))) - - -def _prepare_alltoall(op: Op) -> dict: - a = op.args - ws = a["world_size"] - if ws > jax.device_count(): - raise NotImplementedError(f"world_size={ws} > available {jax.device_count()}") - if a["num_elements_per_rank"] % ws != 0: - raise ValueError("num_elements_per_rank must be divisible by world_size for alltoall") - dt = _resolve(a["dtype"]) - # x is sharded P('i') — each rank sees (num_elements_per_rank,) which holds ws chunks. - x = jax.random.normal(jax.random.PRNGKey(0), (ws * a["num_elements_per_rank"],), dtype=dt) - fn = _alltoall_fn(ws) - return {"x": x, "fn": _aot(fn, x)} - - -def _kernel_alltoall(ctx: dict) -> None: - ctx["out"] = ctx["fn"](ctx["x"]) - - -# moe_forward is not implemented in the jax backend — see runners/tpu/backends/maxtext.py -# for the SOTA TPU implementation via Google's MaxText library. - - -# ---------- dispatch / combine (modeled as all-to-all of routed tokens) ---------- -# -# Real DeepEP dispatch is "send each token to its top_k experts across world_size -# ranks, where (N-1)/N of the data crosses the interconnect." A faithful TPU -# analog using `all_to_all` of size (num_tokens * top_k * hidden) per rank gets -# the same network cost. The kernel does no routing/permutation logic — pure -# comm — which matches the way bytes are scored in ops/collective.py. - - -@functools.lru_cache(maxsize=8) -def _dispatch_fn(world_size: int): - devices = jax.devices()[:world_size] - mesh = Mesh(devices, ("i",)) - - def _d(x): - return jax.lax.all_to_all(x, "i", split_axis=0, concat_axis=0, tiled=True) - - return jax.jit(shard_map(_d, mesh=mesh, in_specs=P("i"), out_specs=P("i"))) - - -def _prepare_dispatch(op: Op) -> dict: - a = op.args - ws = a["world_size"] - if ws > jax.device_count(): - raise UnsupportedOpError( - f"world_size={ws} > available TPU devices ({jax.device_count()})" - ) - dt = _resolve(a["dtype"]) - per_rank_elems = a["num_tokens"] * a["top_k"] * a["hidden"] - if per_rank_elems % ws != 0: - raise UnsupportedOpError( - f"per-rank elems {per_rank_elems} must divide world_size {ws} for all-to-all" - ) - x = jax.random.normal(jax.random.PRNGKey(0), (ws * per_rank_elems,), dtype=dt) - fn = _dispatch_fn(ws) - return {"x": x, "fn": _aot(fn, x)} - - -def _kernel_dispatch(ctx: dict) -> None: - ctx["out"] = ctx["fn"](ctx["x"]) - - -def _prepare_combine(op: Op) -> dict: - # Symmetric to dispatch in network cost — reuse the same all-to-all. - return _prepare_dispatch(op) - - -def _kernel_combine(ctx: dict) -> None: - ctx["out"] = ctx["fn"](ctx["x"]) - - -IMPLS = [ - BackendImpl(op_type="gemm", prepare=_prepare_gemm, kernel=_kernel_gemm), - BackendImpl(op_type="allreduce", prepare=_prepare_allreduce, kernel=_kernel_allreduce), - BackendImpl(op_type="allgather", prepare=_prepare_allgather, kernel=_kernel_allgather), - BackendImpl(op_type="reduce_scatter", prepare=_prepare_reduce_scatter, kernel=_kernel_reduce_scatter), - BackendImpl(op_type="alltoall", prepare=_prepare_alltoall, kernel=_kernel_alltoall), - BackendImpl(op_type="dispatch", prepare=_prepare_dispatch, kernel=_kernel_dispatch), - BackendImpl(op_type="combine", prepare=_prepare_combine, kernel=_kernel_combine), -] diff --git a/operatorx/runners/tpu/backends/sglang.py b/operatorx/runners/tpu/backends/sglang.py deleted file mode 100644 index f0351f60af..0000000000 --- a/operatorx/runners/tpu/backends/sglang.py +++ /dev/null @@ -1,598 +0,0 @@ -"""SGLang-JAX backend for operatorx on TPU. - -Canonical entrypoints from https://github.com/sgl-project/sglang-jax -(install from git main — the 0.0.2 pip release has a different EPMoE API and a -dense-einsum single-device path that main has since replaced with megablox -gmm), matching sgl-jax's production defaults: - - gemm -> the computation inside `sgl_jax.srt.layers.linear.LinearBase` - (`lax.dot_general(..., preferred_element_type)`) over the - module's own weights. No custom kernel for unquantized dense. - moe_forward -> `sgl_jax.srt.layers.moe.EPMoE` — sgl-jax's default MoE - (`--moe-backend epmoe`): shard_map + megablox Pallas gmm, - used for all ep_size including 1. Routing (softmax + topk + - renormalize, the qwen3 norm_topk_prob semantics) happens - caller-side on main, so we compute it inline in the timed - region from a LinearBase router — matching how the vllm - backend's kernels time their internal routing. - moe_gemm -> same EPMoE with ep_size=1 and per-rank-equivalent shapes - (num_tokens*top_k/EP tokens over num_experts/EP experts, - topk=1) — expert compute only, via the same gmm kernel. - attention_mha -> `sgl_jax.srt.kernels.ragged_paged_attention. - ragged_paged_attention_v3.ragged_paged_attention` — the - Pallas RPA-v3 kernel behind sgl-jax's default "fa" - attention backend (fused KV-cache variant). - -sgl-jax has no standalone collective kernels — those stay with the `jax` -backend. -""" -from __future__ import annotations - -import math - -import numpy as np - -import jax -import jax.numpy as jnp -from jax.sharding import Mesh - -from operatorx.core import BackendImpl, Op, UnsupportedOpError, lookup_versions - - -def versions() -> dict[str, str]: - return lookup_versions("sglang-jax", "jax", "jaxlib", "libtpu", "flax") - - -try: - from sgl_jax.srt.layers.moe import EPMoE as _EPMoE - _MOE_AVAILABLE = True - _MOE_ERR: Exception | None = None -except Exception as e: - _EPMoE = None # type: ignore - _MOE_AVAILABLE = False - _MOE_ERR = e - -try: - from sgl_jax.srt.layers.linear import LinearBase as _LinearBase - _LINEAR_AVAILABLE = True - _LINEAR_ERR: Exception | None = None -except Exception as e: - _LinearBase = None # type: ignore - _LINEAR_AVAILABLE = False - _LINEAR_ERR = e - -try: - from sgl_jax.srt.kernels.ragged_paged_attention.ragged_paged_attention_v3 import ( - ragged_paged_attention as _rpa, - ) - _RPA_AVAILABLE = True - _RPA_ERR: Exception | None = None -except Exception as e: - _rpa = None # type: ignore - _RPA_AVAILABLE = False - _RPA_ERR = e - -try: - from sgl_jax.srt.kernels.mla.v2.kernel import ( - mla_ragged_paged_attention as _mla, - get_kv_cache_shape as _mla_kv_cache_shape, - ) - _MLA_AVAILABLE = True - _MLA_ERR: Exception | None = None -except Exception as e: - _mla = None # type: ignore - _mla_kv_cache_shape = None # type: ignore - _MLA_AVAILABLE = False - _MLA_ERR = e - - -_DTYPES = { - "bf16": jnp.bfloat16, - "fp16": jnp.float16, - "fp32": jnp.float32, -} - - -def _resolve_dtype(name: str) -> jnp.dtype: - if name not in _DTYPES: - raise UnsupportedOpError( - f"sglang backend doesn't support dtype={name!r} on TPU " - "(supported: bf16/fp16/fp32)" - ) - return _DTYPES[name] - - -_MODULE_CACHE: dict[tuple, object] = {} -_AOT_CACHE: dict[tuple, object] = {} - - -def _cache_module(key: tuple, module): - if key not in _MODULE_CACHE: - _MODULE_CACHE.clear() - _AOT_CACHE.clear() - _MODULE_CACHE[key] = module - return _MODULE_CACHE[key] - - -def _aot(fn, *args): - key = (id(fn),) + tuple( - ("arr", a.shape, str(a.dtype)) if hasattr(a, "shape") else ("v", a) - for a in args - ) - exe = _AOT_CACHE.get(key) - if exe is None: - exe = fn.lower(*args).compile() - _AOT_CACHE[key] = exe - return exe - - -# EPMoE derives its internal (expert, tensor) moe_mesh from the mesh we pass; -# it expects the standard sgl-jax ("data", "tensor") axis names. Cached per -# ep_size so jit closures keyed on mesh identity stay stable across prepare(). -_MESH_CACHE: dict[int, Mesh] = {} - - -def _make_mesh(ep_size: int) -> Mesh: - if ep_size > jax.device_count(): - raise UnsupportedOpError( - f"ep_size={ep_size} > available TPU devices ({jax.device_count()})" - ) - mesh = _MESH_CACHE.get(ep_size) - if mesh is None: - devices = np.asarray(jax.devices()[:ep_size]).reshape(ep_size, 1) - # sgl-jax builds its meshes with Explicit axis types (see - # srt/utils/mesh_utils.py) — nnx eager weight sharding requires it. - mesh = Mesh(devices, ("data", "tensor"), - axis_types=(jax.sharding.AxisType.Explicit,) * 2) - _MESH_CACHE[ep_size] = mesh - return mesh - - -# ---------- gemm via LinearBase ----------------------------------------------- - - -def _build_linear(K: int, N: int, dtype: jnp.dtype, mesh: Mesh | None = None): - if not _LINEAR_AVAILABLE: - raise UnsupportedOpError( - f"sgl_jax LinearBase not importable: {_LINEAR_ERR!r}" - ) - # The router must live on the SAME mesh as the module it feeds — mixing - # arrays from different meshes in one jit raises incompatible-devices. - mesh = mesh if mesh is not None else _make_mesh(1) - key = ("linear", K, N, str(dtype), tuple(mesh.shape.items())) - if key in _MODULE_CACHE: - return _MODULE_CACHE[key] - # main-branch signature: (input_size, output_size, mesh, use_bias, - # skip_bias_add, params_dtype, kernel_axes, scope_name) — no rngs. - # kernel_axes must be explicit: the default None crashes in - # `P(*kernel_axes)` upstream. (None, None) = replicated weight. Init - # resolves that PartitionSpec eagerly, which requires an active mesh - # context (same requirement as EPMoE). - with jax.sharding.set_mesh(mesh): - module = _LinearBase( - input_size=K, output_size=N, mesh=mesh, - use_bias=False, params_dtype=dtype, - kernel_axes=(None, None), - ) - return _cache_module(key, module) - - -# LinearBase.__call__ computes -# `lax.dot_general(x, weight, (((x.ndim-1,), (0,)), ((), ())), -# preferred_element_type=params_dtype)` (see sgl_jax/srt/layers/linear.py). -# We reproduce that exact call over the module's own weight array, AOT-compiled -# over plain arrays — passing the nnx module through the compiled call would -# flatten its pytree per invocation (~1ms python-side), polluting the timing. -def _make_gemm_call(dt): - @jax.jit - def _gemm_call(x, w): - return jax.lax.dot_general( - x, w, (((x.ndim - 1,), (0,)), ((), ())), - preferred_element_type=dt, - ) - return _gemm_call - - -_GEMM_CALL_CACHE: dict[str, object] = {} - - -def _prepare_gemm(op: Op) -> dict: - a = op.args - if a["dtype_a"] != a["dtype_b"]: - raise UnsupportedOpError("sglang gemm requires dtype_a == dtype_b") - dt = _resolve_dtype(a["dtype_a"]) - - module = _build_linear(a["k"], a["n"], dt) - W = module.weight.value # library-initialized weight, library layout+dtype - A = jax.random.normal(jax.random.PRNGKey(0), (a["m"], a["k"]), dtype=dt) - - fn = _GEMM_CALL_CACHE.setdefault(str(dt), _make_gemm_call(dt)) - exe = _aot(fn, A, W) - return {"fn": exe, "A": A, "W": W} - - -def _kernel_gemm(ctx: dict) -> None: - ctx["out"] = ctx["fn"](ctx["A"], ctx["W"]) - - -# ---------- EPMoE (git main API) ------------------------------------------------ - - -def _build_epmoe(num_experts: int, num_experts_per_tok: int, ep_size: int, - hidden: int, intermediate: int, dtype: jnp.dtype, mesh: Mesh): - if not _MOE_AVAILABLE: - raise UnsupportedOpError( - f"sgl_jax EPMoE not importable: {_MOE_ERR!r}" - ) - key = ("epmoe", num_experts, num_experts_per_tok, ep_size, - hidden, intermediate, str(dtype), tuple(mesh.shape.items())) - if key in _MODULE_CACHE: - return _MODULE_CACHE[key] - with jax.sharding.set_mesh(mesh): - module = _EPMoE( - hidden_size=hidden, - num_experts=num_experts, - num_experts_per_tok=num_experts_per_tok, - ep_size=ep_size, - mesh=mesh, - intermediate_dim=intermediate, - weight_dtype=dtype, - dtype=dtype, - activation="silu", - ) - return _cache_module(key, module) - - -# Compiled MoE executables cached across per-iteration prepare() calls. The -# nnx modules are split once (graphdef static / state pytree of arrays); only -# the cheap state pytrees flow through the compiled call — passing the module -# itself would flatten its full pytree per invocation (~1ms python overhead in -# the timed region) and a fresh nnx.jit closure per prepare would recompile -# every iteration. -_MOE_EXE_CACHE: dict[tuple, tuple] = {} - - -def _moe_executable(cache_key, module, router, tokens, top_k: int, - shared=None): - """shared: optional (sh_gu [H, 2*I_sh], sh_d [I_sh, H]) dense shared-expert - MLP weights, composed with the routed output the way model code does - (n parallel shared MLPs summed == one MLP with n*intermediate, via the - library's lax.dot_general path).""" - from flax import nnx as _nnx - hit = _MOE_EXE_CACHE.get(cache_key) - if hit is not None: - return hit - gd_m, st_m = _nnx.split(module) - gd_r, st_r = _nnx.split(router) - - def _routed(ms, rs, x): - moe = _nnx.merge(gd_m, ms) - rt = _nnx.merge(gd_r, rs) - logits, _ = rt(x) - # main EPMoE takes pre-routed (topk_weights, topk_ids); reproduce the - # qwen3 routing: softmax (fp32) -> top_k -> renormalize. - probs = jax.nn.softmax(logits.astype(jnp.float32), axis=-1) - topk_w, topk_ids = jax.lax.top_k(probs, top_k) - topk_w = topk_w / jnp.sum(topk_w, axis=-1, keepdims=True) - return moe(x, topk_w.astype(moe.dtype), topk_ids) - - if shared is not None: - def _call(ms, rs, x, sh_gu, sh_d): - out = _routed(ms, rs, x) - x2 = x.reshape(-1, x.shape[-1]) - dn = (((1,), (0,)), ((), ())) - proj = jax.lax.dot_general(x2, sh_gu, dn, - preferred_element_type=x.dtype) - gate, up = jnp.split(proj, 2, axis=-1) - sh = jax.lax.dot_general(jax.nn.silu(gate) * up, sh_d, dn, - preferred_element_type=x.dtype) - return out + sh.reshape(out.shape) - - exe = jax.jit(_call).lower(st_m, st_r, tokens, *shared).compile() - else: - exe = jax.jit(_routed).lower(st_m, st_r, tokens).compile() - _MOE_EXE_CACHE[cache_key] = (exe, st_m, st_r) - return exe, st_m, st_r - - -def _prepare_moe_forward(op: Op) -> dict: - a = op.args - dt = _resolve_dtype(a["dtype_act"]) - if a["dtype_weight"] != a["dtype_act"]: - raise UnsupportedOpError( - "sglang moe_forward requires dtype_act == dtype_weight" - ) - ep = a.get("expert_parallel_size", a["world_size"]) - if a["num_experts"] % ep != 0: - raise UnsupportedOpError( - f"num_experts={a['num_experts']} not divisible by EP={ep}" - ) - - mesh = _make_mesh(ep) - module = _build_epmoe( - num_experts=a["num_experts"], - num_experts_per_tok=a["top_k"], - ep_size=ep, - hidden=a["hidden"], intermediate=a["intermediate"], - dtype=dt, mesh=mesh, - ) - router = _build_linear(a["hidden"], a["num_experts"], dt, mesh=mesh) - - key = jax.random.PRNGKey(0) - tokens = jax.random.normal(key, (a["num_tokens"], a["hidden"]), dtype=dt) - - n_shared = a.get("n_shared_experts", 0) - shared_tp = a.get("shared_tensor_parallel_size", 1) - shared = None - if n_shared > 0: - if (n_shared * a["intermediate"]) % shared_tp != 0: - raise UnsupportedOpError( - f"n_shared*intermediate not divisible by shared_tp={shared_tp}") - i_sh = n_shared * a["intermediate"] // shared_tp - shared = ( - jax.random.normal(jax.random.fold_in(key, 4), - (a["hidden"], 2 * i_sh), dtype=dt), - jax.random.normal(jax.random.fold_in(key, 5), - (i_sh, a["hidden"]), dtype=dt), - ) - - cache_key = ("fwd", a["num_experts"], a["top_k"], ep, - a["hidden"], a["intermediate"], str(dt), tokens.shape, - n_shared, shared_tp) - exe, st_m, st_r = _moe_executable(cache_key, module, router, tokens, - top_k=a["top_k"], shared=shared) - ctx = {"fn": exe, "st_m": st_m, "st_r": st_r, "tokens": tokens} - if shared is not None: - ctx["sh_gu"], ctx["sh_d"] = shared - return ctx - - -def _kernel_moe_forward(ctx: dict) -> None: - if "sh_gu" in ctx: - ctx["out"] = ctx["fn"](ctx["st_m"], ctx["st_r"], ctx["tokens"], - ctx["sh_gu"], ctx["sh_d"]) - else: - ctx["out"] = ctx["fn"](ctx["st_m"], ctx["st_r"], ctx["tokens"]) - - -def _prepare_moe_gemm(op: Op) -> dict: - a = op.args - dt = _resolve_dtype(a["dtype_act"]) - if a["dtype_weight"] != a["dtype_act"]: - raise UnsupportedOpError( - "sglang moe_gemm requires dtype_act == dtype_weight" - ) - ep = a.get("expert_parallel_size", 1) - if a["num_experts"] % ep != 0: - raise UnsupportedOpError( - f"num_experts={a['num_experts']} not divisible by EP={ep}" - ) - if (a["num_tokens"] * a["top_k"]) % ep != 0: - raise UnsupportedOpError( - f"num_tokens*top_k={a['num_tokens']*a['top_k']} not divisible by EP={ep}" - ) - - local_tokens = (a["num_tokens"] * a["top_k"]) // ep - local_experts = a["num_experts"] // ep - - mesh = _make_mesh(1) - module = _build_epmoe( - num_experts=local_experts, - num_experts_per_tok=1, - ep_size=1, - hidden=a["hidden"], intermediate=a["intermediate"], - dtype=dt, mesh=mesh, - ) - router = _build_linear(a["hidden"], local_experts, dt, mesh=mesh) - key = jax.random.PRNGKey(0) - tokens = jax.random.normal(key, (local_tokens, a["hidden"]), dtype=dt) - - n_shared = a.get("n_shared_experts", 0) - shared_tp = a.get("shared_tensor_parallel_size", 1) - shared = None - if n_shared > 0: - if (n_shared * a["intermediate"]) % shared_tp != 0: - raise UnsupportedOpError( - f"n_shared*intermediate not divisible by shared_tp={shared_tp}") - i_sh = n_shared * a["intermediate"] // shared_tp - shared = ( - jax.random.normal(jax.random.fold_in(key, 4), - (a["hidden"], 2 * i_sh), dtype=dt), - jax.random.normal(jax.random.fold_in(key, 5), - (i_sh, a["hidden"]), dtype=dt), - ) - - cache_key = ("gemm", local_experts, 1, 1, - a["hidden"], a["intermediate"], str(dt), tokens.shape, - n_shared, shared_tp) - exe, st_m, st_r = _moe_executable(cache_key, module, router, tokens, - top_k=1, shared=shared) - ctx = {"fn": exe, "st_m": st_m, "st_r": st_r, "tokens": tokens} - if shared is not None: - ctx["sh_gu"], ctx["sh_d"] = shared - return ctx - - -def _kernel_moe_gemm(ctx: dict) -> None: - if "sh_gu" in ctx: - ctx["out"] = ctx["fn"](ctx["st_m"], ctx["st_r"], ctx["tokens"], - ctx["sh_gu"], ctx["sh_d"]) - else: - ctx["out"] = ctx["fn"](ctx["st_m"], ctx["st_r"], ctx["tokens"]) - - -# ---------- attention_mha via RPA v3 (fused-KV variant) --------------------------- - -_RPA_PAGE_SIZE = 64 - - -def _align(x: int, n: int) -> int: - return (x + n - 1) // n * n - - -def _prepare_attention_mha(op: Op) -> dict: - if not _RPA_AVAILABLE: - raise UnsupportedOpError( - f"sgl_jax ragged_paged_attention v3 not importable: {_RPA_ERR!r}") - a = op.args - dt = _resolve_dtype(a.get("dtype_q", "bf16")) - B = a["batch_size"] - Sq, Skv = a["seq_len_q"], a["seq_len_kv"] - H = a["num_heads"] - Hkv = a.get("num_heads_kv", H) - D = a["head_dim"] - if Sq > Skv: - raise UnsupportedOpError("attention requires seq_len_q <= seq_len_kv") - - max_num_seqs = _align(B, 8) - pages_per_seq = -(-Skv // _RPA_PAGE_SIZE) - total_pages = max_num_seqs * pages_per_seq + 1 - T = _align(B * Sq, 128) - - key = jax.random.PRNGKey(0) - q = jax.random.normal(key, (T, H, D), dtype=dt) - k = jax.random.normal(jax.random.fold_in(key, 1), (T, Hkv, D), dtype=dt) - v = jax.random.normal(jax.random.fold_in(key, 2), (T, Hkv, D), dtype=dt) - # Fused cache is 5-D packed (see prepare_kv_cache_fused): - # [pages, page_size, (Hkv*2)//packing, packing, D] where packing is the - # number of elements per 32-bit word (bf16 -> 2, fp32 -> 1). - packing = 4 // jnp.dtype(dt).itemsize - kv_cache = jax.random.normal( - jax.random.fold_in(key, 3), - (total_pages, _RPA_PAGE_SIZE, (Hkv * 2) // packing, packing, D), - dtype=dt) - - kv_lens = jnp.asarray([Skv] * B + [0] * (max_num_seqs - B), dtype=jnp.int32) - page_indices = jnp.asarray( - np.arange(max_num_seqs * pages_per_seq, dtype=np.int32)) - cu_q = np.minimum(np.arange(max_num_seqs + 1) * Sq, B * Sq) - cu_q_lens = jnp.asarray(cu_q, dtype=jnp.int32) - # cu_kv_lens = cumsum of page_size-aligned kv lens. - aligned_kv = _align(Skv, _RPA_PAGE_SIZE) - cu_kv = np.minimum(np.arange(max_num_seqs + 1) * aligned_kv, B * aligned_kv) - cu_kv_lens = jnp.asarray(cu_kv, dtype=jnp.int32) - if Sq == 1: - dist = (B, B, B) - elif Sq == Skv: - dist = (0, B, B) - else: - dist = (0, 0, B) - distribution = jnp.asarray(dist, dtype=jnp.int32) - - sm_scale = 1.0 / math.sqrt(D) - - # Their jit donates q/k/v/kv_cache_fused — donation would invalidate our - # buffers across repeated kernel() calls, so re-jit without donation. - base = getattr(_rpa, "__wrapped__", None) - if base is not None: - fn = jax.jit( - lambda q_, k_, v_, c_, kl_, pi_, cq_, ckv_, d_: base( - q_, k_, v_, c_, kl_, pi_, cq_, ckv_, d_, None, - sm_scale=sm_scale, causal=1), - ) - else: - def fn(q_, k_, v_, c_, kl_, pi_, cq_, ckv_, d_): - return _rpa(q_, k_, v_, c_, kl_, pi_, cq_, ckv_, d_, None, - sm_scale=sm_scale, causal=1) - - exe = _aot(fn, q, k, v, kv_cache, kv_lens, page_indices, - cu_q_lens, cu_kv_lens, distribution) - return {"fn": exe, "q": q, "k": k, "v": v, "kv_cache": kv_cache, - "kv_lens": kv_lens, "page_indices": page_indices, - "cu_q_lens": cu_q_lens, "cu_kv_lens": cu_kv_lens, - "distribution": distribution} - - -def _kernel_attention_mha(ctx: dict) -> None: - out, _cache = ctx["fn"]( - ctx["q"], ctx["k"], ctx["v"], ctx["kv_cache"], ctx["kv_lens"], - ctx["page_indices"], ctx["cu_q_lens"], ctx["cu_kv_lens"], - ctx["distribution"]) - ctx["out"] = out - - -# ---------- attention_mla via the MLA v2 Pallas kernel --------------------------- -# -# Absorbed/latent MLA formulation, same as vLLM-TPU's kernel but the sgl v2 -# variant additionally takes cu_kv_lens (page-aligned cumulative kv lens). - - -def _prepare_attention_mla(op: Op) -> dict: - if not _MLA_AVAILABLE: - raise UnsupportedOpError( - f"sgl_jax mla_ragged_paged_attention not importable: {_MLA_ERR!r}") - a = op.args - dt = _resolve_dtype(a.get("dtype_q", "bf16")) - B = a["batch_size"] - Sq, Skv = a["seq_len_q"], a["seq_len_kv"] - H = a["num_heads"] - lkv = a["kv_lora_rank"] - r = a["head_dim_qk_rope"] - if Sq > Skv: - raise UnsupportedOpError("attention requires seq_len_q <= seq_len_kv") - - max_num_seqs = _align(B, 8) - pages_per_seq = -(-Skv // _RPA_PAGE_SIZE) - total_pages = max_num_seqs * pages_per_seq + 1 - T = _align(B * Sq, 128) - - key = jax.random.PRNGKey(0) - ql_nope = jax.random.normal(key, (T, H, lkv), dtype=dt) - q_pe = jax.random.normal(jax.random.fold_in(key, 1), (T, H, r), dtype=dt) - new_kv_c = jax.random.normal(jax.random.fold_in(key, 2), (T, lkv), dtype=dt) - new_k_pe = jax.random.normal(jax.random.fold_in(key, 3), (T, r), dtype=dt) - cache_shape = _mla_kv_cache_shape(total_pages, _RPA_PAGE_SIZE, lkv + r, dt) - cache_kv = jax.random.normal(jax.random.fold_in(key, 4), cache_shape, dtype=dt) - - kv_lens = jnp.asarray([Skv] * B + [0] * (max_num_seqs - B), dtype=jnp.int32) - page_indices = jnp.asarray( - np.arange(max_num_seqs * pages_per_seq, dtype=np.int32)) - cu = np.minimum(np.arange(max_num_seqs + 1) * Sq, B * Sq) - cu_q_lens = jnp.asarray(cu, dtype=jnp.int32) - aligned_kv = _align(Skv, _RPA_PAGE_SIZE) - cu_kv = np.minimum(np.arange(max_num_seqs + 1) * aligned_kv, B * aligned_kv) - cu_kv_lens = jnp.asarray(cu_kv, dtype=jnp.int32) - if Sq == 1: - dist = (B, B, B) - elif Sq == Skv: - dist = (0, B, B) - else: - dist = (0, 0, B) - distribution = jnp.asarray(dist, dtype=jnp.int32) - - sm_scale = 1.0 / math.sqrt(a["head_dim_qk_nope"] + r) - - base = getattr(_mla, "__wrapped__", None) - if base is not None: - fn = jax.jit( - lambda qn_, qp_, kc_, kp_, c_, kl_, pi_, cq_, ckv_, d_: base( - qn_, qp_, kc_, kp_, c_, kl_, pi_, cq_, ckv_, d_, - sm_scale=sm_scale)) - else: - def fn(qn_, qp_, kc_, kp_, c_, kl_, pi_, cq_, ckv_, d_): - return _mla(qn_, qp_, kc_, kp_, c_, kl_, pi_, cq_, ckv_, d_, - sm_scale=sm_scale) - - exe = _aot(fn, ql_nope, q_pe, new_kv_c, new_k_pe, cache_kv, - kv_lens, page_indices, cu_q_lens, cu_kv_lens, distribution) - return {"fn": exe, "ql_nope": ql_nope, "q_pe": q_pe, "new_kv_c": new_kv_c, - "new_k_pe": new_k_pe, "cache_kv": cache_kv, "kv_lens": kv_lens, - "page_indices": page_indices, "cu_q_lens": cu_q_lens, - "cu_kv_lens": cu_kv_lens, "distribution": distribution} - - -def _kernel_attention_mla(ctx: dict) -> None: - out, _cache = ctx["fn"]( - ctx["ql_nope"], ctx["q_pe"], ctx["new_kv_c"], ctx["new_k_pe"], - ctx["cache_kv"], ctx["kv_lens"], ctx["page_indices"], - ctx["cu_q_lens"], ctx["cu_kv_lens"], ctx["distribution"]) - ctx["out"] = out - - -IMPLS = [ - BackendImpl(op_type="gemm", prepare=_prepare_gemm, kernel=_kernel_gemm), - BackendImpl(op_type="moe_forward", prepare=_prepare_moe_forward, kernel=_kernel_moe_forward), - BackendImpl(op_type="moe_gemm", prepare=_prepare_moe_gemm, kernel=_kernel_moe_gemm), - BackendImpl(op_type="attention_mha", prepare=_prepare_attention_mha, kernel=_kernel_attention_mha), - BackendImpl(op_type="attention_mla", prepare=_prepare_attention_mla, kernel=_kernel_attention_mla), -] diff --git a/operatorx/runners/tpu/backends/vllm.py b/operatorx/runners/tpu/backends/vllm.py deleted file mode 100644 index 4545934eaa..0000000000 --- a/operatorx/runners/tpu/backends/vllm.py +++ /dev/null @@ -1,648 +0,0 @@ -"""vLLM-TPU (tpu-inference) backend for operatorx on TPU. - -Canonical entrypoints from https://github.com/vllm-project/tpu-inference -(pip: `tpu-inference`, requires vllm installed as a peer dep), matching what -vLLM-TPU actually selects in production serving: - - gemm -> the computation inside `JaxLinear`/`JaxEinsum` - (`jnp.einsum("mn,np->mp")`) over the module's own weights. - Unquantized dense linears have no custom-kernel path. - moe_forward -> `layers/common/fused_moe_gmm.fused_moe_func` — the GMM_TP / - GMM_EP backend (megablox gmm_v2 + SparseCore ragged_gather), - which is vLLM-TPU's production DEFAULT for bf16 MoE. - GMM_EP when the shape declares expert_parallel_size>1 - (vLLM's --enable-expert-parallel), else GMM_TP. - Set OPERATORX_VLLM_MOE_KERNEL=fused to benchmark the - opt-in `fused_ep_moe` Pallas kernel instead (vLLM-TPU's - USE_MOE_EP_KERNEL=1 path). - moe_gemm -> same entrypoints on a 1-device mesh with shapes rescaled to - the per-rank equivalent of an EP deployment (num_tokens * - top_k / EP tokens over num_experts / EP experts, topk=1) — - no cross-device comms, expert compute only. - attention_mha -> `kernels/ragged_paged_attention/v3/kernel.py` - `ragged_paged_attention` — the production-default Pallas - attention kernel (RPA v3). - -tpu-inference ships no plain reduce_scatter/allreduce kernels (only the fused -all_gather_matmul, a different op shape) — plain collectives stay with the -`jax` backend. -""" -from __future__ import annotations - -import math -import os - -import numpy as np - -import jax -import jax.numpy as jnp -from jax.sharding import Mesh - -from operatorx.core import BackendImpl, Op, UnsupportedOpError, lookup_versions - - -def versions() -> dict[str, str]: - return lookup_versions("tpu-inference", "vllm", "jax", "jaxlib", "libtpu", "flax") - - -try: - from tpu_inference.layers.common.fused_moe_gmm import fused_moe_func as _fused_moe_func - _GMM_MOE_AVAILABLE = True - _GMM_MOE_ERR: Exception | None = None -except Exception as e: - _fused_moe_func = None # type: ignore - _GMM_MOE_AVAILABLE = False - _GMM_MOE_ERR = e - -try: - from tpu_inference.kernels.fused_moe.v1.kernel import fused_ep_moe as _fused_ep_moe - _FUSED_MOE_AVAILABLE = True - _FUSED_MOE_ERR: Exception | None = None -except Exception as e: - _fused_ep_moe = None # type: ignore - _FUSED_MOE_AVAILABLE = False - _FUSED_MOE_ERR = e - -try: - from tpu_inference.layers.jax.linear import JaxLinear as _JaxLinear - _LINEAR_AVAILABLE = True - _LINEAR_ERR: Exception | None = None -except Exception as e: - _JaxLinear = None # type: ignore - _LINEAR_AVAILABLE = False - _LINEAR_ERR = e - -try: - from tpu_inference.kernels.ragged_paged_attention.v3.kernel import ( - ragged_paged_attention as _rpa, - get_kv_cache_shape as _rpa_kv_cache_shape, - ) - _RPA_AVAILABLE = True - _RPA_ERR: Exception | None = None -except Exception as e: - _rpa = None # type: ignore - _rpa_kv_cache_shape = None # type: ignore - _RPA_AVAILABLE = False - _RPA_ERR = e - -# Production MLA is the v2 kernel — layers/common/attention_interface.py calls -# it with hardcoded tuned block sizes (see below); v1 requires explicit block -# sizes and is not what serving uses. -try: - from tpu_inference.kernels.mla.v2.kernel import ( - mla_ragged_paged_attention as _mla, - get_kv_cache_shape as _mla_kv_cache_shape, - ) - _MLA_AVAILABLE = True - _MLA_ERR: Exception | None = None -except Exception as e: - _mla = None # type: ignore - _mla_kv_cache_shape = None # type: ignore - _MLA_AVAILABLE = False - _MLA_ERR = e - -# Mesh axis names as the library defines them (layers/common/sharding.py). -# fused_moe_func's shard_map specs reference these names, so our bench mesh -# must carry them. Fallback literals match the library's current values but -# are validated on-device. -try: - from tpu_inference.layers.common.sharding import ShardingAxisName as _AxN - _AX_MLP_DATA = getattr(_AxN, "MLP_DATA", "data") - _AX_MLP_TENSOR = getattr(_AxN, "MLP_TENSOR", "model") - _AX_EXPERT = getattr(_AxN, "EXPERT", "expert") -except Exception: - _AX_MLP_DATA, _AX_MLP_TENSOR, _AX_EXPERT = "data", "model", "expert" - - -_DTYPES = { - "bf16": jnp.bfloat16, - "fp16": jnp.float16, - "fp32": jnp.float32, -} - - -def _resolve_dtype(name: str) -> jnp.dtype: - if name not in _DTYPES: - raise UnsupportedOpError( - f"vllm backend doesn't support dtype={name!r} on TPU " - "(supported: bf16/fp16/fp32)" - ) - return _DTYPES[name] - - -# Size-1 module cache (nnx modules hold device buffers) + AOT executable cache. -_MODULE_CACHE: dict[tuple, object] = {} -_AOT_CACHE: dict[tuple, object] = {} - - -def _cache_module(key: tuple, module): - if key not in _MODULE_CACHE: - _MODULE_CACHE.clear() - _AOT_CACHE.clear() - _MODULE_CACHE[key] = module - return _MODULE_CACHE[key] - - -def _aot(fn, *args): - key = (id(fn),) + tuple( - ("arr", a.shape, str(a.dtype)) if hasattr(a, "shape") else ("v", a) - for a in args - ) - exe = _AOT_CACHE.get(key) - if exe is None: - exe = fn.lower(*args).compile() - _AOT_CACHE[key] = exe - return exe - - -# ---------- gemm (JaxLinear computation over library weights) ------------------ - - -def _build_linear(K: int, N: int, dtype: jnp.dtype): - if not _LINEAR_AVAILABLE: - raise UnsupportedOpError( - f"tpu-inference JaxLinear not importable: {_LINEAR_ERR!r}" - ) - from flax import nnx - key = ("linear", K, N, str(dtype)) - if key in _MODULE_CACHE: - return _MODULE_CACHE[key] - module = _JaxLinear( - input_size=K, output_size=N, rngs=nnx.Rngs(0), - use_bias=False, quant_config=None, param_dtype=dtype, - ) - return _cache_module(key, module) - - -# JaxLinear.__call__ computes `jnp.einsum("mn,np->mp", x, weight)` (see -# tpu_inference/layers/jax/linear.py). We reproduce that exact computation over -# the module's own weight array, AOT-compiled over plain arrays. Passing the -# nnx module itself through the compiled call would flatten its pytree on every -# invocation (~1ms python-side), polluting the kernel timing — in real serving -# the module is traversed once per model step, not per layer call. -@jax.jit -def _gemm_call(x, w): - return jnp.einsum("mn,np->mp", x, w) - - -def _prepare_gemm(op: Op) -> dict: - a = op.args - if a["dtype_a"] != a["dtype_b"]: - raise UnsupportedOpError("vllm gemm requires dtype_a == dtype_b") - dt = _resolve_dtype(a["dtype_a"]) - - module = _build_linear(a["k"], a["n"], dt) - W = module.weight.value # library-initialized weight, library layout+dtype - A = jax.random.normal(jax.random.PRNGKey(0), (a["m"], a["k"]), dtype=dt) - - exe = _aot(_gemm_call, A, W) - return {"fn": exe, "A": A, "W": W} - - -def _kernel_gemm(ctx: dict) -> None: - ctx["out"] = ctx["fn"](ctx["A"], ctx["W"]) - - -# ---------- MoE meshes ---------------------------------------------------------- - -_MESH_CACHE: dict[tuple, Mesh] = {} - - -def _moe_mesh(n_devices: int, use_ep: bool) -> Mesh: - """Mesh whose axis names satisfy fused_moe_func's shard_map specs. - - In tpu-inference's ShardingAxisName, EXPERT and MLP_TENSOR are the SAME - physical axis ("model") and MLP_DATA/ATTN_DATA are "data" — production - runs a single 2-D (data, model) mesh and GMM_TP vs GMM_EP merely change - which tensor dimension is sharded along "model". So both paths use the - same (data=1, model=n) mesh here (verified on tpu7x, tpu-inference - 0.22.1: MLP_DATA='data', MLP_TENSOR='model', EXPERT='model'). - """ - if n_devices > jax.device_count(): - raise UnsupportedOpError( - f"world_size={n_devices} > available TPU devices ({jax.device_count()})" - ) - del use_ep # same mesh either way; kept in signature for call-site clarity - key = n_devices - mesh = _MESH_CACHE.get(key) - if mesh is None: - devices = np.asarray(jax.devices()[:n_devices]).reshape(1, n_devices) - mesh = Mesh(devices, (_AX_MLP_DATA, _AX_MLP_TENSOR)) - _MESH_CACHE[key] = mesh - return mesh - - -# fused_ep_moe (the opt-in Pallas kernel) needs a 2-D mesh with a "data" axis -# present and all non-EP axes size 1. -def _fused_kernel_mesh(ep_size: int) -> Mesh: - if ep_size > jax.device_count(): - raise UnsupportedOpError( - f"ep_size={ep_size} > available TPU devices ({jax.device_count()})" - ) - key = (ep_size, "fused") - mesh = _MESH_CACHE.get(key) - if mesh is None: - devices = np.asarray(jax.devices()[:ep_size]).reshape(1, ep_size) - mesh = Mesh(devices, ("data", "model")) - _MESH_CACHE[key] = mesh - return mesh - - -def _moe_kernel_choice() -> str: - """'gmm' (production default) or 'fused' (USE_MOE_EP_KERNEL=1 analog).""" - return os.environ.get("OPERATORX_VLLM_MOE_KERNEL", "gmm").lower() - - -# ---------- MoE weights --------------------------------------------------------- - - -def _gmm_weights(num_experts, hidden, intermediate, dtype, key): - """fused_moe_func layout: w1 [E, H, 2*I] (up|gate fused), w2 [E, I, H].""" - w1 = jax.random.normal( - jax.random.fold_in(key, 1), (num_experts, hidden, 2 * intermediate), dtype=dtype) - w2 = jax.random.normal( - jax.random.fold_in(key, 2), (num_experts, intermediate, hidden), dtype=dtype) - return w1, w2 - - -def _fused_weights(num_experts, hidden, intermediate, dtype, key): - """fused_ep_moe layout: w1 [E, 2, H, I], w2 [E, I, H].""" - w1 = jax.random.normal( - jax.random.fold_in(key, 1), (num_experts, 2, hidden, intermediate), dtype=dtype) - w2 = jax.random.normal( - jax.random.fold_in(key, 2), (num_experts, intermediate, hidden), dtype=dtype) - return w1, w2 - - -# ---------- MoE call builders ---------------------------------------------------- - -_MOE_FN_CACHE: dict[tuple, object] = {} - - -def _build_moe_call(kchoice: str, mesh: Mesh, topk: int, use_ep: bool, - has_shared: bool): - """Composed MoE call: routed experts via the selected library kernel, - plus (optionally) the shared-expert dense MLP summed into the output — - the same composition production model code uses (shared experts are - ordinary MLPs at model level in both vLLM-TPU and sgl-jax; n parallel - shared MLPs summed == one MLP with n*intermediate, so we fuse them). - Cached per config so per-iteration prepare() reuses one jitted fn. - """ - key = (kchoice, id(mesh), topk, use_ep, has_shared) - fn = _MOE_FN_CACHE.get(key) - if fn is not None: - return fn - - if kchoice == "fused": - if not _FUSED_MOE_AVAILABLE: - raise UnsupportedOpError( - f"tpu-inference fused_ep_moe not importable: {_FUSED_MOE_ERR!r}") - - def _routed(tokens, w1, w2, gating): - return _fused_ep_moe( - mesh=mesh, tokens=tokens, w1=w1, w2=w2, - gating_output=gating, top_k=topk, - act_fn="silu", scoring_fn="softmax", - ep_axis_name="model", - ) - else: - if not _GMM_MOE_AVAILABLE: - raise UnsupportedOpError( - f"tpu-inference fused_moe_func not importable: {_GMM_MOE_ERR!r}") - - def _routed(tokens, w1, w2, gating): - return _fused_moe_func( - tokens, w1, w2, - None, None, # w1_scale, w2_scale (unquantized) - None, None, # w1_bias, w2_bias - gating, - topk=topk, - renormalize=True, - mesh=mesh, - use_ep=use_ep, - activation="silu", - scoring_fn="softmax", - ) - - if has_shared: - def _call(tokens, w1, w2, gating, sh_gu, sh_d): - out = _routed(tokens, w1, w2, gating) - # Shared-expert dense MLP over all tokens (jnp.einsum — the same - # XLA path vLLM-TPU dense MLPs use), summed with routed output. - proj = jnp.einsum("mn,np->mp", tokens, sh_gu) - gate, up = jnp.split(proj, 2, axis=-1) - shared = jnp.einsum("mn,np->mp", jax.nn.silu(gate) * up, sh_d) - return out + shared - else: - def _call(tokens, w1, w2, gating): - return _routed(tokens, w1, w2, gating) - - fn = jax.jit(_call) - _MOE_FN_CACHE[key] = fn - return fn - - -def _moe_setup(num_tokens: int, num_experts: int, hidden: int, intermediate: int, - top_k: int, n_devices: int, use_ep: bool, dt, - n_shared: int = 0, shared_tp: int = 1) -> dict: - """Shared prepare for moe_forward / moe_gemm across both kernel choices.""" - kchoice = _moe_kernel_choice() - key = jax.random.PRNGKey(0) - tokens = jax.random.normal(key, (num_tokens, hidden), dtype=dt) - gating = jax.random.normal( - jax.random.fold_in(key, 3), (num_tokens, num_experts), dtype=dt) - - if kchoice == "fused": - w1, w2 = _fused_weights(num_experts, hidden, intermediate, dt, key) - mesh = _fused_kernel_mesh(n_devices) - else: - # gmm kernel constraint (fused_moe_gmm.py): (num_tokens*topk) % 16 == 0. - # Production doesn't reject such batches — vLLM-TPU bucket-pads token - # counts (VLLM_TPU_BUCKET_PADDING_GAP); mirror that by padding tokens - # up to the smallest count satisfying the constraint. - if (num_tokens * top_k) % 16 != 0: - step = 16 // math.gcd(top_k, 16) - padded = -(-num_tokens // step) * step - tokens = jax.random.normal(key, (padded, hidden), dtype=dt) - gating = jax.random.normal( - jax.random.fold_in(key, 3), (padded, num_experts), dtype=dt) - w1, w2 = _gmm_weights(num_experts, hidden, intermediate, dt, key) - mesh = _moe_mesh(n_devices, use_ep) - - has_shared = n_shared > 0 - fn = _build_moe_call(kchoice, mesh, top_k, use_ep, has_shared) - - if has_shared: - if (n_shared * intermediate) % shared_tp != 0: - raise UnsupportedOpError( - f"n_shared*intermediate not divisible by shared_tp={shared_tp}") - i_sh = n_shared * intermediate // shared_tp # per-rank shard - sh_gu = jax.random.normal( - jax.random.fold_in(key, 4), (hidden, 2 * i_sh), dtype=dt) - sh_d = jax.random.normal( - jax.random.fold_in(key, 5), (i_sh, hidden), dtype=dt) - exe = _aot(fn, tokens, w1, w2, gating, sh_gu, sh_d) - return {"fn": exe, "tokens": tokens, "w1": w1, "w2": w2, - "gating": gating, "sh_gu": sh_gu, "sh_d": sh_d} - - exe = _aot(fn, tokens, w1, w2, gating) - return {"fn": exe, "tokens": tokens, "w1": w1, "w2": w2, "gating": gating} - - -def _kernel_moe(ctx: dict) -> None: - if "sh_gu" in ctx: - ctx["out"] = ctx["fn"](ctx["tokens"], ctx["w1"], ctx["w2"], - ctx["gating"], ctx["sh_gu"], ctx["sh_d"]) - else: - ctx["out"] = ctx["fn"](ctx["tokens"], ctx["w1"], ctx["w2"], ctx["gating"]) - - -# ---------- moe_forward ---------------------------------------------------------- - - -def _prepare_moe_forward(op: Op) -> dict: - a = op.args - dt = _resolve_dtype(a["dtype_act"]) - if a["dtype_weight"] != a["dtype_act"]: - raise UnsupportedOpError( - "vllm moe_forward requires dtype_act == dtype_weight on this path") - ep = a.get("expert_parallel_size", 1) - ws = a.get("world_size", max(ep, 1)) - if a["num_experts"] % max(ep, 1) != 0: - raise UnsupportedOpError( - f"num_experts={a['num_experts']} not divisible by EP={ep}") - # vLLM semantics: EP>1 => --enable-expert-parallel => GMM_EP; otherwise the - # devices form a TP group over the MoE weights => GMM_TP. - use_ep = ep > 1 - return _moe_setup(a["num_tokens"], a["num_experts"], a["hidden"], - a["intermediate"], a["top_k"], ws, use_ep, dt, - n_shared=a.get("n_shared_experts", 0), - shared_tp=a.get("shared_tensor_parallel_size", 1)) - - -# ---------- moe_gemm (per-rank expert compute, no comms) -------------------------- - - -def _prepare_moe_gemm(op: Op) -> dict: - a = op.args - dt = _resolve_dtype(a["dtype_act"]) - if a["dtype_weight"] != a["dtype_act"]: - raise UnsupportedOpError( - "vllm moe_gemm requires dtype_act == dtype_weight on this path") - ep = a.get("expert_parallel_size", 1) - if a["num_experts"] % max(ep, 1) != 0: - raise UnsupportedOpError( - f"num_experts={a['num_experts']} not divisible by EP={ep}") - if (a["num_tokens"] * a["top_k"]) % max(ep, 1) != 0: - raise UnsupportedOpError( - f"num_tokens*top_k not divisible by EP={ep}") - - local_tokens = (a["num_tokens"] * a["top_k"]) // max(ep, 1) - local_experts = a["num_experts"] // max(ep, 1) - return _moe_setup(local_tokens, local_experts, a["hidden"], - a["intermediate"], 1, 1, False, dt, - n_shared=a.get("n_shared_experts", 0), - shared_tp=a.get("shared_tensor_parallel_size", 1)) - - -# ---------- attention_mha via RPA v3 ---------------------------------------------- - -_RPA_PAGE_SIZE = 64 - - -def _align(x: int, n: int) -> int: - return (x + n - 1) // n * n - - -def _prepare_attention_mha(op: Op) -> dict: - if not _RPA_AVAILABLE: - raise UnsupportedOpError( - f"tpu-inference ragged_paged_attention v3 not importable: {_RPA_ERR!r}") - a = op.args - dt_name = a.get("dtype_q", "bf16") - dt = _resolve_dtype(dt_name) - B = a["batch_size"] - Sq, Skv = a["seq_len_q"], a["seq_len_kv"] - H = a["num_heads"] - Hkv = a.get("num_heads_kv", H) - D = a["head_dim"] - if Sq > Skv: - raise UnsupportedOpError("attention requires seq_len_q <= seq_len_kv") - - max_num_seqs = _align(B, 8) - pages_per_seq = -(-Skv // _RPA_PAGE_SIZE) - total_pages = max_num_seqs * pages_per_seq + 1 - T = _align(B * Sq, 128) - - key = jax.random.PRNGKey(0) - q = jax.random.normal(key, (T, H, D), dtype=dt) - k = jax.random.normal(jax.random.fold_in(key, 1), (T, Hkv, D), dtype=dt) - v = jax.random.normal(jax.random.fold_in(key, 2), (T, Hkv, D), dtype=dt) - - cache_shape = _rpa_kv_cache_shape(total_pages, _RPA_PAGE_SIZE, Hkv, D, dt) - # Pre-populate so the "existing context" (Skv - Sq tokens) reads real data. - kv_cache = jax.random.normal(jax.random.fold_in(key, 3), cache_shape, dtype=dt) - - kv_lens = jnp.asarray( - [Skv] * B + [0] * (max_num_seqs - B), dtype=jnp.int32) - page_indices = jnp.asarray( - np.arange(max_num_seqs * pages_per_seq, dtype=np.int32)) - cu = np.minimum(np.arange(max_num_seqs + 1) * Sq, B * Sq) - cu_q_lens = jnp.asarray(cu, dtype=jnp.int32) - if Sq == 1: - dist = (B, B, B) # decode-only - elif Sq == Skv: - dist = (0, B, B) # pure prefill - else: - dist = (0, 0, B) # extend/mixed - distribution = jnp.asarray(dist, dtype=jnp.int32) - - sm_scale = 1.0 / math.sqrt(D) - - # The library jit donates q/k/v/kv_cache — donation would invalidate our - # buffers across the runner's repeated kernel() calls (pipelined timing), - # so we re-jit the unwrapped function without donation when possible. - base = getattr(_rpa, "__wrapped__", None) - if base is not None: - fn = jax.jit( - lambda q_, k_, v_, c_, kl_, pi_, cq_, d_: base( - q_, k_, v_, c_, kl_, pi_, cq_, d_, - sm_scale=sm_scale, use_causal_mask=True), - ) - else: - def fn(q_, k_, v_, c_, kl_, pi_, cq_, d_): - return _rpa(q_, k_, v_, c_, kl_, pi_, cq_, d_, - sm_scale=sm_scale, use_causal_mask=True) - - exe = _aot(fn, q, k, v, kv_cache, kv_lens, page_indices, cu_q_lens, distribution) - return {"fn": exe, "q": q, "k": k, "v": v, "kv_cache": kv_cache, - "kv_lens": kv_lens, "page_indices": page_indices, - "cu_q_lens": cu_q_lens, "distribution": distribution} - - -def _kernel_attention_mha(ctx: dict) -> None: - out, _cache = ctx["fn"]( - ctx["q"], ctx["k"], ctx["v"], ctx["kv_cache"], ctx["kv_lens"], - ctx["page_indices"], ctx["cu_q_lens"], ctx["distribution"]) - ctx["out"] = out - - -# ---------- attention_mla via the MLA Pallas kernel ------------------------------- -# -# `mla_ragged_paged_attention` implements the absorbed/latent MLA formulation: -# queries are pre-projected into the kv_lora_rank latent space (ql_nope -# [T, H, lkv]) plus the rope part (q_pe [T, H, r]); the paged cache stores -# latent+rope per token, and the output is in latent space (the o-projection -# happens outside, scored separately as gemm). This is exactly the kernel -# vLLM-TPU serves DeepSeek-style models with. - - -def _prepare_attention_mla(op: Op) -> dict: - if not _MLA_AVAILABLE: - raise UnsupportedOpError( - f"tpu-inference mla_ragged_paged_attention not importable: {_MLA_ERR!r}") - a = op.args - dt = _resolve_dtype(a.get("dtype_q", "bf16")) - B = a["batch_size"] - Sq, Skv = a["seq_len_q"], a["seq_len_kv"] - H = a["num_heads"] - lkv = a["kv_lora_rank"] - r = a["head_dim_qk_rope"] - if Sq > Skv: - raise UnsupportedOpError("attention requires seq_len_q <= seq_len_kv") - - max_num_seqs = _align(B, 8) - pages_per_seq = -(-Skv // _RPA_PAGE_SIZE) - total_pages = max_num_seqs * pages_per_seq + 1 - T = _align(B * Sq, 128) - - key = jax.random.PRNGKey(0) - # v2 kernel takes ql_nope HEAD-major [H, T, lkv] (production call site: - # "q_NTA: (num_query_heads, tokens_query, q_lora_rank) # head-major"); - # q_pe stays token-major [T, H, r]. - ql_nope = jax.random.normal(key, (H, T, lkv), dtype=dt) - q_pe = jax.random.normal(jax.random.fold_in(key, 1), (T, H, r), dtype=dt) - new_kv_c = jax.random.normal(jax.random.fold_in(key, 2), (T, lkv), dtype=dt) - new_k_pe = jax.random.normal(jax.random.fold_in(key, 3), (T, r), dtype=dt) - cache_shape = _mla_kv_cache_shape(total_pages, _RPA_PAGE_SIZE, lkv + r, dt) - cache_kv = jax.random.normal(jax.random.fold_in(key, 4), cache_shape, dtype=dt) - - kv_lens = jnp.asarray([Skv] * B + [0] * (max_num_seqs - B), dtype=jnp.int32) - page_indices = jnp.asarray( - np.arange(max_num_seqs * pages_per_seq, dtype=np.int32)) - cu = np.minimum(np.arange(max_num_seqs + 1) * Sq, B * Sq) - cu_q_lens = jnp.asarray(cu, dtype=jnp.int32) - if Sq == 1: - dist = (B, B, B) - elif Sq == Skv: - dist = (0, B, B) - else: - dist = (0, 0, B) - distribution = jnp.asarray(dist, dtype=jnp.int32) - - # Softmax over the full qk dim (nope + rope), DeepSeek convention. - sm_scale = 1.0 / math.sqrt(a["head_dim_qk_nope"] + r) - - # Block sizes exactly as vLLM-TPU's production call site hardcodes them - # (layers/common/attention_interface.py — "TODO: use auto tuner"). - kw = dict(sm_scale=sm_scale, - num_kv_pages_per_block=(3, 1, 1), - num_queries_per_block=(1, 16, 16), - decode_batch_size=4) - base = getattr(_mla, "__wrapped__", None) or _mla - - # Production runs MLA TP-sharded: query heads split across cores, the - # latent KV cache REPLICATED on every rank (that's MLA's memory trade). - # The kernel's internal transpose stack-allocates [H_local, T, lkv] in - # VMEM, which overflows for full H=128/lkv=512 on one core — so we mirror - # the production sharding whenever the head count divides the chip's - # cores; else fall back to single-core. - n_dev = jax.device_count() - if H % n_dev == 0 and n_dev > 1: - from jax.sharding import PartitionSpec as P - devices = np.asarray(jax.devices()).reshape(n_dev) - mesh = Mesh(devices, ("model",)) - - def _sharded(qn_, qp_, kc_, kp_, c_, kl_, pi_, cq_, d_): - return jax.shard_map( - lambda *args: base(*args, **kw), - mesh=mesh, - in_specs=(P("model", None, None), # ql_nope [H, T, lkv] - P(None, "model", None), # q_pe [T, H, r] - P(None, None), # new_kv_c (replicated) - P(None, None), # new_k_pe (replicated) - P(*(None,) * len(c_.shape)), # cache (replicated) - P(None), P(None), P(None), P(None)), - out_specs=(P(None, "model", None), P(*(None,) * len(c_.shape))), - check_vma=False, - )(qn_, qp_, kc_, kp_, c_, kl_, pi_, cq_, d_) - - fn = jax.jit(_sharded) - else: - def fn(qn_, qp_, kc_, kp_, c_, kl_, pi_, cq_, d_): - return base(qn_, qp_, kc_, kp_, c_, kl_, pi_, cq_, d_, **kw) - fn = jax.jit(fn) - - exe = _aot(fn, ql_nope, q_pe, new_kv_c, new_k_pe, cache_kv, - kv_lens, page_indices, cu_q_lens, distribution) - return {"fn": exe, "ql_nope": ql_nope, "q_pe": q_pe, "new_kv_c": new_kv_c, - "new_k_pe": new_k_pe, "cache_kv": cache_kv, "kv_lens": kv_lens, - "page_indices": page_indices, "cu_q_lens": cu_q_lens, - "distribution": distribution} - - -def _kernel_attention_mla(ctx: dict) -> None: - out, _cache = ctx["fn"]( - ctx["ql_nope"], ctx["q_pe"], ctx["new_kv_c"], ctx["new_k_pe"], - ctx["cache_kv"], ctx["kv_lens"], ctx["page_indices"], - ctx["cu_q_lens"], ctx["distribution"]) - ctx["out"] = out - - -IMPLS = [ - BackendImpl(op_type="gemm", prepare=_prepare_gemm, kernel=_kernel_gemm), - BackendImpl(op_type="moe_forward", prepare=_prepare_moe_forward, kernel=_kernel_moe), - BackendImpl(op_type="moe_gemm", prepare=_prepare_moe_gemm, kernel=_kernel_moe), - BackendImpl(op_type="attention_mha", prepare=_prepare_attention_mha, kernel=_kernel_attention_mha), - BackendImpl(op_type="attention_mla", prepare=_prepare_attention_mla, kernel=_kernel_attention_mla), -] diff --git a/operatorx/runners/tpu/runner.py b/operatorx/runners/tpu/runner.py deleted file mode 100644 index cf341ddfa2..0000000000 --- a/operatorx/runners/tpu/runner.py +++ /dev/null @@ -1,94 +0,0 @@ -from __future__ import annotations - -import time -from importlib import import_module - -import jax - -from operatorx.core import BackendImpl, Op, Result, UnsupportedOpError - -_BACKENDS = ["jax", "vllm", "sglang"] -_DISPATCH: dict[tuple[str, str], BackendImpl] = {} - - -def _load() -> None: - if _DISPATCH: - return - for name in _BACKENDS: - try: - mod = import_module(f"operatorx.runners.tpu.backends.{name}") - except ImportError: - continue - for impl in getattr(mod, "IMPLS", []): - _DISPATCH[(impl.op_type, name)] = impl - - -_WARMUP = 3 -_ITERS = 10 - - -def run(op: Op) -> Result: - _load() - impl = _DISPATCH.get((op.type, op.backend)) - if impl is None: - raise UnsupportedOpError( - f"tpu/{op.backend} has no impl for op_type={op.type!r}" - ) - - # Per-iter prepare + per-iter timer. Prepare runs outside the timed region - # so each kernel call sees a fresh input buffer. This defeats two JAX-side - # caching pitfalls that bias measurements: - # 1. `jax.device_get` caches the host numpy buffer on its source array, - # so repeated D2H against the same `jax.Array` reads the cache (we - # observed 7us flat for 2GB transfers). - # 2. XLA's ConstantFolding pass can elide work when the same value flows - # through identical inputs across iterations. - # Trade-off: re-allocates input buffers every iter; relies on refcount- - # driven HBM cleanup when ctx is rebound. Per-iter timer noise is amortized - # over _ITERS samples. - for _ in range(_WARMUP): - ctx = impl.prepare(op) - impl.kernel(ctx) - jax.block_until_ready(ctx.get("out")) - - times = [] - for _ in range(_ITERS): - ctx = impl.prepare(op) - for v in ctx.values(): - if hasattr(v, "block_until_ready"): - v.block_until_ready() - t0 = time.perf_counter() - impl.kernel(ctx) - jax.block_until_ready(ctx.get("out")) - times.append((time.perf_counter() - t0) * 1e6) - - times.sort() - med = times[_ITERS // 2] - metrics = {"latency_us": med} - - # Single-shot host timing is dominated by dispatch overhead (~50-500us) - # once the kernel itself is sub-millisecond, so small (e.g. decode-shape) - # ops read as launch latency rather than kernel time. Re-time those in a - # pipelined loop: enqueue K calls back-to-back and block once — JAX - # dispatch is async, so the host runs ahead and the device executes the - # queue contiguously. Per-call time then approaches true device latency - # (still an upper bound if host enqueue can't keep up). - if med < 1000.0: - K = min(512, max(16, int(4000.0 / max(med, 1.0)))) - ctx = impl.prepare(op) - for v in ctx.values(): - if hasattr(v, "block_until_ready"): - v.block_until_ready() - impl.kernel(ctx) # one unmeasured call so the executable is hot - jax.block_until_ready(ctx.get("out")) - t0 = time.perf_counter() - for _ in range(K): - impl.kernel(ctx) - jax.block_until_ready(ctx.get("out")) - pipelined = (time.perf_counter() - t0) * 1e6 / K - metrics["latency_us_single_shot"] = med - metrics["latency_us_pipelined"] = pipelined - # Best per-call kernel-latency estimate. - metrics["latency_us"] = min(med, pipelined) - - return Result(op=op, metrics=metrics) diff --git a/operatorx/runners/tpu/runtime.py b/operatorx/runners/tpu/runtime.py deleted file mode 100644 index b9d9a521fa..0000000000 --- a/operatorx/runners/tpu/runtime.py +++ /dev/null @@ -1,5 +0,0 @@ -from __future__ import annotations - - -def collect() -> dict[str, str]: - return {} # FIXME: libtpu / TPU runtime version if useful beyond importlib.metadata diff --git a/operatorx/runners/trainium/__init__.py b/operatorx/runners/trainium/__init__.py deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/operatorx/runners/trainium/backends/__init__.py b/operatorx/runners/trainium/backends/__init__.py deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/operatorx/runners/trainium/backends/nkilib.py b/operatorx/runners/trainium/backends/nkilib.py deleted file mode 100644 index 509e729783..0000000000 --- a/operatorx/runners/trainium/backends/nkilib.py +++ /dev/null @@ -1,468 +0,0 @@ -"""NKI backend for Trainium. - -Compute ops are @nki.jit kernels (real NKI): - - gemm -> LNC=2-sharded matmul (bundled NKI, in this file) - - attention_mha -> nkilib.core.attention.attention_cte - -Collectives route through torch_xla's xm.* primitives: - - allreduce, allgather, reduce_scatter, alltoall -> xm.* - -Why not nki.collectives? In the currently shipped Neuron SDK (2.29.0 / -neuronx-cc 2.24 / nki 0.3.0) on Trainium3, the NKI-emitted collective NEFFs -trigger a runtime assertion in libnrt (`enc.cc:3663: -Assertion alg == ENC_ALG_MESH failed`) — independent of LNC mode, replica -group shape, or torchrun init. Compiler internal errors (`NCC_ILLC059`, -`NCC_INLA001`) also block the canonical SBUF-staged pattern under LNC=2. -The error messages themselves say "Please open a support ticket". torch_xla -collectives use a different (HLO-level) lowering that *does* work, so we use -that for the operatorx collective ops until the NKI path is fixed in a -future SDK release. - -The `nl.spmd_dim` / `nl.nc()` SPMD-launch APIs that the AWS docs reference -are NOT yet shipped (NKI 0.3.0 only has `kernel[N]` with N ∈ {1, 2}). We use -the bracket form + in-kernel `nl.program_id` for multi-NC sharding. -""" -from __future__ import annotations - -import math - -import torch -import torch_neuronx # noqa: F401 registers the neuron device -import torch_xla.core.xla_model as xm - -import nki -import nki.isa as nisa -import nki.language as nl - - -from operatorx.core import BackendImpl, Op, UnsupportedOpError, lookup_versions - -# Import once at module load so the @nki.jit dispatchers are ready. -from nkilib.core.attention.attention_cte import attention_cte - - - -def versions() -> dict[str, str]: - return lookup_versions("nki", "nkilib", "neuronx-cc", "torch_neuronx", "torch_xla") - - -_DTYPES = { - "bf16": torch.bfloat16, - "fp16": torch.float16, - "fp32": torch.float32, - # NKI accepts both float8_e4m3 and float8_e5m2 for nc_matmul, but PyTorch - # only ships e4m3fn (finite-NaN variant) which doesn't map cleanly: - # _torch_to_neuron_dtype either rejects it ("float8_e4m3fn is not - # supported in nki") or requires UNSAFE_FP8FNCAST which then creates a - # mixed legacy/OCP type at the compiler level (NCC_EOCP001). float8_e5m2 - # maps to nl.float8_e5m2 directly with no casting layer, so use that as - # our "fp8" dtype on Trainium. Numerically different from e4m3 (more - # exponent, less mantissa) but supported end-to-end. - "fp8": torch.float8_e5m2, -} - - -def _resolve(dtype: str) -> torch.dtype: - if dtype not in _DTYPES or _DTYPES[dtype] is None: - raise UnsupportedOpError(f"nkilib doesn't support dtype={dtype!r}") - return _DTYPES[dtype] - - -def _device(): - return xm.xla_device() - - -# ---------- gemm: LNC=2 matmul via standalone NKI ---------- -# -# FIXME: revisit for LNC=2 gemm entrypoint -# -# Shape constraint: M%2048, K%1024, N%2048 (so NUM_BLOCK_N = N/1024 is even). - - -_GEMM_TILES_IN_BLOCK_M = 16 -_GEMM_TILES_IN_BLOCK_N = 2 -_GEMM_TILES_IN_BLOCK_K = 8 -_GEMM_TILES_IN_BLOCK_K_FP8 = 4 # double_row consumes 256 K-elts/call vs 128 → half the K-subtiles for same 1024 BLOCK_K -_GEMM_BLOCK_M = 128 * _GEMM_TILES_IN_BLOCK_M # 2048 -_GEMM_BLOCK_N = 512 * _GEMM_TILES_IN_BLOCK_N # 1024 -_GEMM_BLOCK_K = 128 * _GEMM_TILES_IN_BLOCK_K # 1024 - - -@nki.jit -def _matmul_lnc2(lhsT, rhs): - """N-block-sharded matmul. With kernel[2], num_programs(0)=2 and each - physical NC handles NUM_BLOCK_N/2 N-blocks. With kernel[1] (or no - bracket), it falls back to one program handling all N-blocks.""" - K, M = lhsT.shape - _, N = rhs.shape - TILE_M = nl.tile_size.gemm_stationary_fmax - TILE_K = nl.tile_size.pmax - TILE_N = nl.tile_size.gemm_moving_fmax - BLOCK_M = TILE_M * _GEMM_TILES_IN_BLOCK_M - BLOCK_N = TILE_N * _GEMM_TILES_IN_BLOCK_N - BLOCK_K = TILE_K * _GEMM_TILES_IN_BLOCK_K - NUM_BLOCK_M = M // BLOCK_M - NUM_BLOCK_N = N // BLOCK_N - NUM_BLOCK_K = K // BLOCK_K - - result = nl.ndarray((M, N), dtype=lhsT.dtype, buffer=nl.shared_hbm) - n_progs = nl.num_programs(axes=0) - pid = nl.program_id(axis=0) - BLOCKS_PER_PID = NUM_BLOCK_N // n_progs - N_START = pid * BLOCKS_PER_PID - - for ln in nl.affine_range(BLOCKS_PER_PID): - n = N_START + ln - - result_tmps = [] - for m_idx in range(NUM_BLOCK_M): - block_m = [] - for bm_idx in range(_GEMM_TILES_IN_BLOCK_M): - block_n = [] - for bn_idx in range(_GEMM_TILES_IN_BLOCK_N): - t = nl.ndarray(shape=(TILE_M, TILE_N), dtype=lhsT.dtype, buffer=nl.sbuf) - nisa.memset(dst=t, value=0.0) - block_n.append(t) - block_m.append(block_n) - result_tmps.append(block_m) - - for k in nl.sequential_range(NUM_BLOCK_K): - rhs_tiles = [] - for bk_r in range(_GEMM_TILES_IN_BLOCK_K): - rt = nl.ndarray(shape=(TILE_K, BLOCK_N), dtype=rhs.dtype, buffer=nl.sbuf) - nisa.dma_copy(dst=rt[0:TILE_K, 0:BLOCK_N], - src=rhs[(_GEMM_TILES_IN_BLOCK_K * k + bk_r) * TILE_K:(_GEMM_TILES_IN_BLOCK_K * k + bk_r + 1) * TILE_K, - BLOCK_N * n:BLOCK_N * (n + 1)]) - rhs_tiles.append(rt) - for m in nl.affine_range(NUM_BLOCK_M): - lhsT_tiles = [] - for bk_l in nl.affine_range(_GEMM_TILES_IN_BLOCK_K): - lt = nl.ndarray(shape=(TILE_K, BLOCK_M), dtype=lhsT.dtype, buffer=nl.sbuf) - nisa.dma_copy(dst=lt[0:TILE_K, 0:BLOCK_M], - src=lhsT[(_GEMM_TILES_IN_BLOCK_K * k + bk_l) * TILE_K:(_GEMM_TILES_IN_BLOCK_K * k + bk_l + 1) * TILE_K, - BLOCK_M * m:BLOCK_M * (m + 1)]) - lhsT_tiles.append(lt) - for bn in nl.affine_range(_GEMM_TILES_IN_BLOCK_N): - for bm in nl.affine_range(_GEMM_TILES_IN_BLOCK_M): - rt2 = nl.ndarray(shape=(TILE_M, TILE_N), dtype=nl.float32, buffer=nl.psum) - for bk in nl.affine_range(_GEMM_TILES_IN_BLOCK_K): - nisa.nc_matmul(dst=rt2, - stationary=lhsT_tiles[bk][0:TILE_K, bm * TILE_M:(bm + 1) * TILE_M], - moving=rhs_tiles[bk][0:TILE_K, bn * TILE_N:(bn + 1) * TILE_N]) - nisa.tensor_tensor(dst=result_tmps[m][bm][bn], - data1=result_tmps[m][bm][bn], data2=rt2, op=nl.add) - - for m in nl.affine_range(NUM_BLOCK_M): - for bm in nl.affine_range(_GEMM_TILES_IN_BLOCK_M): - packed = nl.ndarray(shape=(TILE_M, BLOCK_N), dtype=nl.float32, buffer=nl.sbuf) - for bn in nl.affine_range(_GEMM_TILES_IN_BLOCK_N): - nisa.tensor_copy(dst=packed[0:TILE_M, bn * TILE_N:(bn + 1) * TILE_N], - src=result_tmps[m][bm][bn][0:TILE_M, 0:TILE_N]) - nisa.dma_copy(dst=result[(_GEMM_TILES_IN_BLOCK_M * m + bm) * TILE_M:(_GEMM_TILES_IN_BLOCK_M * m + bm + 1) * TILE_M, - BLOCK_N * n:BLOCK_N * (n + 1)], - src=packed[0:TILE_M, 0:BLOCK_N]) - return result - - -@nki.jit -def _matmul_lnc2_fp8(lhsT, rhs, out_dtype=nl.bfloat16): - """Same N-block sharding as _matmul_lnc2, but stationary+moving tiles are - laid out (partition=128, 2, free) and fed to nc_matmul with - perf_mode="double_row" — each nc_matmul consumes 256 K-elements (2x the - bf16 path), accumulator in fp32, output cast to out_dtype on writeback.""" - K, M = lhsT.shape - _, N = rhs.shape - TILE_M = nl.tile_size.gemm_stationary_fmax # 128 - TILE_K = nl.tile_size.pmax # 128 (partition dim) - TILE_N = nl.tile_size.gemm_moving_fmax # 512 - DOUBLE = 2 - K_PER_CALL = TILE_K * DOUBLE # 256 K-elements per nc_matmul - BLOCK_M = TILE_M * _GEMM_TILES_IN_BLOCK_M # 2048 - BLOCK_N = TILE_N * _GEMM_TILES_IN_BLOCK_N # 1024 - BLOCK_K = K_PER_CALL * _GEMM_TILES_IN_BLOCK_K_FP8 # 256 * 4 = 1024 - NUM_BLOCK_M = M // BLOCK_M - NUM_BLOCK_N = N // BLOCK_N - NUM_BLOCK_K = K // BLOCK_K - - result = nl.ndarray((M, N), dtype=out_dtype, buffer=nl.shared_hbm) - n_progs = nl.num_programs(axes=0) - pid = nl.program_id(axis=0) - BLOCKS_PER_PID = NUM_BLOCK_N // n_progs - N_START = pid * BLOCKS_PER_PID - - for ln in nl.affine_range(BLOCKS_PER_PID): - n = N_START + ln - - result_tmps = [] - for m_idx in range(NUM_BLOCK_M): - block_m = [] - for bm_idx in range(_GEMM_TILES_IN_BLOCK_M): - block_n = [] - for bn_idx in range(_GEMM_TILES_IN_BLOCK_N): - t = nl.ndarray(shape=(TILE_M, TILE_N), dtype=nl.float32, buffer=nl.sbuf) - nisa.memset(dst=t, value=0.0) - block_n.append(t) - block_m.append(block_n) - result_tmps.append(block_m) - - for k in nl.sequential_range(NUM_BLOCK_K): - rhs_tiles = [] - for bk_r in range(_GEMM_TILES_IN_BLOCK_K_FP8): - rt = nl.ndarray(shape=(TILE_K, DOUBLE, BLOCK_N), dtype=rhs.dtype, buffer=nl.sbuf) - k_start = k * BLOCK_K + bk_r * K_PER_CALL - nisa.dma_copy(dst=rt[0:TILE_K, 0, 0:BLOCK_N], - src=rhs[k_start:k_start + TILE_K, - BLOCK_N * n:BLOCK_N * (n + 1)]) - nisa.dma_copy(dst=rt[0:TILE_K, 1, 0:BLOCK_N], - src=rhs[k_start + TILE_K:k_start + K_PER_CALL, - BLOCK_N * n:BLOCK_N * (n + 1)]) - rhs_tiles.append(rt) - for m in nl.affine_range(NUM_BLOCK_M): - lhsT_tiles = [] - for bk_l in nl.affine_range(_GEMM_TILES_IN_BLOCK_K_FP8): - lt = nl.ndarray(shape=(TILE_K, DOUBLE, BLOCK_M), dtype=lhsT.dtype, buffer=nl.sbuf) - k_start = k * BLOCK_K + bk_l * K_PER_CALL - nisa.dma_copy(dst=lt[0:TILE_K, 0, 0:BLOCK_M], - src=lhsT[k_start:k_start + TILE_K, - BLOCK_M * m:BLOCK_M * (m + 1)]) - nisa.dma_copy(dst=lt[0:TILE_K, 1, 0:BLOCK_M], - src=lhsT[k_start + TILE_K:k_start + K_PER_CALL, - BLOCK_M * m:BLOCK_M * (m + 1)]) - lhsT_tiles.append(lt) - for bn in nl.affine_range(_GEMM_TILES_IN_BLOCK_N): - for bm in nl.affine_range(_GEMM_TILES_IN_BLOCK_M): - rt2 = nl.ndarray(shape=(TILE_M, TILE_N), dtype=nl.float32, buffer=nl.psum) - for bk in nl.affine_range(_GEMM_TILES_IN_BLOCK_K_FP8): - nisa.nc_matmul( - dst=rt2, - stationary=lhsT_tiles[bk][0:TILE_K, 0:DOUBLE, - bm * TILE_M:(bm + 1) * TILE_M], - moving=rhs_tiles[bk][0:TILE_K, 0:DOUBLE, - bn * TILE_N:(bn + 1) * TILE_N], - perf_mode="double_row", - ) - nisa.tensor_tensor(dst=result_tmps[m][bm][bn], - data1=result_tmps[m][bm][bn], data2=rt2, op=nl.add) - - for m in nl.affine_range(NUM_BLOCK_M): - for bm in nl.affine_range(_GEMM_TILES_IN_BLOCK_M): - packed = nl.ndarray(shape=(TILE_M, BLOCK_N), dtype=out_dtype, buffer=nl.sbuf) - for bn in nl.affine_range(_GEMM_TILES_IN_BLOCK_N): - nisa.tensor_copy(dst=packed[0:TILE_M, bn * TILE_N:(bn + 1) * TILE_N], - src=result_tmps[m][bm][bn][0:TILE_M, 0:TILE_N]) - nisa.dma_copy(dst=result[(_GEMM_TILES_IN_BLOCK_M * m + bm) * TILE_M:(_GEMM_TILES_IN_BLOCK_M * m + bm + 1) * TILE_M, - BLOCK_N * n:BLOCK_N * (n + 1)], - src=packed[0:TILE_M, 0:BLOCK_N]) - return result - - -# Map operatorx dtype strings -> nki.language dtypes for the kernel's -# out_dtype kwarg (compile-time constant). -_NL_DTYPES = { - "bf16": nl.bfloat16, - "fp16": nl.float16, - "fp32": nl.float32, -} - - -def _ceil_to(x: int, multiple: int) -> int: - """Round x up to the next multiple of `multiple`. Used to pad gemm dims - so any shape can run through the LNC=2 block-tiled kernel — the kernel's - extra rows/cols see random padding and waste a small slice of compute, - but the testlist no longer has to be filtered to aligned-only shapes.""" - return ((x + multiple - 1) // multiple) * multiple - - -def _prepare_gemm(op: Op) -> dict: - a = op.args - if a["dtype_b"] != a["dtype_a"]: - raise UnsupportedOpError("nkilib gemm requires dtype_a == dtype_b") - if a.get("bias") or a.get("activation"): - raise UnsupportedOpError("nkilib gemm has no bias / activation fusion") - M_req, N_req, K_req = a["m"], a["n"], a["k"] - # Pad up to the next LNC=2-tiled block multiple. The kernel runs on the - # padded shape; latency reflects that work. Dashboard FLOPs/bytes still - # use the original (M_req, N_req, K_req) so %SOL is reported against the - # logical op, not the padded one — that understates SOL when the pad - # ratio is large but matches what a real model run with this op would - # actually see latency-wise. - M = _ceil_to(M_req, _GEMM_BLOCK_M) - K = _ceil_to(K_req, _GEMM_BLOCK_K) - N = _ceil_to(N_req, _GEMM_BLOCK_N * 2) - dt_in = _resolve(a["dtype_a"]) - is_fp8 = (a["dtype_a"] == "fp8") - out_dt_str = a.get("dtype_out", "bf16") - if out_dt_str not in _NL_DTYPES: - raise UnsupportedOpError(f"nkilib gemm dtype_out={out_dt_str!r} not in {list(_NL_DTYPES)}") - out_nl_dtype = _NL_DTYPES[out_dt_str] - # FP8 has no native randn — generate bf16 noise and cast. - if is_fp8: - lhsT = torch.randn(K, M, dtype=torch.bfloat16, device="cpu").to(dt_in).to(_device()) - rhs = torch.randn(K, N, dtype=torch.bfloat16, device="cpu").to(dt_in).to(_device()) - else: - lhsT = torch.randn(K, M, dtype=dt_in, device=_device()) - rhs = torch.randn(K, N, dtype=dt_in, device=_device()) - return {"lhsT": lhsT, "rhs": rhs, "is_fp8": is_fp8, "out_dtype": out_nl_dtype} - - -def _kernel_gemm(ctx: dict) -> None: - # kernel[2] sets LNC=2 mode: 2 SPMD programs across the 2 NC-v4 of the - # LNC=2 logical NC. The kernel uses nl.program_id(0) to differentiate. - # OPERATORX_NKILIB_LNC=1 forces single-NC for A/B benchmarking. - import os as _os - _lnc = int(_os.environ.get("OPERATORX_NKILIB_LNC", "2")) - if ctx.get("is_fp8"): - ctx["out"] = _matmul_lnc2_fp8[_lnc](ctx["lhsT"], ctx["rhs"], out_dtype=ctx["out_dtype"]) - else: - ctx["out"] = _matmul_lnc2[_lnc](ctx["lhsT"], ctx["rhs"]) - xm.mark_step() - - -def _prepare_attention_mha(op: Op) -> dict: - """attention_cte kernel layout (defaults: tp_q=True, tp_k=False): - q: (batch_size * num_heads_q, seqlen_q, head_dim) - k: (batch_size * num_heads_kv, head_dim, seqlen_kv) <- seq is LAST - v: (batch_size * num_heads_kv, seqlen_kv, head_dim) - GQA is expressed by the bs/bs_kv ratio (q's first dim vs k's first dim). - """ - a = op.args - if a.get("kv_layout", "contig") != "contig": - raise UnsupportedOpError("nkilib attention_cte: only contiguous KV") - for k in ("dtype_q", "dtype_k", "dtype_v"): - if a[k] == "fp8": - raise UnsupportedOpError( - f"nkilib attention_cte fp8 KV not wired here ({k}={a[k]})" - ) - if not (a["dtype_q"] == a["dtype_k"] == a["dtype_v"]): - raise UnsupportedOpError( - f"nkilib attention_cte needs uniform Q/K/V dtype " - f"(got {a['dtype_q']}/{a['dtype_k']}/{a['dtype_v']})" - ) - - B, S_q, S_kv = a["batch_size"], a["seq_len_q"], a["seq_len_kv"] - H, H_kv, D = a["num_heads"], a["num_heads_kv"], a["head_dim"] - dt = _resolve(a["dtype_q"]) - causal = a.get("causal", True) - scale = 1.0 / math.sqrt(D) - - q = torch.randn(B * H, S_q, D, dtype=dt, device=_device()) - k = torch.randn(B * H_kv, D, S_kv, dtype=dt, device=_device()) - v = torch.randn(B * H_kv, S_kv, D, dtype=dt, device=_device()) - return {"q": q, "k": k, "v": v, "scale": scale, "causal": causal} - - -def _kernel_attention_mha(ctx: dict) -> None: - ctx["out"] = attention_cte( - ctx["q"], ctx["k"], ctx["v"], - scale=ctx["scale"], - causal_mask=ctx["causal"], - ) - xm.mark_step() - - -# ---------- Collectives via nki.collectives ---------- -# -# Each @nki.jit wrapper takes the local tensor + world_size (compile-time -# constant), builds a ReplicaGroup spanning every rank, and dispatches to the -# corresponding nki.collectives primitive. Rank assignment is provided by the -# Neuron PJRT plugin at runtime from the torchrun-set RANK / WORLD_SIZE / -# MASTER_ADDR env vars. - - -# Collectives route through torch_xla's xm.* (see file docstring for why). - - -def _ensure_dist() -> None: - """Initialize torch.distributed with the xla backend if not yet.""" - import torch.distributed as dist - if dist.is_initialized(): - return - import torch_xla.distributed.xla_backend # noqa: F401 registers backend - dist.init_process_group("xla") - - -# NKI collectives require tensors with access patterns of 2D–5D, so the 1D -# byte counts from the op spec are reshaped to (N, 1) for the kernel call. - - -def _prepare_allreduce(op: Op) -> dict: - _ensure_dist() - a = op.args - dt = _resolve(a["dtype"]) - t = torch.randn(a["num_elements"], dtype=dt, device=_device()) - return {"t": t} - - -def _kernel_allreduce(ctx: dict) -> None: - xm.all_reduce(xm.REDUCE_SUM, [ctx["t"]]) - # allreduce is in-place on ctx["t"]; expose it as ctx["out"] so the - # runner's _sync(ctx) readback can force a real device-side wait. - ctx["out"] = ctx["t"] - xm.mark_step() - - -def _prepare_allgather(op: Op) -> dict: - _ensure_dist() - a = op.args - dt = _resolve(a["dtype"]) - t = torch.randn(a["num_elements_per_rank"], dtype=dt, device=_device()) - return {"t": t} - - -def _kernel_allgather(ctx: dict) -> None: - ctx["out"] = xm.all_gather(ctx["t"], dim=0) - xm.mark_step() - - -def _prepare_reduce_scatter(op: Op) -> dict: - _ensure_dist() - a = op.args - ws = int(a["world_size"]) - if a["num_elements"] % ws != 0: - raise UnsupportedOpError( - f"reduce_scatter requires num_elements divisible by world_size " - f"({a['num_elements']} % {ws} != 0)" - ) - dt = _resolve(a["dtype"]) - t = torch.randn(a["num_elements"], dtype=dt, device=_device()) - return {"t": t, "ws": ws} - - -def _kernel_reduce_scatter(ctx: dict) -> None: - ctx["out"] = xm.reduce_scatter( - xm.REDUCE_SUM, ctx["t"], scale=1.0, scatter_dim=0, shard_count=ctx["ws"], - ) - xm.mark_step() - - -def _prepare_alltoall(op: Op) -> dict: - # On Trainium3 with neuronx-cc 2.24 (the only available build in Neuron - # 2.29.0), both lowerings of all_to_all fail: - # - xm.all_to_all with default groups -> `NCC_IVRF100` on the resulting - # `mhlo.all-to-all` HLO with `replica_groups={}`. - # - xm.all_to_all with explicit groups -> compiler reports "CustomCallOp - # unsupported target: mhlo.all_to_all" (no lowering at all). - # - nki.collectives.all_to_all -> runtime asserts ENC_ALG_MESH (broken). - # No user-side workaround is possible until the compiler supports it. We - # surface this as Unsupported so the sweep reports a clear reason instead - # of an opaque ValueError mid-compile. - raise UnsupportedOpError( - "alltoall: neuronx-cc 2.24 (Neuron SDK 2.29) has no working lowering " - "for mhlo.all_to_all on Trainium3. Both xm.all_to_all paths (empty + " - "explicit replica_groups) and nki.collectives.all_to_all fail. Needs " - "a future SDK release. Tracking: aws-neuron/aws-neuron-sdk." - ) - - -def _kernel_alltoall(ctx: dict) -> None: # pragma: no cover unreachable - raise UnsupportedOpError("alltoall not reachable") - - -IMPLS = [ - BackendImpl(op_type="gemm", prepare=_prepare_gemm, kernel=_kernel_gemm), - BackendImpl(op_type="attention_mha", prepare=_prepare_attention_mha, kernel=_kernel_attention_mha), - BackendImpl(op_type="allreduce", prepare=_prepare_allreduce, kernel=_kernel_allreduce), - BackendImpl(op_type="allgather", prepare=_prepare_allgather, kernel=_kernel_allgather), - BackendImpl(op_type="reduce_scatter", prepare=_prepare_reduce_scatter, kernel=_kernel_reduce_scatter), - BackendImpl(op_type="alltoall", prepare=_prepare_alltoall, kernel=_kernel_alltoall), -] diff --git a/operatorx/runners/trainium/backends/nxd.py b/operatorx/runners/trainium/backends/nxd.py deleted file mode 100644 index 68b61aaa49..0000000000 --- a/operatorx/runners/trainium/backends/nxd.py +++ /dev/null @@ -1,165 +0,0 @@ -"""neuronx_distributed (NxD) backend for Trainium collectives. - -NxD wraps torch_xla's xm.* primitives but, before any collective runs, calls -parallel_state.initialize_model_parallel(...) to register a tensor-parallel -ProcessGroup. Every NxD collective then passes that group's replica-mesh -through to xm.* as the `groups=` arg. That populates `replica_groups` on -the emitted HLO instead of leaving it empty, which lets the Neuron PJRT -compile passes attach the `collective_type`/`stream_id` frontend_attributes -the runtime needs to fill in `channel_n` and pick a working algorithm at -ws >= 8. - -The bare-xm path (operatorx torch backend) emits collectives with -`replica_groups={}` and fails at ws >= 8 with `Invalid channel_n(0) for -kangaring algorithm` (see compute-comm-overlap and intranode-collective -docs). -""" -from __future__ import annotations - -import os - -import torch -import torch_neuronx # noqa: F401 registers the neuron device -import torch_xla.core.xla_model as xm - -from operatorx.core import BackendImpl, Op, UnsupportedOpError, lookup_versions - - -def versions() -> dict[str, str]: - return lookup_versions("neuronx_distributed", "torch_neuronx", "torch_xla") - - -_DTYPES = { - "bf16": torch.bfloat16, - "fp16": torch.float16, - "fp32": torch.float32, -} - - -def _resolve(dtype: str) -> torch.dtype: - if dtype not in _DTYPES: - raise UnsupportedOpError(f"nxd doesn't support dtype={dtype!r}") - return _DTYPES[dtype] - - -def _device(): - return xm.xla_device() - - -_NXD_READY = False - - -def _ensure_ready() -> None: - """Bring up torch.distributed (xla backend) + NxD's parallel_state with - a single TP group spanning all ranks.""" - global _NXD_READY - if _NXD_READY: - return - import torch.distributed as dist - if not dist.is_initialized(): - import torch_xla.distributed.xla_backend # noqa: F401 - dist.init_process_group("xla") - - from neuronx_distributed.parallel_layers import parallel_state - if not parallel_state.model_parallel_is_initialized(): - ws = dist.get_world_size() - lnc = int(os.environ.get("NEURON_LOGICAL_NC_CONFIG", "2")) - parallel_state.initialize_model_parallel( - tensor_model_parallel_size=ws, - pipeline_model_parallel_size=1, - context_parallel_size=1, - expert_model_parallel_size=1, - lnc_size=lnc, - ) - _NXD_READY = True - - -def _tp_groups() -> list[list[int]]: - from neuronx_distributed.parallel_layers import parallel_state - return parallel_state.get_tensor_model_parallel_replica_groups() - - -def _prepare_allreduce(op: Op) -> dict: - _ensure_ready() - dt = _resolve(op.args["dtype"]) - t = torch.randn(op.args["num_elements"], dtype=dt, device=_device()) - return {"t": t} - - -def _kernel_allreduce(ctx: dict) -> None: - from neuronx_distributed.parallel_layers.comm import all_reduce - all_reduce(xm.REDUCE_SUM, ctx["t"], groups=_tp_groups()) - ctx["out"] = ctx["t"] - xm.mark_step() - - -def _prepare_allgather(op: Op) -> dict: - _ensure_ready() - dt = _resolve(op.args["dtype"]) - t = torch.randn(op.args["num_elements_per_rank"], dtype=dt, device=_device()) - return {"t": t} - - -def _kernel_allgather(ctx: dict) -> None: - from neuronx_distributed.parallel_layers.comm import all_gather - ctx["out"] = all_gather(ctx["t"], dim=0, groups=_tp_groups()) - xm.mark_step() - - -def _prepare_reduce_scatter(op: Op) -> dict: - _ensure_ready() - ws = int(op.args["world_size"]) - if op.args["num_elements"] % ws != 0: - raise UnsupportedOpError( - f"reduce_scatter requires num_elements divisible by world_size " - f"({op.args['num_elements']} % {ws} != 0)" - ) - dt = _resolve(op.args["dtype"]) - t = torch.randn(op.args["num_elements"], dtype=dt, device=_device()) - return {"t": t, "shard_count": ws} - - -def _kernel_reduce_scatter(ctx: dict) -> None: - from neuronx_distributed.parallel_layers.comm import reduce_scatter - ctx["out"] = reduce_scatter( - reduce_type=xm.REDUCE_SUM, - input=ctx["t"], - scale=1.0, - scatter_dim=0, - shard_count=ctx["shard_count"], - groups=_tp_groups(), - ) - xm.mark_step() - - -# NxD doesn't expose an all_to_all wrapper. Call xm.all_to_all directly with -# the explicit TP group so the HLO has populated replica_groups (same pattern -# nccom-test uses). -def _prepare_alltoall(op: Op) -> dict: - _ensure_ready() - ws = int(op.args["world_size"]) - n_per = op.args["num_elements_per_rank"] - if n_per % ws != 0: - raise UnsupportedOpError( - f"alltoall requires num_elements_per_rank divisible by world_size " - f"({n_per} % {ws} != 0)" - ) - dt = _resolve(op.args["dtype"]) - t = torch.randn(n_per, dtype=dt, device=_device()) - return {"t": t, "split_count": ws} - - -def _kernel_alltoall(ctx: dict) -> None: - ctx["out"] = xm.all_to_all( - ctx["t"], split_dimension=0, concat_dimension=0, - split_count=ctx["split_count"], groups=_tp_groups(), - ) - xm.mark_step() - - -IMPLS = [ - BackendImpl(op_type="allreduce", prepare=_prepare_allreduce, kernel=_kernel_allreduce), - BackendImpl(op_type="allgather", prepare=_prepare_allgather, kernel=_kernel_allgather), - BackendImpl(op_type="reduce_scatter", prepare=_prepare_reduce_scatter, kernel=_kernel_reduce_scatter), - BackendImpl(op_type="alltoall", prepare=_prepare_alltoall, kernel=_kernel_alltoall), -] diff --git a/operatorx/runners/trainium/backends/torch.py b/operatorx/runners/trainium/backends/torch.py deleted file mode 100644 index 692a61843b..0000000000 --- a/operatorx/runners/trainium/backends/torch.py +++ /dev/null @@ -1,205 +0,0 @@ -from __future__ import annotations - -import torch -import torch.nn.functional as F -import torch_neuronx # noqa: F401 registers the neuron backend -import torch_xla.core.xla_model as xm - -from operatorx.core import BackendImpl, Op, UnsupportedOpError, lookup_versions - - -def versions() -> dict[str, str]: - return lookup_versions("torch", "torch_neuronx", "torch_xla") - -_DTYPES = { - "bf16": torch.bfloat16, - "fp16": torch.float16, - "fp32": torch.float32, -} - - -def _resolve(dtype: str) -> torch.dtype: - if dtype not in _DTYPES or _DTYPES[dtype] is None: - raise UnsupportedOpError(f"trainium torch doesn't support dtype={dtype!r}") - return _DTYPES[dtype] - - -def _device(): - return xm.xla_device() - - -def _prepare_gemm(op: Op) -> dict: - a = op.args - if a["dtype_b"] != a["dtype_a"]: - raise UnsupportedOpError("trainium torch gemm requires dtype_a == dtype_b") - dt = _resolve(a["dtype_a"]) - A = torch.randn(a["m"], a["k"], dtype=dt, device=_device()) - B = torch.randn(a["k"], a["n"], dtype=dt, device=_device()) - return {"A": A, "B": B} - - -def _kernel_gemm(ctx: dict) -> None: - ctx["out"] = torch.matmul(ctx["A"], ctx["B"]) - xm.mark_step() - - -def _prepare_attention_mha(op: Op) -> dict: - a = op.args - if a.get("kv_layout", "contig") != "contig": - raise NotImplementedError("trainium torch attention_mha here only handles contiguous KV") - - B, S_q, S_kv = a["batch_size"], a["seq_len_q"], a["seq_len_kv"] - H, H_kv, D = a["num_heads"], a["num_heads_kv"], a["head_dim"] - dt = _resolve(a["dtype_q"]) - causal = a.get("causal", True) and S_q == S_kv - - q = torch.randn(B, H, S_q, D, dtype=dt, device=_device()) - k = torch.randn(B, H_kv, S_kv, D, dtype=dt, device=_device()) - v = torch.randn(B, H_kv, S_kv, D, dtype=dt, device=_device()) - - if H_kv != H: - rep = H // H_kv - k = k.repeat_interleave(rep, dim=1) - v = v.repeat_interleave(rep, dim=1) - - return {"q": q, "k": k, "v": v, "causal": causal} - - -def _kernel_attention_mha(ctx: dict) -> None: - ctx["out"] = F.scaled_dot_product_attention( - ctx["q"], ctx["k"], ctx["v"], is_causal=ctx["causal"] - ) - xm.mark_step() - - - - - - -def _prepare_allreduce(op: Op) -> dict: - a = op.args - dt = _resolve(a["dtype"]) - t = torch.randn(a["num_elements"], dtype=dt, device=_device()) - return {"t": t} - - -def _kernel_allreduce(ctx: dict) -> None: - xm.all_reduce(xm.REDUCE_SUM, [ctx["t"]]) - ctx["out"] = ctx["t"] # for runner._sync readback - xm.mark_step() - - -def _prepare_allgather(op: Op) -> dict: - a = op.args - dt = _resolve(a["dtype"]) - t = torch.randn(a["num_elements_per_rank"], dtype=dt, device=_device()) - return {"t": t} - - -def _kernel_allgather(ctx: dict) -> None: - ctx["out"] = xm.all_gather(ctx["t"], dim=0) - xm.mark_step() - - -def _prepare_reduce_scatter(op: Op) -> dict: - a = op.args - dt = _resolve(a["dtype"]) - ws = a["world_size"] - if a["num_elements"] % ws != 0: - raise UnsupportedOpError( - f"reduce_scatter requires num_elements divisible by world_size ({a['num_elements']} % {ws} != 0)" - ) - t = torch.randn(a["num_elements"], dtype=dt, device=_device()) - return {"t": t, "shard_count": ws} - - -def _kernel_reduce_scatter(ctx: dict) -> None: - ctx["out"] = xm.reduce_scatter( - xm.REDUCE_SUM, ctx["t"], scale=1.0, scatter_dim=0, shard_count=ctx["shard_count"], - ) - xm.mark_step() - - -def _prepare_alltoall(op: Op) -> dict: - a = op.args - dt = _resolve(a["dtype"]) - ws = a["world_size"] - n_per = a["num_elements_per_rank"] - if n_per % ws != 0: - raise UnsupportedOpError( - f"alltoall requires num_elements_per_rank divisible by world_size ({n_per} % {ws} != 0)" - ) - t = torch.randn(n_per, dtype=dt, device=_device()) - return {"t": t, "split_count": ws} - - -def _kernel_alltoall(ctx: dict) -> None: - ctx["out"] = xm.all_to_all( - ctx["t"], split_dimension=0, concat_dimension=0, split_count=ctx["split_count"], - ) - xm.mark_step() - - -# Token routing — reference baseline modeled as a flat all_to_all. -def _moe_routed_buffer(op: Op) -> torch.Tensor: - a = op.args - dt = _resolve(a["dtype"]) - nt, k, h = a["num_tokens"], a["top_k"], a["hidden"] - return torch.randn(nt * k * h, dtype=dt, device=_device()) - - -def _prepare_dispatch(op: Op) -> dict: - return {"t": _moe_routed_buffer(op), "split_count": op.args["world_size"]} - - -def _kernel_dispatch(ctx: dict) -> None: - ctx["out"] = xm.all_to_all( - ctx["t"], split_dimension=0, concat_dimension=0, split_count=ctx["split_count"], - ) - xm.mark_step() - - -def _prepare_combine(op: Op) -> dict: - return _prepare_dispatch(op) - - -def _kernel_combine(ctx: dict) -> None: - _kernel_dispatch(ctx) - - -# MoE forward (compute-only reference): top_k SwiGLU passes with expert-0 weights. -def _prepare_moe_forward(op: Op) -> dict: - a = op.args - dt_act = _resolve(a["dtype_act"]) - dt_w = _resolve(a["dtype_weight"]) - nt, h, im = a["num_tokens"], a["hidden"], a["intermediate"] - x = torch.randn(nt, h, dtype=dt_act, device=_device()) - gate = torch.randn(h, im, dtype=dt_w, device=_device()) - up = torch.randn(h, im, dtype=dt_w, device=_device()) - down = torch.randn(im, h, dtype=dt_w, device=_device()) - return {"x": x, "gate": gate, "up": up, "down": down, "k": a["top_k"]} - - -def _kernel_moe_forward(ctx: dict) -> None: - x = ctx["x"] - out = torch.zeros_like(x) - for _ in range(ctx["k"]): - gate = x @ ctx["gate"] - up = x @ ctx["up"] - mid = F.silu(gate) * up - out = out + mid @ ctx["down"] - ctx["out"] = out - xm.mark_step() - - -IMPLS = [ - BackendImpl(op_type="gemm", prepare=_prepare_gemm, kernel=_kernel_gemm), - BackendImpl(op_type="attention_mha", prepare=_prepare_attention_mha, kernel=_kernel_attention_mha), - BackendImpl(op_type="allreduce", prepare=_prepare_allreduce, kernel=_kernel_allreduce), - BackendImpl(op_type="allgather", prepare=_prepare_allgather, kernel=_kernel_allgather), - BackendImpl(op_type="reduce_scatter", prepare=_prepare_reduce_scatter, kernel=_kernel_reduce_scatter), - BackendImpl(op_type="alltoall", prepare=_prepare_alltoall, kernel=_kernel_alltoall), - BackendImpl(op_type="dispatch", prepare=_prepare_dispatch, kernel=_kernel_dispatch), - BackendImpl(op_type="combine", prepare=_prepare_combine, kernel=_kernel_combine), - BackendImpl(op_type="moe_forward", prepare=_prepare_moe_forward, kernel=_kernel_moe_forward), -] diff --git a/operatorx/runners/trainium/runner.py b/operatorx/runners/trainium/runner.py deleted file mode 100644 index 9d5a74a701..0000000000 --- a/operatorx/runners/trainium/runner.py +++ /dev/null @@ -1,102 +0,0 @@ -from __future__ import annotations - -import os -import time -from importlib import import_module - -# Force LNC=2 (4 logical NCs per Trainium3 chip — each LNC=2 logical NC owns -# one of the chip's 4 HBM banks and consists of 2 paired physical NC-v4 cores -# sharing that bank's address space). This is the unit operatorx counts as -# "world_size=1" on Trainium: one HBM bank + 2 tensor engines + ~64 MiB SBUF. -# Going below this (LNC=1 / single physical NC) is rarely useful since the 2 -# cores in a pair share HBM anyway; going above means crossing into another -# bank which requires CC-Core collectives. -# -# Per AWS Neuron docs, the *compiler* must produce a NEFF matching the -# runtime's LNC mode — otherwise the runtime rejects the binary at load time -# with "Cannot run Neff [...] with Logical Core Size of N". Pass -# `--logical-nc-config=2` to neuronx-cc via NEURON_CC_FLAGS. -os.environ.setdefault("NEURON_LOGICAL_NC_CONFIG", "2") -_cc_flags = os.environ.get("NEURON_CC_FLAGS", "") -if "--logical-nc-config" not in _cc_flags: - os.environ["NEURON_CC_FLAGS"] = (_cc_flags + " --logical-nc-config=2").strip() - -# Default Neuron compile cache lives at /var/tmp/neuron-compile-cache, which -# is often owned by a different user on shared dev boxes. Point it at a -# per-user path under $HOME so writes don't fail with permission denied. -os.environ.setdefault( - "NEURON_COMPILE_CACHE_URL", - os.path.join(os.path.expanduser("~"), ".cache", "neuron-compile-cache"), -) - -import torch_xla.core.xla_model as xm - -from operatorx.core import BackendImpl, Op, Result, UnsupportedOpError - -_BACKENDS = ["torch", "nkilib", "nxd"] -_DISPATCH: dict[tuple[str, str], BackendImpl] = {} -_WARMUP = 3 -_ITERS = 10 -_NUM_BUFFER_SETS = 1 - - -def _load() -> None: - if _DISPATCH: - return - for name in _BACKENDS: - try: - mod = import_module(f"operatorx.runners.trainium.backends.{name}") - except ImportError: - continue - for impl in getattr(mod, "IMPLS", []): - _DISPATCH[(impl.op_type, name)] = impl - - -def _sync(ctx: dict) -> None: - """Force device synchronization. Under PJRT-Neuron + LNC=2, - xm.wait_device_ops() / torch_xla.sync() / xm.mark_step() all return - BEFORE the Neuron device actually finishes — they only flush the queue. - The only reliable way to wait is a CPU readback, so the backend's kernel - function must place its output tensor under ctx["out"] and we tear off - one element from it to force the sync.""" - out = ctx.get("out") - if out is None: - # Last-resort fallback if a backend doesn't expose ctx["out"]. - # This still measures dispatch only — flagged loudly in description. - xm.wait_device_ops() - return - # Single-element readback is enough to force the entire output (and - # everything it depends on) to materialize. Far cheaper than .cpu() on - # the full tensor and dominates per-iter wall time only minimally. - _ = out.flatten()[0].item() - - -def run(op: Op) -> Result: - _load() - impl = _DISPATCH.get((op.type, op.backend)) - if impl is None: - raise UnsupportedOpError( - f"trainium/{op.backend} has no impl for op_type={op.type!r}" - ) - - ctxs = [impl.prepare(op) for _ in range(_NUM_BUFFER_SETS)] - - # Backends that self-benchmark (e.g. nkilib via SpikeModel.benchmark) populate - # ctx["latency_us"] in prepare and the runner trusts that number. - if "latency_us" in ctxs[0]: - return Result(op=op, metrics={"latency_us": ctxs[0]["latency_us"]}) - - for i in range(_WARMUP): - impl.kernel(ctxs[i % _NUM_BUFFER_SETS]) - _sync(ctxs[0]) - - times = [] - for i in range(_ITERS): - ctx = ctxs[i % _NUM_BUFFER_SETS] - t0 = time.perf_counter() - impl.kernel(ctx) - _sync(ctx) - times.append((time.perf_counter() - t0) * 1e6) - - times.sort() - return Result(op=op, metrics={"latency_us": times[_ITERS // 2]}) diff --git a/operatorx/runners/trainium/runtime.py b/operatorx/runners/trainium/runtime.py deleted file mode 100644 index ec5e7c1fd9..0000000000 --- a/operatorx/runners/trainium/runtime.py +++ /dev/null @@ -1,6 +0,0 @@ -"""Trainium-specific runtime probes.""" -from __future__ import annotations - - -def collect() -> dict[str, str]: - return {} # FIXME: neuron-ls -j (driver/firmware) or dpkg -s aws-neuronx-runtime-lib diff --git a/operatorx/runtime.py b/operatorx/runtime.py index b51b52f055..e33c682721 100644 --- a/operatorx/runtime.py +++ b/operatorx/runtime.py @@ -1,13 +1,6 @@ -"""Capture a snapshot of the runtime environment for one smoke-test invocation. - -Generic host info (hostname, importlib.metadata library versions, git sha, -whitelisted env, run id) lives here. Platform-specific driver/runtime probes -live alongside their kernel runners at ``operatorx/runners//runtime.py`` -and expose a ``collect() -> dict[str, str]`` function whose output is merged -into RunInfo.software. Each probe is fail-soft. - -Env-passed (no detection): cluster, container_image, instance_type via -``$OPERATORX_CLUSTER`` / ``$OPERATORX_CONTAINER_IMAGE`` / ``$OPERATORX_INSTANCE_TYPE``. +"""RunInfo snapshot of the runtime: host info here, driver probes in runners//runtime.py +(collect(), fail-soft). cluster/container_image/instance_type come from $OPERATORX_CLUSTER, +$OPERATORX_CONTAINER_IMAGE, $OPERATORX_INSTANCE_TYPE. """ from __future__ import annotations @@ -30,7 +23,7 @@ "CUDA_VISIBLE_DEVICES", "ROCR_VISIBLE_DEVICES", "HIP_VISIBLE_DEVICES", "SLURM_JOB_ID", "SLURM_NODELIST", } -_ENV_PREFIXES = ("NCCL_", "RCCL_", "NEURON_", "XLA_", "JAX_", "TPU_", "PYTORCH_", "OPERATORX_") +_ENV_PREFIXES = ("NCCL_", "RCCL_", "PYTORCH_", "OPERATORX_") def utc_now_iso() -> str: @@ -65,15 +58,12 @@ def _operatorx_version() -> str: def _generic_software() -> dict[str, str]: - # Host-level only: python version + platform driver/runtime probes. - # Per-backend library versions are merged in by main.py via the backend's - # own ``versions()`` function. + # backend library versions are added by main.py return {"python": sys.version.split()[0]} def _platform_software() -> dict[str, str]: - # Each subpackage of operatorx.runners is a platform. The convention is - # that it ships a `runtime.collect() -> dict[str, str]`. + # each operatorx.runners subpackage is a platform with runtime.collect() out: dict[str, str] = {} for info in pkgutil.iter_modules(_runners_pkg.__path__): if not info.ispkg: diff --git a/operatorx/scripts/consolidate_results.py b/operatorx/scripts/consolidate_results.py index e5df032e39..565f72ac4f 100644 --- a/operatorx/scripts/consolidate_results.py +++ b/operatorx/scripts/consolidate_results.py @@ -1,15 +1,7 @@ #!/usr/bin/env python3 -"""Per-chip dedup pass. - -For each results/// directory: - 1. Read every *.json run file. - 2. For each unique (op_type, args, backend) tuple, keep the row from the - latest run (by run.started_at). - 3. Emit a single new run file with id/timestamp=NOW, run metadata copied - from the most-recent input run for that chip. - 4. Delete the original files. - -Run from the project root (the directory containing `results/`). +"""Per-chip dedup of results///*.json: keep the latest row (by run.started_at) +per (op_type, args, backend) in one new run file, then delete the originals. +Run from the directory containing results/. """ from __future__ import annotations @@ -24,7 +16,6 @@ def _row_key(row: dict) -> tuple: op = row["op"] args = op.get("args", {}) - # Deterministic ordering for dict; args are JSON-loaded so plain dicts. args_t = tuple(sorted(args.items(), key=lambda kv: kv[0])) return (op["type"], op.get("backend"), args_t) @@ -45,7 +36,7 @@ def consolidate_chip(chip_dir: Path, dry_run: bool) -> None: if not files: return - # Map key -> (started_at, row); keep latest. + # key -> (started_at, row) latest: dict[tuple, tuple[str, dict]] = {} latest_run_meta: dict | None = None latest_started = "" @@ -66,7 +57,6 @@ def consolidate_chip(chip_dir: Path, dry_run: bool) -> None: if not latest_run_meta: return - # Build the consolidated run record. cluster = latest_run_meta.get("cluster") new_id = _run_id(cluster) now_iso = _utc_now_iso() @@ -87,7 +77,6 @@ def consolidate_chip(chip_dir: Path, dry_run: bool) -> None: return out_path.write_text(json.dumps(out, indent=2)) - # Remove the originals (excluding the freshly-written one). for f in files: if f != out_path: f.unlink() diff --git a/operatorx/scripts/inferencex_testlist/README.md b/operatorx/scripts/inferencex_testlist/README.md deleted file mode 100644 index 33b31e0d43..0000000000 --- a/operatorx/scripts/inferencex_testlist/README.md +++ /dev/null @@ -1,84 +0,0 @@ -# inferencex_testlist - -Derive `operatorx/testlists/*.json` from the InferenceX matrix configs. - -## Approach - -This is a purely static enumeration with no GPU time or tracing. - -For each `(model, framework, precision, runner, TP, EP, dp-attn, ISL, OSL, -conc, spec-decoding)` tuple in the InferenceX matrix, we: - -1. Look up the model's HuggingFace `config.json` (locally cached). -2. Resolve the architecture (DeepSeek-MLA / GLM-MLA / Kimi-MLA / MiniMax-GQA / - GPT-OSS-GQA / Qwen3.5-MoE). -3. Emit canonical OperatorX op shapes that the workload would touch: - - **GEMM**: attention projections, MLP gate/up/down (for dense layers), - sharded by TP. - - **Attention**: MHA or MLA, parameterised by TP head-shard and DP-attn - batch-shard. Prefill uses the full ISL. Decode uses `mtp_factor` query tokens - and `ISL + OSL/2` KV length. - - **MoE**: `moe_forward` + `topk_routing` at prefill (`conc*ISL` tokens) - and decode (`conc*mtp_factor` tokens) regimes. - - **Collectives**: AllReduce (plain TP), AllGather + ReduceScatter - (DP-attn), dispatch/combine (EP > 1). -4. Aggregate across rows, dedupe by canonical `(op_type, args)`, and write - testlist JSONs. Existing entries are unioned with new ones unless - `--no-merge-existing` is passed. - -Matrix expansion is delegated to InferenceX's own -`python -m infx.matrix.generate full-sweep` so the enumerator -stays in sync with InferenceX semantics automatically. - -## Known limitations - -Static enumeration captures canonical shapes, not engine-specific transformations: - -- Fused QKV / fused gate-up GEMMs are emitted as separate canonical projections. -- MLA "absorbed" decode form (one big GEMM instead of separate Q/KV ups) is not - modeled. We emit the naive MLA decomposition. -- Speculative-decoding draft-model GEMMs (EAGLE-style drafters) are not - enumerated separately. MTP is modeled by inflating the decode token count. -- Qwen3.5's hybrid linear/full attention layers are modeled as full-attention - only (linear-attention layers TODO). -- Engine-version drift is not detected. - -If you need engine-faithful shapes, plug in the empirical-trace fingerprint -path on top (separately, optional). - -## Run - -```bash -# From operatorx/ in a clean venv with pydantic + pyyaml installed: -pip install pydantic pyyaml - -# Enumerate against both NVIDIA and AMD matrices: -python -m scripts.inferencex_testlist enumerate \ - --inferencex /path/to/InferenceX \ - --config configs/nvidia-master.yaml \ - --config configs/amd-master.yaml \ - --models-dir /models \ - --models-dir /scratch/fsw/models \ - --out testlists/ -``` - -`--inferencex` accepts either the repository root or its `inferencex-e2e/` directory. Config paths are relative to that end-to-end project directory; older checkouts with a flat layout remain supported. - -Each `--models-dir` option adds another location to search for `/config.json`. -HuggingFace cache (`~/.cache/huggingface/hub/models----/snapshots/`) -is searched automatically. - -Use `--no-merge-existing` to overwrite the testlists rather than union. - -## Adding a new architecture - -When InferenceX adopts a new model family, add a `_build_(cfg, name)` -function in `models.py` and wire it into `_BUILDERS`. The function should -populate `AttentionArch` (mha vs mla, sharded dims) and `MoeArch` -(num_experts, top_k, intermediate, n_shared_experts, optional dense MLP). -The rest of the enumerator picks up from there. - -## Adding a new parameterisation - -No action is needed. New `(TP, EP, ISL, OSL, conc, spec-decoding)` tuples added to -`configs/*.yaml` are picked up automatically on the next run. diff --git a/operatorx/scripts/inferencex_testlist/__init__.py b/operatorx/scripts/inferencex_testlist/__init__.py deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/operatorx/scripts/inferencex_testlist/__main__.py b/operatorx/scripts/inferencex_testlist/__main__.py deleted file mode 100644 index 5bb688fcf3..0000000000 --- a/operatorx/scripts/inferencex_testlist/__main__.py +++ /dev/null @@ -1,108 +0,0 @@ -"""CLI: enumerate canonical OperatorX ops from InferenceX matrix configs. - -Usage: - python -m scripts.inferencex_testlist enumerate \ - --inferencex /path/to/InferenceX \ - --config configs/nvidia-master.yaml \ - --config configs/amd-master.yaml \ - --models-dir /models \ - --out testlists/ -""" -from __future__ import annotations - -import argparse -import sys -from collections import defaultdict -from typing import Iterable - -from .emit import write_testlists -from .enumerate import ops_for_row -from .matrix import WorkloadRow, load_rows -from .models import Arch, load_arch - - -def _dedupe_rows(rows: Iterable[WorkloadRow]) -> list[WorkloadRow]: - """Drop rows whose op set would be identical to one we've already enumerated.""" - seen = set() - out: list[WorkloadRow] = [] - for r in rows: - key = ( - r.model, r.precision, r.framework, r.tp, r.ep, r.dp_attn, - r.isl, r.osl, r.conc, r.spec_decoding, - ) - if key in seen: - continue - seen.add(key) - out.append(r) - return out - - -def cmd_enumerate(args: argparse.Namespace) -> int: - extra_dirs = tuple(args.models_dir) if args.models_dir else () - - rows = [] - for cfg in args.config: - rows.extend(load_rows(args.inferencex, [cfg])) - print(f"[load] {len(rows)} matrix rows from {len(args.config)} configs", file=sys.stderr) - - rows = _dedupe_rows(rows) - print(f"[dedupe] {len(rows)} unique rows after collapsing concurrency repeats", file=sys.stderr) - - # Cache per-model arch lookups; we'll hit the same HF config many times. - arch_cache: dict[str, Arch] = {} - missing_models: dict[str, list[WorkloadRow]] = defaultdict(list) - - all_ops: list = [] - skipped = 0 - for row in rows: - if row.model not in arch_cache: - try: - arch_cache[row.model] = load_arch(row.model, extra_dirs=extra_dirs) - except FileNotFoundError as e: - missing_models[row.model].append(row) - skipped += 1 - continue - except Exception as e: - print(f"[warn] could not load arch for {row.model!r}: {e}", file=sys.stderr) - missing_models[row.model].append(row) - skipped += 1 - continue - try: - all_ops.extend(ops_for_row(row, arch_cache[row.model])) - except Exception as e: - print(f"[warn] enumeration failed for {row.exp_name} {row.model} tp{row.tp} conc{row.conc}: {e}", - file=sys.stderr) - skipped += 1 - - if missing_models: - print(f"[warn] {len(missing_models)} models had no local config; skipped {skipped} rows", file=sys.stderr) - for m, rs in list(missing_models.items())[:10]: - print(f" {m!r}: {len(rs)} rows", file=sys.stderr) - - counts = write_testlists(all_ops, args.out, merge_with_existing=args.merge_existing) - for fname, n in sorted(counts.items()): - print(f"[emit] {fname}.json: {n} entries", file=sys.stderr) - return 0 - - -def main(argv: list[str] | None = None) -> int: - parser = argparse.ArgumentParser(prog="python -m scripts.inferencex_testlist") - sub = parser.add_subparsers(dest="cmd", required=True) - - p_enum = sub.add_parser("enumerate", help="Build testlists from InferenceX configs") - p_enum.add_argument("--inferencex", required=True, help="Path to InferenceX repo root") - p_enum.add_argument("--config", required=True, action="append", - help="InferenceX master config path (relative to the end-to-end project directory). Repeatable.") - p_enum.add_argument("--models-dir", action="append", default=[], - help="Local directory with model config.json subdirs. Repeatable.") - p_enum.add_argument("--out", required=True, help="Output dir (operatorx/testlists/)") - p_enum.add_argument("--no-merge-existing", dest="merge_existing", action="store_false", - help="Overwrite testlist files rather than unioning with existing entries.") - p_enum.set_defaults(merge_existing=True, func=cmd_enumerate) - - args = parser.parse_args(argv) - return args.func(args) - - -if __name__ == "__main__": - raise SystemExit(main()) diff --git a/operatorx/scripts/inferencex_testlist/dtypes.py b/operatorx/scripts/inferencex_testlist/dtypes.py deleted file mode 100644 index 4f0d806449..0000000000 --- a/operatorx/scripts/inferencex_testlist/dtypes.py +++ /dev/null @@ -1,183 +0,0 @@ -"""Per-op-class dtype resolver. - -Real InferenceX invocations use different (weight, activation) dtype pairs -for attention/dense/MoE GEMMs within the same workload — e.g. DSv4 Instruct -ships FP4 MoE experts + FP8 attention/dense; AMD's gpt-oss `-w-mxfp4-a-fp8` -checkpoint runs W4A8 MoE. The single `precision` tag in InferenceX is too -coarse to capture this, so we keep a lookup table keyed on -(precision, framework, runner-family, model-family) → per-op-class dtypes. - -The table was derived by reading benchmarks/single_node/*.sh launch scripts -plus the SGLang/TRT-LLM cookbooks for each model. When a new (framework, -precision) tuple is added to InferenceX, add an entry here. -""" -from __future__ import annotations - -from dataclasses import dataclass -from typing import Literal - -from .matrix import WorkloadRow - - -@dataclass(frozen=True) -class OpDtype: - """A single op's dtype triple.""" - weight: str # storage dtype of the weight tensor - activation: str # dtype of the activation tensor input to the GEMM - output: str # dtype of the GEMM output (usually bf16) - - -@dataclass(frozen=True) -class RowDtypes: - """Per-op-class dtypes for one workload row.""" - attn: OpDtype # attention block GEMMs (q/k/v/o projections) - dense: OpDtype # dense MLP GEMMs (when first_k_dense_replace > 0) - moe: OpDtype # MoE expert MLP GEMMs - routing: str # topk_routing op dtype (always high precision) - kv: str # KV cache storage dtype - - -# --------------------------------------------------------------------------- -# Helpers -# --------------------------------------------------------------------------- - - -_BF16 = "bf16" -_FP16 = "fp16" - - -def _runner_family(runner: str) -> str: - """Reduce a specific runner tag (e.g. "cluster:b200-nscale", "b300-p1") to a - GPU family ("b200", "b300", "h200", "h100", "gb200", "gb300", "mi300x", - "mi325x", "mi355x").""" - r = runner.lower() - # order matters: longest match first - for fam in ("b300", "b200", "gb300", "gb200", "h200", "h100", - "mi355x", "mi325x", "mi300x", "trn", "tpu"): - if fam in r: - return fam - return r - - -def _is_blackwell(runner: str) -> bool: - return _runner_family(runner) in ("b200", "b300", "gb200", "gb300") - - -def _is_hopper(runner: str) -> bool: - return _runner_family(runner) in ("h100", "h200") - - -def _is_amd(runner: str) -> bool: - return _runner_family(runner).startswith("mi") - - -# --------------------------------------------------------------------------- -# Resolver -# --------------------------------------------------------------------------- - - -def _resolve_attn_dense(precision: str, runner: str, model_prefix: str) -> OpDtype: - """Attention + dense MLP dtypes. These follow the model's "non-MoE" quant - regime, which for mixed-precision checkpoints (DSv4) is FP8 even when MoE - is FP4.""" - if precision == "bf16": - return OpDtype(_BF16, _BF16, _BF16) - if precision == "fp16": - return OpDtype(_FP16, _FP16, _FP16) - if precision == "fp8": - # FP8 block-128 weight + per-token dynamic FP8 activation - return OpDtype("fp8", "fp8", _BF16) - if precision in ("fp4", "nvfp4", "mxfp4"): - if _is_amd(runner): - # AMD: only MoE layers are quantized to MXFP4 weights. From SGLang - # docs (quark_int4fp8_moe): "Other layers (e.g. projections in the - # attention layers) have their weights quantized online to float8 - # directly." So attention/dense GEMMs run W=fp8, A=fp8. - return OpDtype("fp8", "fp8", _BF16) - if model_prefix == "dsv4": - # DSv4 Instruct: attention/dense stay FP8 even on FP4 row - return OpDtype("fp8", "fp8", _BF16) - if _is_blackwell(runner): - # NVFP4 weight + NVFP4 activation (Blackwell native FP4 tensor cores). - # Verified against SGLang's CompressedTensorsW4A4 / modelopt_fp4 path. - return OpDtype("nvfp4", "nvfp4", _BF16) - # Hopper FP4 uses W4A16 (marlin / cutlass W4A16). Verified against - # FlashInfer SM90 cutlass MXFP4 MoE backend (PR #24816, explicit W4A16). - return OpDtype("mxfp4", _BF16, _BF16) - if precision == "int4": - # Weight-only quantization (e.g. Kimi K2.5 int4 compressed-tensors). - # SGLang's CompressedTensorsWNA16 path: int4 weight × bf16 activation. - return OpDtype("int4", _BF16, _BF16) - if precision == "int8": - return OpDtype("int8", "int8", _BF16) - raise ValueError(f"unknown precision {precision!r}") - - -def _resolve_moe(precision: str, runner: str, framework: str, model_prefix: str) -> OpDtype: - """MoE expert MLP dtypes. Depends on the moe-runner-backend chosen by the - launch script for each (framework, precision, hardware) combination.""" - if precision == "bf16": - return OpDtype(_BF16, _BF16, _BF16) - if precision == "fp16": - return OpDtype(_FP16, _FP16, _FP16) - if precision == "fp8": - return OpDtype("fp8", "fp8", _BF16) - if precision in ("fp4", "nvfp4", "mxfp4"): - if _is_amd(runner): - # AMD MoE runs as W=mxfp4, A=fp8 across MI300x/MI325x/MI355x - # (matches the published `amd/*-w-mxfp4-a-fp8` checkpoint contract - # for both vLLM and ATOM frameworks). - return OpDtype("mxfp4", "fp8", _BF16) - if model_prefix == "dsv4": - # DSv4 Instruct ships MXFP4 MoE experts. On Blackwell SGLang's - # flashinfer_mxfp4 MoE backend runs FP4 weight × NVFP4 activation. - # On Hopper it falls back to W4A16 marlin. - if _is_hopper(runner): - return OpDtype("mxfp4", _BF16, _BF16) - return OpDtype("mxfp4", "nvfp4", _BF16) - # Standard FP4 rows: NVFP4 weight from modelopt_fp4 quantization. - if _is_blackwell(runner): - return OpDtype("nvfp4", "nvfp4", _BF16) - # Hopper FP4 marlin: W4A16 - return OpDtype("mxfp4", _BF16, _BF16) - if precision == "int4": - return OpDtype("int4", _BF16, _BF16) - if precision == "int8": - return OpDtype("int8", "int8", _BF16) - raise ValueError(f"unknown precision {precision!r}") - - -def _resolve_kv(precision: str, runner: str, model_prefix: str) -> str: - """KV cache storage dtype. - - Most quantized configurations (FP4/FP8) keep the KV cache in FP8 to save - bandwidth + memory; BF16/FP16 paths leave it native. InferenceX scripts - consistently pass `--kv-cache-dtype fp8_e4m3` on FP4 launches. - """ - if precision in ("bf16", "fp16"): - return precision - if precision == "int4": - # Weight-only quant; KV cache is BF16 - return _BF16 - # fp8/fp4/nvfp4/mxfp4 → fp8 KV cache - return "fp8" - - -def _resolve_routing(precision: str) -> str: - """topk_routing scoring is always done in a high-precision dtype because - softmax/sigmoid need numerical headroom. SGLang/TRT-LLM run it in BF16 - (or FP32 for the gate matmul, then BF16 for softmax).""" - if precision == "fp16": - return _FP16 - return _BF16 - - -def resolve_dtypes(row: WorkloadRow) -> RowDtypes: - """Resolve per-op-class dtypes for one InferenceX matrix row.""" - return RowDtypes( - attn=_resolve_attn_dense(row.precision, row.runner, row.model_prefix), - dense=_resolve_attn_dense(row.precision, row.runner, row.model_prefix), - moe=_resolve_moe(row.precision, row.runner, row.framework, row.model_prefix), - routing=_resolve_routing(row.precision), - kv=_resolve_kv(row.precision, row.runner, row.model_prefix), - ) diff --git a/operatorx/scripts/inferencex_testlist/emit.py b/operatorx/scripts/inferencex_testlist/emit.py deleted file mode 100644 index 7b20548271..0000000000 --- a/operatorx/scripts/inferencex_testlist/emit.py +++ /dev/null @@ -1,84 +0,0 @@ -"""Aggregate per-row op enumerations into deduped testlist JSON files.""" -from __future__ import annotations - -import json -import os -from collections import defaultdict -from typing import Any - -from .enumerate import OpTriple - - -# Which top-level op_type lands in which testlist file. -_OP_TO_TESTLIST = { - "gemm": "gemm", - "attention_mha": "attention", - "attention_mla": "attention", - "topk_routing": "moe", - "moe_forward": "moe", - "allreduce": "collectives", - "allgather": "collectives", - "reduce_scatter": "collectives", - "alltoall": "collectives", - "dispatch": "collectives", - "combine": "collectives", - "kvcache_swap": "memory", -} - - -def _dedupe_key(op_type: str, args: dict[str, Any]) -> tuple: - """Treat entries with the same op_type + args as the same testlist entry.""" - return (op_type, tuple(sorted(args.items()))) - - -def write_testlists( - ops_with_names: list[OpTriple], out_dir: str, merge_with_existing: bool = True -) -> dict[str, int]: - """Write deduped testlists. Returns counts per file. - - If `merge_with_existing` is True and the testlist files already exist, - union the new entries into them so we don't drop hand-curated shapes. - """ - by_file: dict[str, list[dict[str, Any]]] = defaultdict(list) - seen: dict[str, set[tuple]] = defaultdict(set) - - # Preload existing entries so we union rather than overwrite. - if merge_with_existing: - for fname in set(_OP_TO_TESTLIST.values()): - path = os.path.join(out_dir, f"{fname}.json") - if os.path.isfile(path): - try: - existing = json.load(open(path)) - except Exception: - existing = [] - for entry in existing: - if "type" not in entry or "args" not in entry: - continue - key = _dedupe_key(entry["type"], entry["args"]) - if key in seen[fname]: - continue - seen[fname].add(key) - by_file[fname].append(entry) - - for op_type, args, name in ops_with_names: - target = _OP_TO_TESTLIST.get(op_type) - if target is None: - continue - key = _dedupe_key(op_type, args) - if key in seen[target]: - continue - seen[target].add(key) - entry: dict[str, Any] = {"type": op_type, "args": args} - if name: - entry["name"] = name - by_file[target].append(entry) - - os.makedirs(out_dir, exist_ok=True) - counts: dict[str, int] = {} - for fname, entries in by_file.items(): - path = os.path.join(out_dir, f"{fname}.json") - with open(path, "w") as f: - json.dump(entries, f, indent=2) - f.write("\n") - counts[fname] = len(entries) - return counts diff --git a/operatorx/scripts/inferencex_testlist/enumerate.py b/operatorx/scripts/inferencex_testlist/enumerate.py deleted file mode 100644 index cbeb84382f..0000000000 --- a/operatorx/scripts/inferencex_testlist/enumerate.py +++ /dev/null @@ -1,460 +0,0 @@ -"""Per-row enumeration of canonical OperatorX ops. - -Input: a `WorkloadRow` (one InferenceX matrix entry) and the model `Arch`. -Output: a list of `(op_type, args_dict, name)` triples that can be folded -into the testlist JSONs. -""" -from __future__ import annotations - -from typing import Any, Iterable - -from .dtypes import OpDtype, RowDtypes, resolve_dtypes -from .matrix import WorkloadRow -from .models import Arch, AttentionArch, MoeArch -from .parallelism import Parallelism - - -OpTriple = tuple[str, dict[str, Any], str] - - -# Existing testlists name entries by model (e.g. "dsv3", "llama3-8b") for -# layer-shaped ops (attention/moe) and leave `name` empty for shape-only ops -# (gemm/collectives/memory). That convention drives dedupe behavior: identical -# (m,n,k,dtype) GEMMs that arise from different models should collapse to one -# entry in gemm.json. -_NAMED_OP_TYPES = {"attention_mha", "attention_mla", "moe_forward", "topk_routing"} - - -def _name(op_type: str, model_prefix: str) -> str: - return model_prefix if op_type in _NAMED_OP_TYPES else "" - - -# --------------------------------------------------------------------------- -# Phase helpers -# --------------------------------------------------------------------------- - - -def _mtp_factor(row: WorkloadRow) -> int: - """How many draft tokens decode evaluates per step. 1 if no spec-decoding. - - InferenceX scripts uniformly run `--speculative-num-steps 1` for MTP and - most draft-model setups, so a decode step verifies 1 token + 1 draft. - """ - if row.spec_decoding in ("mtp", "draft_model"): - return 2 - return 1 - - -# Per-kernel chunked-prefill cap. The InferenceX launch scripts pass -# --chunked-prefill-size in the 8192..32768 range; we use the upper bound as a -# realistic per-kernel ceiling. Without this cap, multinode disagg rows -# (conc=21504, ISL=8192) would emit 176M-token MoE ops that no kernel can run — -# the framework chunks them into ~5000 waves of `chunked-prefill-size` each. -CHUNKED_PREFILL_MAX = 32768 - - -def _prefill_tokens(row: WorkloadRow, par: Parallelism) -> int: - """Per-kernel tokens-per-step at prefill, per attention rank. - - Prefill chunking + DP-attn sharding + multinode disagg per-worker split: - 1. Each prefill worker handles `conc * ISL / num_workers` total tokens. - 2. The engine streams these in waves of `--chunked-prefill-size` - (capped at CHUNKED_PREFILL_MAX). One kernel call sees one wave. - 3. Under DP-attn each rank handles `1/attn_dp` of the wave (token-axis - sharding); without DP-attn each rank sees the full wave (heads-axis - sharding handles parallelism instead). - """ - per_worker_total = (row.conc * row.isl) // max(1, row.num_workers) - capped = min(per_worker_total, CHUNKED_PREFILL_MAX) - return max(1, capped // par.attn_dp) - - -def _decode_tokens(row: WorkloadRow, par: Parallelism) -> int: - """Per-kernel tokens-per-step at decode, per attention rank. - - Multinode disagg: each decode worker handles `1/num_workers` of total - concurrent requests. DP-attn: each rank handles `1/attn_dp` of the batch. - """ - per_worker = (row.conc * _mtp_factor(row)) // max(1, row.num_workers) - return max(1, per_worker // par.attn_dp) - - -def _moe_tokens_prefill(row: WorkloadRow) -> int: - """Per-kernel tokens visible to the MoE block at prefill. - - MoE runs on the *full* batch regardless of DP-attn (AllGather brings the - sharded attention output back to full batch before dispatch). For multinode - disagg, the matrix `conc` is the total target throughput; each prefill - worker sees `conc / num_workers` worth of requests. The per-kernel batch is - further bounded by `--chunked-prefill-size` (typically 32768). Floored to - 1 so int-division rounding at low conc still emits a benchable shape. - """ - per_worker_total = (row.conc * row.isl) // max(1, row.num_workers) - return max(1, min(per_worker_total, CHUNKED_PREFILL_MAX)) - - -def _moe_tokens_decode(row: WorkloadRow) -> int: - """Per-kernel tokens visible to the MoE block at decode. - - Single-node: `conc * mtp_factor` (one MTP-extended decode step on the full - batch). Multinode disagg: divided by `num_workers` (each decode worker - handles its share of the total concurrency). Floored to 1 so - `(conc=1, mtp=1, num_workers=2..5)` still emits a benchable shape rather - than rounding to 0. - """ - return max(1, (row.conc * _mtp_factor(row)) // max(1, row.num_workers)) - - -# --------------------------------------------------------------------------- -# GEMM enumeration -# --------------------------------------------------------------------------- - - -def _gemm( - M: int, N: int, K: int, row: WorkloadRow, dt: OpDtype, - role: str, shard: int, bias: bool = False, -) -> OpTriple: - """Emit a canonical GEMM op. - - `role` is a short tag identifying which projection this is (e.g. - "q_proj", "mla_q_b", "mlp_gate_up"). `shard` records the TP shard factor - applied to the N-axis weight of this GEMM so the name field captures the - sharding context (and identical (M,N,K,dtype) GEMMs from different - shardings stay distinguishable when wanted).""" - name = f"{row.model_prefix}-{role}" + (f"-tp{shard}" if shard > 1 else "") - args: dict[str, Any] = { - "m": int(M), - "n": int(N), - "k": int(K), - "dtype_a": dt.activation, - "dtype_b": dt.weight, - "dtype_out": dt.output, - } - if bias: - args["bias"] = True - return ("gemm", args, name) - - -def _attn_block_gemms( - M: int, arch: Arch, par: Parallelism, row: WorkloadRow, phase: str, dt: OpDtype -) -> Iterable[OpTriple]: - """Emit projection GEMMs for one attention block. - - Conventions: - - MHA: q_proj [H, num_heads*head_dim], k_proj/v_proj [H, num_kv_heads*head_dim], - o_proj [num_heads*head_dim, H]. With TP=N, heads are sharded N-way. - - MLA (DeepSeek/Kimi/GLM): q_a_proj [H, q_lora_rank], q_b_proj [q_lora_rank, num_heads*(qk_nope+qk_rope)], - kv_a_proj_with_mqa [H, kv_lora_rank+qk_rope], kv_b_proj [kv_lora_rank, num_heads*(qk_nope+v_head_dim)], - o_proj [num_heads*v_head_dim, H]. - """ - H = arch.hidden_size - a = arch.attention - tp = par.attn_tp - - # GPT-OSS has `attention_bias=True` in its HF config; QKV/O projections - # all carry biases. Other InferenceX models leave attention_bias=False. - attn_bias = (row.model_prefix == "gptoss") - - if a.kind == "mha": - n_q = a.num_heads * a.head_dim - n_kv = a.num_kv_heads * a.head_dim - kv_shard = max(1, min(tp, a.num_kv_heads)) - # SGLang/vLLM both fuse Q+K+V into one GEMM with width - # (num_heads + 2*num_kv_heads)*head_dim. We emit that fused shape since - # it's what the kernel actually runs. - fused_qkv_n = (n_q + 2 * n_kv) // tp if tp == kv_shard else None - if fused_qkv_n is not None: - yield _gemm(M, fused_qkv_n, H, row, dt, f"attn_qkv_proj_{phase}", tp, bias=attn_bias) - else: - yield _gemm(M, n_q // tp, H, row, dt, f"attn_q_proj_{phase}", tp, bias=attn_bias) - yield _gemm(M, n_kv // kv_shard, H, row, dt, f"attn_k_proj_{phase}", kv_shard, bias=attn_bias) - yield _gemm(M, n_kv // kv_shard, H, row, dt, f"attn_v_proj_{phase}", kv_shard, bias=attn_bias) - yield _gemm(M, H, n_q // tp, row, dt, f"attn_o_proj_{phase}", tp, bias=attn_bias) - return - - # MLA path: emit canonical 5 projections (q_a, q_b, kv_a, kv_b, o). - q_lora = a.q_lora_rank - kv_lora = a.kv_lora_rank - head_qk = a.qk_nope_head_dim + a.qk_rope_head_dim - yield _gemm(M, q_lora, H, row, dt, f"mla_q_a_proj_{phase}", 1) - yield _gemm(M, (a.num_heads * head_qk) // tp, q_lora, row, dt, f"mla_q_b_proj_{phase}", tp) - yield _gemm(M, kv_lora + a.qk_rope_head_dim, H, row, dt, f"mla_kv_a_proj_{phase}", 1) - yield _gemm(M, (a.num_heads * (a.qk_nope_head_dim + a.v_head_dim)) // tp, kv_lora, row, dt, - f"mla_kv_b_proj_{phase}", tp) - yield _gemm(M, H, (a.num_heads * a.v_head_dim) // tp, row, dt, f"mla_o_proj_{phase}", tp) - - -def _dense_mlp_gemms( - M: int, arch: Arch, par: Parallelism, row: WorkloadRow, phase: str, dt: OpDtype -) -> Iterable[OpTriple]: - if arch.moe.dense_intermediate_size <= 0: - return - H = arch.hidden_size - interm = arch.moe.dense_intermediate_size - tp = par.tp - # SGLang/vLLM fuse gate + up into one GEMM with width 2*intermediate, then - # apply silu_and_mul. Emit the fused shape — that's what the kernel runs. - yield _gemm(M, (2 * interm) // tp, H, row, dt, f"mlp_gate_up_proj_{phase}", tp) - yield _gemm(M, H, interm // tp, row, dt, f"mlp_down_proj_{phase}", tp) - - -# --------------------------------------------------------------------------- -# Attention op enumeration -# --------------------------------------------------------------------------- - - -def _attention_ops( - arch: Arch, par: Parallelism, row: WorkloadRow, phase: str, dts: RowDtypes -) -> Iterable[OpTriple]: - a = arch.attention - if phase == "prefill": - # Treat as one big extend step at full ISL (chunked prefill produces - # the same canonical shape modulo the chunk size, which we don't - # model explicitly here). - s_q = row.isl - s_kv = row.isl - batch = max(1, row.conc // par.attn_dp) - else: - s_q = _mtp_factor(row) - # Sweep S_kv over representative points to cover the decode trajectory. - # We yield only the worst case (full prefix + half output); other - # points get emitted at the per-row level by callers if they want. - s_kv = row.isl + row.osl // 2 - batch = max(1, row.conc // par.attn_dp) - - out_dtype = dts.attn.output - q_dtype = dts.attn.activation - kvd = dts.kv - - if a.kind == "mha": - yield ( - "attention_mha", - { - "batch_size": int(batch), - "seq_len_q": int(s_q), - "seq_len_kv": int(s_kv), - "num_heads": int(a.num_heads // par.attn_tp), - "num_heads_kv": int(max(1, a.num_kv_heads // par.attn_tp)), - "head_dim": int(a.head_dim), - "dtype_q": q_dtype, - "dtype_k": kvd, - "dtype_v": kvd, - "dtype_o": out_dtype, - "causal": True, - **({"sliding_window": int(a.sliding_window)} if a.sliding_window else {}), - }, - _name("attention_mha", row.model_prefix), - ) - return - - # MLA - yield ( - "attention_mla", - { - "batch_size": int(batch), - "seq_len_q": int(s_q), - "seq_len_kv": int(s_kv), - "num_heads": int(a.num_heads // par.attn_tp), - "head_dim_qk_nope": int(a.qk_nope_head_dim), - "head_dim_qk_rope": int(a.qk_rope_head_dim), - "head_dim_v": int(a.v_head_dim), - "kv_lora_rank": int(a.kv_lora_rank), - "dtype_q": q_dtype, - "dtype_kv": kvd, - "dtype_o": out_dtype, - "rope": True, - "causal": True, - }, - _name("attention_mla", row.model_prefix), - ) - - -# --------------------------------------------------------------------------- -# MoE op enumeration -# --------------------------------------------------------------------------- - - -def _shared_tp(row: WorkloadRow, par: Parallelism) -> int: - """Effective TP shard for the SHARED expert MLP under SGLang. - - SGLang's behavior (`deepseek_v2.py` / `qwen2_moe.py`): - - DSv4 sets `disable_shared_experts_fusion=True`. The shared expert is a - separate MLP and runs with `tp_size=1` so its (clamped SwiGLU) - activation can be kept distinct from the routed experts' (un-clamped). - - When `--moe-a2a-backend deepep` is selected (full EP path), the shared - expert is either DeepEP-fused as one extra slot per EP rank, or — for - the non-fused path — explicitly created with `tp_rank=0, tp_size=1`. - Either way, weights are replicated per rank: `shared_tp = 1`. - - Otherwise the shared expert is folded into the routed FusedMoE kernel - as an extra expert slot, and its weights are TP-sharded along - intermediate the same way routed weights are: `shared_tp = moe_tp`. - """ - if row.model_prefix == "dsv4": - return 1 - if par.moe_ep > 1: - return 1 - return par.moe_tp - - -def _moe_ops( - arch: Arch, par: Parallelism, row: WorkloadRow, phase: str, tokens: int, dts: RowDtypes -) -> Iterable[OpTriple]: - moe = arch.moe - if moe.num_experts <= 0: - return - - H = arch.hidden_size - yield ( - "topk_routing", - { - "num_tokens": int(tokens), - "num_experts": int(moe.num_experts), - "top_k": int(moe.num_experts_per_tok), - "dtype": dts.routing, - }, - _name("topk_routing", row.model_prefix), - ) - yield ( - "moe_forward", - { - "num_tokens": int(tokens), - "hidden": int(H), - "intermediate": int(moe.moe_intermediate_size), - "num_experts": int(moe.num_experts), - "top_k": int(moe.num_experts_per_tok), - "dtype_act": dts.moe.activation, - "dtype_weight": dts.moe.weight, - "world_size": int(par.world_size), - "expert_parallel_size": int(par.moe_ep), - "routed_tensor_parallel_size": int(par.moe_tp), - "shared_tensor_parallel_size": int(_shared_tp(row, par)), - "n_shared_experts": int(moe.n_shared_experts), - }, - _name("moe_forward", row.model_prefix), - ) - - -# --------------------------------------------------------------------------- -# Collective enumeration -# --------------------------------------------------------------------------- - - -def _collective_ops( - arch: Arch, par: Parallelism, row: WorkloadRow, phase: str, tokens: int, dts: RowDtypes -) -> Iterable[OpTriple]: - if par.world_size <= 1: - return - - H = arch.hidden_size - moe = arch.moe - # Collectives carry the MoE-layer activations, which are exchanged in the - # MoE op's activation dtype (post-attention output is dequantized to bf16 - # before MoE input). Use the attention output dtype as the comm dtype since - # that's what's in flight during AG/RS/AR. For dispatch/combine the MoE - # activation dtype applies. - comm_dtype = dts.attn.output - moe_comm_dtype = dts.moe.activation - - if par.dp_attn and par.attn_dp > 1: - # Under DP-attn, attention output is sharded by tokens; an AllGather - # along the token axis brings activations back to full batch before - # MoE dispatch. Conversely, after MoE combine, a ReduceScatter - # re-shards by token. SGLang's deepseek path does this. - yield ( - "allgather", - { - "num_elements_per_rank": int(tokens * H), - "dtype": comm_dtype, - "world_size": int(par.attn_dp), - }, - _name("allgather", row.model_prefix), - ) - yield ( - "reduce_scatter", - { - "num_elements": int(tokens * par.attn_dp * H), - "dtype": comm_dtype, - "world_size": int(par.attn_dp), - }, - _name("reduce_scatter", row.model_prefix), - ) - elif par.tp > 1: - # Plain TP: one AllReduce after attention out-proj, another after MLP. - yield ( - "allreduce", - { - "num_elements": int(tokens * H), - "dtype": comm_dtype, - "world_size": int(par.tp), - }, - _name("allreduce", row.model_prefix), - ) - - if moe.num_experts > 0 and par.moe_ep > 1: - # MoE expert-parallel dispatch + combine. - yield ( - "dispatch", - { - "num_tokens": int(tokens), - "num_experts": int(moe.num_experts), - "top_k": int(moe.num_experts_per_tok), - "hidden": int(H), - "dtype": moe_comm_dtype, - "world_size": int(par.moe_ep), - "mode": "low_latency" if phase == "decode" else "normal", - }, - _name("dispatch", row.model_prefix), - ) - yield ( - "combine", - { - "num_tokens": int(tokens), - "num_experts": int(moe.num_experts), - "top_k": int(moe.num_experts_per_tok), - "hidden": int(H), - "dtype": moe_comm_dtype, - "world_size": int(par.moe_ep), - "mode": "low_latency" if phase == "decode" else "normal", - }, - _name("combine", row.model_prefix), - ) - - -# --------------------------------------------------------------------------- -# Top-level row enumeration -# --------------------------------------------------------------------------- - - -def ops_for_row(row: WorkloadRow, arch: Arch) -> list[OpTriple]: - par = Parallelism(tp=row.tp, ep=row.ep, dp_attn=row.dp_attn) - dts = resolve_dtypes(row) - - # Attention runs per-rank (DP-attn slices tokens, plain TP shards heads). - prefill_M = _prefill_tokens(row, par) - decode_M = _decode_tokens(row, par) - # MoE runs on the full batch after AllGather under DP-attn; EP only shards weights. - prefill_moe_tokens = _moe_tokens_prefill(row) - decode_moe_tokens = _moe_tokens_decode(row) - - out: list[OpTriple] = [] - - # Multinode disagg rows are tagged with the phase the worker handles; emit - # only that phase's ops. Single-node rows (phase=None) emit both phases. - emit_prefill = row.phase in (None, "prefill") - emit_decode = row.phase in (None, "decode") - - if emit_prefill: - out.extend(_attn_block_gemms(prefill_M, arch, par, row, "prefill", dts.attn)) - out.extend(_attention_ops(arch, par, row, "prefill", dts)) - out.extend(_dense_mlp_gemms(prefill_M, arch, par, row, "prefill", dts.dense)) - out.extend(_moe_ops(arch, par, row, "prefill", prefill_moe_tokens, dts)) - out.extend(_collective_ops(arch, par, row, "prefill", prefill_moe_tokens, dts)) - - if emit_decode: - out.extend(_attn_block_gemms(decode_M, arch, par, row, "decode", dts.attn)) - out.extend(_attention_ops(arch, par, row, "decode", dts)) - out.extend(_dense_mlp_gemms(decode_M, arch, par, row, "decode", dts.dense)) - out.extend(_moe_ops(arch, par, row, "decode", decode_moe_tokens, dts)) - out.extend(_collective_ops(arch, par, row, "decode", decode_moe_tokens, dts)) - - return out diff --git a/operatorx/scripts/inferencex_testlist/matrix.py b/operatorx/scripts/inferencex_testlist/matrix.py deleted file mode 100644 index eaca559768..0000000000 --- a/operatorx/scripts/inferencex_testlist/matrix.py +++ /dev/null @@ -1,180 +0,0 @@ -"""Load and flatten InferenceX matrix entries from master YAML configs. - -Delegates the heavy expansion (conc-range -> per-conc rows, search-space -fan-out, etc.) to InferenceX's own `python -m infx.matrix.generate full-sweep` so -that this module stays in sync with InferenceX semantics automatically. -""" -from __future__ import annotations - -import json -import os -import subprocess -import sys -from dataclasses import dataclass, field -from typing import Any - - -@dataclass(frozen=True) -class WorkloadRow: - """One InferenceX matrix entry, normalised for shape enumeration.""" - - model: str # HF id, e.g. "deepseek-ai/DeepSeek-R1-0528" - model_prefix: str # InferenceX canonical model family, e.g. "dsr1" - framework: str # sglang / trt / vllm / dynamo-trt / ... - precision: str # fp4 / fp8 / bf16 / int4 / mxfp4 / nvfp4 - runner: str # b200 / b300 / h200 / mi300x / ... - isl: int - osl: int - conc: int # single-node: scalar; multi-node: representative value - tp: int - ep: int - dp_attn: bool - spec_decoding: str # "none" | "mtp" | "draft_model" - disagg: bool - multinode: bool - exp_name: str - # For multinode disagg rows: which phase this row represents and how many - # worker instances exist for that phase. Single-node rows leave phase=None - # (both prefill+decode ops emitted) and num_workers=1. - phase: str | None = None # "prefill" | "decode" | None - num_workers: int = 1 - - @property - def world_size(self) -> int: - # Single-node: world_size == tp * ep_dim. EP and TP can overlap in - # InferenceX configs (e.g. tp=8, ep=4 means EP shards across 4 of the 8 - # ranks). For collective sizing we use max(tp, ep) as the dominant - # group size — refined per-collective in collectives.py. - return max(self.tp, self.ep) - - -def _run_inferencex_generator( - inferencex_dir: str, - config_files: list[str], - extra_flags: list[str] | None = None, -) -> list[dict[str, Any]]: - """Invoke InferenceX's matrix generator and return the JSON entries.""" - if not os.path.isdir(inferencex_dir): - raise FileNotFoundError(f"InferenceX directory not found: {inferencex_dir}") - - inferencex_dir = os.path.abspath(inferencex_dir) - nested = os.path.join(inferencex_dir, "inferencex-e2e") - if os.path.isdir(nested): - inferencex_dir = nested - module_path = os.path.join(inferencex_dir, "infx", "matrix", "generate.py") - legacy_script = os.path.join(inferencex_dir, "utils", "matrix_logic", "generate_sweep_configs.py") - import_paths = [inferencex_dir] - if os.path.isfile(module_path): - generator = ["-m", "infx.matrix.generate"] - elif os.path.isfile(legacy_script): - generator = [legacy_script] - import_paths.insert(0, os.path.dirname(legacy_script)) - else: - raise FileNotFoundError(f"InferenceX generator not found: {module_path}") - - # Explicit paths also pin the selected revision with PYTHONSAFEPATH enabled. - env = os.environ.copy() - if env.get("PYTHONPATH"): - import_paths.append(env["PYTHONPATH"]) - env["PYTHONPATH"] = os.pathsep.join(import_paths) - args = [sys.executable, *generator, "full-sweep", "--config-files", *config_files] - if extra_flags: - args.extend(extra_flags) - - # Run from the end-to-end project so relative paths in the master config resolve. - proc = subprocess.run( - args, capture_output=True, text=True, cwd=inferencex_dir, env=env, check=False - ) - if proc.returncode != 0: - raise RuntimeError( - f"infx.matrix.generate failed (exit {proc.returncode}):\n" - f"stderr:\n{proc.stderr}" - ) - try: - return json.loads(proc.stdout) - except json.JSONDecodeError as e: - raise RuntimeError( - f"Could not parse generator output as JSON: {e}\n" - f"stdout head:\n{proc.stdout[:1000]}" - ) from e - - -_AGENTIC_DEFAULT_ISL = 8192 # representative for agentic-coding traces -_AGENTIC_DEFAULT_OSL = 1024 - - -def _normalize_entry(entry: dict[str, Any]) -> list[WorkloadRow]: - """Turn one matrix entry (single-node or multi-node) into one+ WorkloadRows. - - Multi-node entries have nested prefill/decode worker configs and a list of - concs; we emit one row per (worker phase, conc) so prefill and decode - shapes get enumerated independently. - - Agentic-coding scenarios don't carry isl/osl in the matrix entry — the - actual lengths come from the trace replay. We substitute representative - defaults so the canonical-shape enumerator still emits useful entries. - """ - multinode = "prefill" in entry and "decode" in entry - base = dict( - model=entry["model"], - model_prefix=entry["model-prefix"], - framework=entry["framework"], - precision=entry["precision"], - runner=entry["runner"], - isl=int(entry.get("isl", _AGENTIC_DEFAULT_ISL)), - osl=int(entry.get("osl", _AGENTIC_DEFAULT_OSL)), - spec_decoding=entry.get("spec-decoding", "none"), - disagg=bool(entry.get("disagg", False)), - multinode=multinode, - exp_name=entry["exp-name"], - ) - - if multinode: - concs = entry["conc"] - if isinstance(concs, int): - concs = [concs] - rows: list[WorkloadRow] = [] - # Each multinode entry yields one row per (phase, conc). The row is - # tagged with `phase` so the enumerator only emits that phase's ops, - # and with `num_workers` so per-step batch sizes get divided by the - # number of worker instances handling that phase. - for phase in ("prefill", "decode"): - worker = entry[phase] - num_workers = int(worker.get("num-worker", 1)) - for c in concs: - rows.append(WorkloadRow( - conc=int(c), - tp=int(worker["tp"]), - ep=int(worker.get("ep", 1)), - dp_attn=bool(worker.get("dp-attn", False)), - phase=phase, - num_workers=num_workers, - **base, - )) - return rows - - concs = entry["conc"] - if isinstance(concs, int): - concs = [concs] - return [ - WorkloadRow( - conc=int(c), - tp=int(entry["tp"]), - ep=int(entry.get("ep", 1)), - dp_attn=bool(entry.get("dp-attn", False)), - **base, - ) - for c in concs - ] - - -def load_rows( - inferencex_dir: str, - config_files: list[str], - extra_flags: list[str] | None = None, -) -> list[WorkloadRow]: - raw = _run_inferencex_generator(inferencex_dir, config_files, extra_flags) - rows: list[WorkloadRow] = [] - for entry in raw: - rows.extend(_normalize_entry(entry)) - return rows diff --git a/operatorx/scripts/inferencex_testlist/models.py b/operatorx/scripts/inferencex_testlist/models.py deleted file mode 100644 index a6d3bef654..0000000000 --- a/operatorx/scripts/inferencex_testlist/models.py +++ /dev/null @@ -1,344 +0,0 @@ -"""HuggingFace model config loader + per-architecture shape descriptors. - -`load_arch()` returns an `Arch` dataclass that exposes everything the -enumerator needs about a model's structure (hidden dims, attention shape, -MoE config, attention type). Per-arch wrappers translate the various -HF config layouts (top-level, `text_config`, etc.) into a uniform shape. -""" -from __future__ import annotations - -import json -import os -from dataclasses import dataclass, field -from typing import Any - - -_DEFAULT_LOCAL_DIRS = ( - "/models", - "/scratch/fsw/models", -) - - -@dataclass(frozen=True) -class AttentionArch: - kind: str # "mha" | "mla" - num_heads: int - num_kv_heads: int - head_dim: int # MHA: per-head dim. MLA: not directly used (see MLA fields). - # MLA-only: - qk_nope_head_dim: int = 0 - qk_rope_head_dim: int = 0 - v_head_dim: int = 0 - kv_lora_rank: int = 0 - q_lora_rank: int = 0 - # Optional: per-layer attention types ("full" vs "sliding"/"linear"). When - # heterogeneous, the canonical shape set should include all variants. - sliding_window: int | None = None - - -@dataclass(frozen=True) -class MoeArch: - """Empty / disabled when this model has no MoE layers.""" - num_experts: int = 0 - num_experts_per_tok: int = 0 - moe_intermediate_size: int = 0 - n_shared_experts: int = 0 - # If the model has dense (non-MoE) layers in addition to MoE layers, set - # `dense_intermediate_size` so we also emit MLP-GEMM shapes for those. - dense_intermediate_size: int = 0 - num_dense_layers: int = 0 # number of leading dense layers (first_k_dense_replace etc.) - - -@dataclass(frozen=True) -class Arch: - name: str - family: str # "deepseek" | "glm" | "kimi" | "minimax" | "gptoss" | "qwen3moe" - hidden_size: int - num_layers: int - attention: AttentionArch - moe: MoeArch - mtp_num_layers: int = 0 # additional speculative-decode layers (DeepSeek MTP, Qwen MTP) - - -# --------------------------------------------------------------------------- -# Config loading -# --------------------------------------------------------------------------- - - -def _candidate_paths(model_id: str, extra_dirs: tuple[str, ...]) -> list[str]: - """Where to look for a config.json for `model_id`.""" - paths: list[str] = [] - # Bare ids from the master configs are usually "/"; the local - # caches store them under "". - short = model_id.split("/")[-1] - for base in extra_dirs: - paths.append(os.path.join(base, short, "config.json")) - paths.append(os.path.join(base, model_id.replace("/", "_"), "config.json")) - # HF default cache layout: - hf_home = os.environ.get("HF_HOME") or os.path.expanduser("~/.cache/huggingface") - paths.append(os.path.join(hf_home, "hub", f"models--{model_id.replace('/', '--')}", "snapshots")) - return paths - - -def _load_hf_config(model_id: str, extra_dirs: tuple[str, ...]) -> dict[str, Any]: - for p in _candidate_paths(model_id, extra_dirs): - if p.endswith("snapshots") and os.path.isdir(p): - # pick first snapshot dir - for d in sorted(os.listdir(p)): - cand = os.path.join(p, d, "config.json") - if os.path.isfile(cand): - with open(cand) as f: - return json.load(f) - elif os.path.isfile(p): - with open(p) as f: - return json.load(f) - raise FileNotFoundError( - f"Could not locate config.json for {model_id!r}. Searched: {_candidate_paths(model_id, extra_dirs)}" - ) - - -def _text_subconfig(raw: dict[str, Any]) -> dict[str, Any]: - """Some configs nest the LM under `text_config` (Kimi, Qwen3.5 multimodal). - - Top-level architectures/model_type wins so family detection sees the brand - (e.g. KimiK25ForConditionalGeneration) rather than the LM-class it reuses - (Kimi K2.5 reuses DeepseekV3ForCausalLM under the hood).""" - if "text_config" in raw and isinstance(raw["text_config"], dict): - merged = dict(raw["text_config"]) - if raw.get("architectures"): - merged["architectures"] = raw["architectures"] - if raw.get("model_type"): - merged["model_type"] = raw["model_type"] - return merged - return raw - - -# --------------------------------------------------------------------------- -# Family detection -# --------------------------------------------------------------------------- - - -def _family(cfg: dict[str, Any]) -> str: - arch = (cfg.get("architectures") or [""])[0] - mt = cfg.get("model_type", "") - if "Deepseek" in arch or mt.startswith("deepseek"): - return "deepseek" - if "Glm" in arch or mt.startswith("glm"): - return "glm" - if "Kimi" in arch or mt.startswith("kimi"): - return "kimi" - if "MiniMax" in arch or mt.startswith("minimax"): - return "minimax" - if "GptOss" in arch or "gpt_oss" in mt or "gptoss" in arch.lower(): - return "gptoss" - if "Qwen" in arch or mt.startswith("qwen"): - return "qwen3moe" - raise ValueError(f"Unsupported model architecture: arch={arch!r} model_type={mt!r}") - - -# --------------------------------------------------------------------------- -# Per-family builders -# --------------------------------------------------------------------------- - - -def _build_deepseek(cfg: dict[str, Any], name: str) -> Arch: - # DeepSeek V3 / R1: qk_nope_head_dim, qk_rope_head_dim, v_head_dim, kv_lora_rank - # are all explicit; `head_dim` is absent. - # DeepSeek V4: head_dim is set (= v_head_dim), qk_rope_head_dim is set, - # qk_nope_head_dim is implicit (= head_dim - qk_rope_head_dim), - # kv_lora_rank is named `o_lora_rank`. - qk_rope = cfg.get("qk_rope_head_dim", 0) - head_dim_explicit = cfg.get("head_dim") or 0 - v_head_dim = cfg.get("v_head_dim") or head_dim_explicit - qk_nope = cfg.get("qk_nope_head_dim") - if qk_nope is None: - qk_nope = head_dim_explicit - qk_rope if head_dim_explicit else 0 - kv_lora = cfg.get("kv_lora_rank") or cfg.get("o_lora_rank") or 0 - return Arch( - name=name, - family="deepseek", - hidden_size=cfg["hidden_size"], - num_layers=cfg["num_hidden_layers"], - attention=AttentionArch( - kind="mla", - num_heads=cfg["num_attention_heads"], - num_kv_heads=cfg.get("num_key_value_heads", 1), - head_dim=v_head_dim, - qk_nope_head_dim=qk_nope, - qk_rope_head_dim=qk_rope, - v_head_dim=v_head_dim, - kv_lora_rank=kv_lora, - q_lora_rank=cfg["q_lora_rank"], - sliding_window=cfg.get("sliding_window"), - ), - moe=MoeArch( - num_experts=cfg["n_routed_experts"], - num_experts_per_tok=cfg["num_experts_per_tok"], - moe_intermediate_size=cfg["moe_intermediate_size"], - n_shared_experts=cfg.get("n_shared_experts", 0), - dense_intermediate_size=cfg.get("intermediate_size", 0), - num_dense_layers=cfg.get("first_k_dense_replace", 0) or 0, - ), - mtp_num_layers=cfg.get("num_nextn_predict_layers", 0) or 0, - ) - - -def _build_glm(cfg: dict[str, Any], name: str) -> Arch: - # GLM-5 MoE+DSA: it has MLA-style attention (qk_nope/qk_rope/v_head_dim/kv_lora). - return Arch( - name=name, - family="glm", - hidden_size=cfg["hidden_size"], - num_layers=cfg["num_hidden_layers"], - attention=AttentionArch( - kind="mla", - num_heads=cfg["num_attention_heads"], - num_kv_heads=cfg.get("num_key_value_heads", cfg["num_attention_heads"]), - head_dim=cfg.get("head_dim", 0), - qk_nope_head_dim=cfg["qk_nope_head_dim"], - qk_rope_head_dim=cfg["qk_rope_head_dim"], - v_head_dim=cfg["v_head_dim"], - kv_lora_rank=cfg["kv_lora_rank"], - q_lora_rank=cfg["q_lora_rank"], - ), - moe=MoeArch( - num_experts=cfg["n_routed_experts"], - num_experts_per_tok=cfg["num_experts_per_tok"], - moe_intermediate_size=cfg["moe_intermediate_size"], - n_shared_experts=cfg.get("n_shared_experts", 0), - dense_intermediate_size=cfg.get("intermediate_size", 0), - num_dense_layers=cfg.get("first_k_dense_replace", 0) or 0, - ), - mtp_num_layers=cfg.get("num_nextn_predict_layers", 0) or 0, - ) - - -def _build_kimi(cfg: dict[str, Any], name: str) -> Arch: - # Kimi K2.5: MLA + MoE. Same shape as DeepSeek. - return Arch( - name=name, - family="kimi", - hidden_size=cfg["hidden_size"], - num_layers=cfg["num_hidden_layers"], - attention=AttentionArch( - kind="mla", - num_heads=cfg["num_attention_heads"], - num_kv_heads=cfg.get("num_key_value_heads", cfg["num_attention_heads"]), - head_dim=cfg.get("head_dim", 0), - qk_nope_head_dim=cfg["qk_nope_head_dim"], - qk_rope_head_dim=cfg["qk_rope_head_dim"], - v_head_dim=cfg["v_head_dim"], - kv_lora_rank=cfg["kv_lora_rank"], - q_lora_rank=cfg["q_lora_rank"], - ), - moe=MoeArch( - num_experts=cfg["n_routed_experts"], - num_experts_per_tok=cfg["num_experts_per_tok"], - moe_intermediate_size=cfg["moe_intermediate_size"], - n_shared_experts=cfg.get("n_shared_experts", 0), - dense_intermediate_size=cfg.get("intermediate_size", 0), - num_dense_layers=cfg.get("first_k_dense_replace", 0) or 0, - ), - mtp_num_layers=cfg.get("num_nextn_predict_layers", 0) or 0, - ) - - -def _build_minimax(cfg: dict[str, Any], name: str) -> Arch: - # MiniMax-M2.5: GQA-style MHA + MoE (no shared experts in published config). - return Arch( - name=name, - family="minimax", - hidden_size=cfg["hidden_size"], - num_layers=cfg["num_hidden_layers"], - attention=AttentionArch( - kind="mha", - num_heads=cfg["num_attention_heads"], - num_kv_heads=cfg["num_key_value_heads"], - head_dim=cfg["head_dim"], - ), - moe=MoeArch( - num_experts=cfg.get("num_local_experts", cfg.get("num_experts", 0)), - num_experts_per_tok=cfg["num_experts_per_tok"], - moe_intermediate_size=cfg.get("moe_intermediate_size", cfg.get("intermediate_size", 0)), - n_shared_experts=cfg.get("n_shared_experts", 0), - dense_intermediate_size=cfg.get("intermediate_size", 0), - num_dense_layers=0, - ), - ) - - -def _build_gptoss(cfg: dict[str, Any], name: str) -> Arch: - # GPT-OSS-120B: GQA MHA + MoE. layer_types alternates sliding/full. - sw = cfg.get("sliding_window") - return Arch( - name=name, - family="gptoss", - hidden_size=cfg["hidden_size"], - num_layers=cfg["num_hidden_layers"], - attention=AttentionArch( - kind="mha", - num_heads=cfg["num_attention_heads"], - num_kv_heads=cfg["num_key_value_heads"], - head_dim=cfg["head_dim"], - sliding_window=sw, - ), - moe=MoeArch( - num_experts=cfg.get("num_local_experts", cfg.get("num_experts", 0)), - num_experts_per_tok=cfg["num_experts_per_tok"], - moe_intermediate_size=cfg.get("intermediate_size", 0), - n_shared_experts=0, - dense_intermediate_size=0, - num_dense_layers=0, - ), - ) - - -def _build_qwen3moe(cfg: dict[str, Any], name: str) -> Arch: - # Qwen3.5-MoE: hybrid linear/full attention, MoE with `num_experts` and - # `shared_expert_intermediate_size`. We model the full-attention layers - # only (the linear-attention layers don't fit the canonical attention op - # cleanly yet — TODO). - return Arch( - name=name, - family="qwen3moe", - hidden_size=cfg["hidden_size"], - num_layers=cfg["num_hidden_layers"], - attention=AttentionArch( - kind="mha", - num_heads=cfg["num_attention_heads"], - num_kv_heads=cfg["num_key_value_heads"], - head_dim=cfg["head_dim"], - ), - moe=MoeArch( - num_experts=cfg.get("num_experts", cfg.get("num_local_experts", 0)), - num_experts_per_tok=cfg["num_experts_per_tok"], - moe_intermediate_size=cfg.get("moe_intermediate_size", 0), - n_shared_experts=1 if cfg.get("shared_expert_intermediate_size") else 0, - dense_intermediate_size=cfg.get("shared_expert_intermediate_size", 0), - num_dense_layers=0, - ), - mtp_num_layers=cfg.get("mtp_num_hidden_layers", 0) or 0, - ) - - -_BUILDERS = { - "deepseek": _build_deepseek, - "glm": _build_glm, - "kimi": _build_kimi, - "minimax": _build_minimax, - "gptoss": _build_gptoss, - "qwen3moe": _build_qwen3moe, -} - - -# --------------------------------------------------------------------------- -# Public entry point -# --------------------------------------------------------------------------- - - -def load_arch(model_id: str, extra_dirs: tuple[str, ...] = _DEFAULT_LOCAL_DIRS) -> Arch: - """Load the architecture descriptor for a HuggingFace model id.""" - raw = _load_hf_config(model_id, extra_dirs) - cfg = _text_subconfig(raw) - family = _family(cfg) - return _BUILDERS[family](cfg, name=model_id) diff --git a/operatorx/scripts/inferencex_testlist/parallelism.py b/operatorx/scripts/inferencex_testlist/parallelism.py deleted file mode 100644 index 192ff3990f..0000000000 --- a/operatorx/scripts/inferencex_testlist/parallelism.py +++ /dev/null @@ -1,52 +0,0 @@ -"""TP / EP / DP-attn sharding helpers. - -The InferenceX matrix exposes `tp`, `ep`, `dp_attn`. We translate those into: - - `attn_tp`: how attention weight matrices are sharded across heads (TP). - - `attn_dp`: data-parallel factor for the attention block; when `dp_attn` - is true, attention is replicated per rank and each rank processes a - `1/attn_tp` slice of the batch instead of sharding heads. - - `moe_tp`: TP shard count for MoE expert weights (only when EP doesn't - cover the full world). - - `moe_ep`: expert-parallel rank count. - - `world_size`: max of the parallelism dimensions, used for collectives. - -This mirrors the SGLang / vLLM convention where `ep > 1` shards experts -across ranks while attention stays TP-replicated (or DP-replicated under -`dp_attn=true`). -""" -from __future__ import annotations - -from dataclasses import dataclass - - -@dataclass(frozen=True) -class Parallelism: - tp: int - ep: int - dp_attn: bool - - @property - def world_size(self) -> int: - # In InferenceX configs `tp` is the total tensor-parallel rank count - # and `ep` <= tp partitions experts across a subset (or all) of those - # ranks. World size for collectives equals tp. - return self.tp - - @property - def attn_tp(self) -> int: - """Attention head-sharding factor.""" - return 1 if self.dp_attn else self.tp - - @property - def attn_dp(self) -> int: - """Attention data-parallel factor (batch is split this many ways).""" - return self.tp if self.dp_attn else 1 - - @property - def moe_ep(self) -> int: - return max(1, self.ep) - - @property - def moe_tp(self) -> int: - # Remaining TP after EP carves out its dimension; SGLang convention. - return max(1, self.tp // self.moe_ep) diff --git a/operatorx/scripts/profile_sglang_moe.py b/operatorx/scripts/profile_sglang_moe.py deleted file mode 100755 index 38cad6f238..0000000000 --- a/operatorx/scripts/profile_sglang_moe.py +++ /dev/null @@ -1,63 +0,0 @@ -#!/usr/bin/env python3 -"""Run a single (shape, dtype) sglang MoE forward inside torch.profiler so we -can see per-CUDA-kernel FLOPs and time. The dashboard reports >100% bf16 SOL -for some sglang MoE shapes and we want to understand whether the kernel is -actually doing the work the op_spec counts. - -Usage (inside the sglang container): - OPERATORX_BACKENDS=sglang OPERATORX_CLUSTER=b200_dgx_8x \ - WORLD_SIZE=8 RANK=$SLURM_PROCID LOCAL_RANK=$SLURM_LOCALID \ - torchrun --nnodes=1 --nproc-per-node=8 scripts/profile_sglang_moe.py -""" -from __future__ import annotations - -import os -import sys -import torch - -# Force import path -sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) - -from operatorx.core import Op -from operatorx.runners.nvidia.backends import sglang as sglang_be - -# The shape we're suspicious of (b300 sglang showed 231% bf16 SOL) -ARGS = { - "num_tokens": 16384, "hidden": 4096, "intermediate": 2048, - "num_experts": 128, "top_k": 6, "world_size": 8, "tp": 1, "ep": 8, - "dtype_act": "bf16", "dtype_weight": "bf16", - "expert_distribution": "uniform", -} - -op = Op(type="moe_forward", args=ARGS, backend="sglang") -ctx = sglang_be._prepare_moe_forward(op) - -# Warmup -for _ in range(3): - sglang_be._kernel_moe_forward(ctx) -torch.cuda.synchronize() - -# Profile a single forward -with torch.profiler.profile( - activities=[torch.profiler.ProfilerActivity.CUDA, torch.profiler.ProfilerActivity.CPU], - record_shapes=True, - with_flops=True, -) as prof: - sglang_be._kernel_moe_forward(ctx) - torch.cuda.synchronize() - -if int(os.environ.get("RANK", "0")) == 0: - print(prof.key_averages().table( - sort_by="self_cuda_time_total", row_limit=20, max_name_column_width=80)) - # Also dump aggregate flops - total_flops = sum(getattr(evt, "flops", 0) or 0 for evt in prof.key_averages()) - total_cuda_us = sum(evt.self_cuda_time_total for evt in prof.key_averages()) / 1.0 - print() - print(f"== aggregate ==") - print(f"profiler flops: {total_flops/1e12:.2f} TFLOP") - print(f"profiler cuda_us: {total_cuda_us:.1f}") - if total_cuda_us > 0: - print(f"profiler tflops/s: {total_flops/(total_cuda_us*1e-6)/1e12:.1f}") - # Theoretical - th_flops = 6 * ARGS["num_tokens"] * ARGS["top_k"] * ARGS["hidden"] * ARGS["intermediate"] - print(f"theoretical (our formula): {th_flops/1e12:.2f} TFLOP") diff --git a/operatorx/scripts/pull_containers.py b/operatorx/scripts/pull_containers.py deleted file mode 100755 index 4f362018ac..0000000000 --- a/operatorx/scripts/pull_containers.py +++ /dev/null @@ -1,98 +0,0 @@ -#!/usr/bin/env python3 -"""Import container images from containers.toml as enroot squash files. - -Run on the cluster head node before submit_smoke.py. Idempotent — already-imported -images are skipped. The actual import runs inside srun on a compute node, matching -the cluster convention. -""" - -from __future__ import annotations - -import shlex -import subprocess -import sys -from pathlib import Path - -try: - import tomllib # Python 3.11+ -except ModuleNotFoundError: # Python 3.10 fallback (e.g. b300 login) - import tomli as tomllib # type: ignore - -import os - -SQUASH_DIR = Path(os.environ.get("OPERATORX_SQUASH_DIR", "/home/sa-shared/containers")) -PROJECT_ROOT = Path(__file__).resolve().parent.parent -MANIFEST = PROJECT_ROOT / "containers.toml" -PARTITION = os.environ.get("OPERATORX_PARTITION", "gpu-2") -ACCOUNT = os.environ.get("OPERATORX_ACCOUNT") # None -> omit --account -QOS = os.environ.get("OPERATORX_QOS") # None -> omit --qos - - -def safe_name(image: str) -> str: - out = image - for ch in "/:@#": - out = out.replace(ch, "_") - return out - - -def squash_path(image: str) -> Path: - return SQUASH_DIR / f"{safe_name(image)}.sqsh" - - -def unique_images(platform: str) -> set[str]: - data = tomllib.loads(MANIFEST.read_text()) - return {entry["image"] for entry in data.get(platform, {}).values()} - - -def import_image(image: str) -> int: - sqsh = squash_path(image) - if sqsh.exists(): - print(f"[skip] {image} -> {sqsh}") - return 0 - SQUASH_DIR.mkdir(parents=True, exist_ok=True) - - sqsh_q = shlex.quote(str(sqsh)) - lock_q = shlex.quote(f"{sqsh}.lock") - image_q = shlex.quote(image) - inner = ( - f"exec 9>{lock_q} && " - f"flock -w 600 9 || {{ echo 'lock timeout'; exit 1; }} && " - f"if unsquashfs -l {sqsh_q} >/dev/null 2>&1; then " - f"echo '[skip] became valid during lock wait'; " - f"else rm -f {sqsh_q} && enroot import -o {sqsh_q} docker://{image_q}; fi" - ) - - cmd = ["srun", f"--partition={PARTITION}", "--gres=gpu:1", "--time=60", - f"--job-name=pull-{safe_name(image)[:40]}"] - if ACCOUNT: - cmd.append(f"--account={ACCOUNT}") - if QOS: - cmd.append(f"--qos={QOS}") - cmd += ["bash", "-c", inner] - print(f"[pull] {image} -> {sqsh}") - return subprocess.run(cmd).returncode - - -def main(argv: list[str]) -> int: - if len(argv) < 2: - print(f"usage: {argv[0]} ") - print(" e.g., nvidia | amd | tpu | trainium") - return 2 - platform = argv[1] - images = unique_images(platform) - if not images: - print(f"no entries for platform={platform!r} in {MANIFEST}") - return 1 - print(f"{len(images)} unique image(s) to ensure for {platform}:") - for img in sorted(images): - print(f" {img}") - print() - rc = 0 - for image in sorted(images): - if import_image(image) != 0: - rc = 1 - return rc - - -if __name__ == "__main__": - sys.exit(main(sys.argv)) diff --git a/operatorx/scripts/submit_run.py b/operatorx/scripts/submit_run.py deleted file mode 100755 index a9e897495a..0000000000 --- a/operatorx/scripts/submit_run.py +++ /dev/null @@ -1,221 +0,0 @@ -#!/usr/bin/env python3 -"""Submit SLURM jobs to run operatorx (python -m operatorx) for every backend on a platform. - -For each (image, world_size) combination, submits one srun job that launches -world_size tasks (potentially across multiple nodes). Backends sharing an image -run in the same job; world sizes are launched separately so torch.distributed -can be sized per group. - -Squash files expected at /home/sa-shared/containers/.sqsh. -Run scripts/pull_containers.py first if any are missing. -""" - -from __future__ import annotations - -import json -import shlex -import subprocess -import sys -from collections import defaultdict -from pathlib import Path - -try: - import tomllib # Python 3.11+ -except ModuleNotFoundError: # Python 3.10 fallback (e.g. b300 login) - import tomli as tomllib # type: ignore - -import os - -SQUASH_DIR = Path(os.environ.get("OPERATORX_SQUASH_DIR", "/home/sa-shared/containers")) -PROJECT_ROOT = Path(__file__).resolve().parent.parent -MANIFEST = PROJECT_ROOT / "containers.toml" -DEFAULT_PLATFORM = "nvidia" -PARTITION = os.environ.get("OPERATORX_PARTITION", "gpu-2") -JOB_NAME = os.environ.get("OPERATORX_JOB_NAME", "benchmark") -ACCOUNT = os.environ.get("OPERATORX_ACCOUNT") # None -> omit --account -QOS = os.environ.get("OPERATORX_QOS") # None -> omit --qos -GPUS_PER_NODE = 8 -WORLD_SIZES = [1, 2, 4, 8] # ws>8 disabled: multi-node NCCL IB bring-up hangs on b200/b300 - -# Default cluster ids per platform (overridable via $OPERATORX_CLUSTER). -DEFAULT_CLUSTER = { - "nvidia": "b200_dgx_8x", - "amd": "mi355x_8x", - "tpu": "v6e_4x", - "trainium": "trn3_16x", -} - - -def safe_name(image: str) -> str: - out = image - for ch in "/:@#": - out = out.replace(ch, "_") - return out - - -def load_groups(platform: str) -> dict[str, list[str]]: - """Group backends in containers.toml by their container image. - - If $OPERATORX_BACKENDS is set (comma-separated allowlist), only backends - in that list are included; groups that end up empty are dropped. main.py - already honors this var at run time — propagating it here also stops us - from submitting jobs that would have produced zero new rows. - """ - allow = {b.strip() for b in os.environ.get("OPERATORX_BACKENDS", "").split(",") if b.strip()} - data = tomllib.loads(MANIFEST.read_text()) - groups: dict[str, list[str]] = defaultdict(list) - for backend, entry in data.get(platform, {}).items(): - if allow and backend not in allow: - continue - groups[entry["image"]].append(backend) - return dict(groups) - - -def slurm_layout(world_size: int) -> tuple[int, int, int]: - """(nodes, ntasks_per_node, gpus_per_node)""" - if world_size <= GPUS_PER_NODE: - return 1, world_size, world_size - nodes = (world_size + GPUS_PER_NODE - 1) // GPUS_PER_NODE - return nodes, GPUS_PER_NODE, GPUS_PER_NODE - - -def _csv(s: str | None) -> list[str]: - if not s: - return [] - return [x.strip() for x in s.split(",") if x.strip()] - - -def _scan_testlists(testlist_names: list[str]) -> tuple[set[tuple[int, int, int, int]], set[int]]: - """Walks selected testlists. Returns - moe_combos = {(world_size, ep, routed_tp, shared_tp)} for moe_forward shapes, - non_moe_ws = {world_size} that has at least one non-moe_forward shape.""" - tl_dir = PROJECT_ROOT / "testlists" - available = {p.stem: p for p in tl_dir.glob("*.json")} - wanted = testlist_names or sorted(available) - moe_combos: set[tuple[int, int, int, int]] = set() - non_moe_ws: set[int] = set() - for name in wanted: - path = available.get(name) - if path is None: - continue - for shape in json.loads(path.read_text()): - a = shape.get("args", {}) - ws = int(a.get("world_size", 1)) - if shape.get("type") == "moe_forward": - moe_combos.add(( - ws, - int(a.get("expert_parallel_size", 1)), - int(a.get("routed_tensor_parallel_size", 1)), - int(a.get("shared_tensor_parallel_size", 1)), - )) - else: - non_moe_ws.add(ws) - return moe_combos, non_moe_ws - - -def submit(image: str, backends: list[str], world_size: int, platform: str, cluster: str, - moe_parallelism: tuple[int, int, int] | None = None) -> int: - sqsh = SQUASH_DIR / f"{safe_name(image)}.sqsh" - if not sqsh.exists(): - print(f"[error] missing squash file: {sqsh}") - print(f" run scripts/pull_containers.py {platform} first") - return 1 - - project = PROJECT_ROOT - nodes, tpn, gpn = slurm_layout(world_size) - job_name = JOB_NAME - backend_list = ",".join(backends) - - inner = " && ".join([ - "set -e", - "export TRITON_CACHE_DIR=/tmp/triton_cache", - "export MPLCONFIGDIR=/tmp/matplotlib_config", - "export HOME=/tmp/home_tmp", - "export NCCL_DEBUG=WARN", - "unset NCCL_ASYNC_ERROR_HANDLING", - "export PYTHONWARNINGS=ignore::SyntaxWarning", - f"cd {project}", - f"export PYTHONPATH={shlex.quote(str(project.parent))}", - "export RANK=$SLURM_PROCID", - "export LOCAL_RANK=$SLURM_LOCALID", - "export WORLD_SIZE=$SLURM_NTASKS", - "export MASTER_ADDR=$(python -c 'import os,re; nl=os.environ.get(\"SLURM_NODELIST\",\"\"); m=re.match(r\"^(.+?)\\[(\\d+)\", nl); print((m.group(1)+m.group(2)) if m else nl)')", - "export MASTER_PORT=29500", - f"OPERATORX_BACKENDS={backend_list} OPERATORX_CLUSTER={cluster}" - + (f" OPERATORX_TESTLISTS={os.environ['OPERATORX_TESTLISTS']}" if os.environ.get("OPERATORX_TESTLISTS") else "") - + (f" OPERATORX_MOE_PARALLELISM={moe_parallelism[0]}:{moe_parallelism[1]}:{moe_parallelism[2]}" if moe_parallelism else "") - + " python -m operatorx", - ]) - - # Use sbatch (async) so SLURM can schedule jobs in parallel as resources free. - # `srun --container-image=` inside the wrap lands the actual work in the container. - inner_srun = ( - f"srun --container-image={sqsh} --container-mounts={project}:{project} " - f"bash -c " + shlex.quote(inner) - ) - log_dir = project / "logs" - log_dir.mkdir(parents=True, exist_ok=True) - tag = f"ws{world_size}" - if moe_parallelism: - tag += f"-moe{moe_parallelism[0]}_{moe_parallelism[1]}_{moe_parallelism[2]}" - out = log_dir / f"opx-{job_name}-{platform}-{tag}-{'_'.join(backends)}-%j.out" - err = log_dir / f"opx-{job_name}-{platform}-{tag}-{'_'.join(backends)}-%j.err" - cmd = ["sbatch", - f"--partition={PARTITION}", - f"--nodes={nodes}", - f"--ntasks={world_size}", - f"--ntasks-per-node={tpn}", - f"--gres=gpu:{gpn}", - f"--time={os.environ.get('OPERATORX_TIME_MIN', '30')}", - f"--job-name={job_name}", - f"--output={out}", - f"--error={err}"] - if ACCOUNT: - cmd.append(f"--account={ACCOUNT}") - if QOS: - cmd.append(f"--qos={QOS}") - cmd += [f"--wrap={inner_srun}"] - - print(f"[submit] {job_name}") - print(f" image: {image}") - print(f" backends: {backends}") - print(f" world_size: {world_size} ({nodes} node(s) x {tpn} task(s))") - if moe_parallelism: - print(f" moe parallelism: ep={moe_parallelism[0]} routed_tp={moe_parallelism[1]} shared_tp={moe_parallelism[2]}") - return subprocess.run(cmd).returncode - - -def main(argv: list[str]) -> int: - platform = argv[1] if len(argv) > 1 else DEFAULT_PLATFORM - cluster = os.environ.get("OPERATORX_CLUSTER", DEFAULT_CLUSTER.get(platform, "")) - if not cluster: - print(f"no default cluster for platform={platform!r}; set $OPERATORX_CLUSTER") - return 1 - groups = load_groups(platform) - if not groups: - print(f"no entries for platform={platform!r} in {MANIFEST}") - return 1 - testlist_names = _csv(os.environ.get("OPERATORX_TESTLISTS")) - moe_combos, non_moe_ws = _scan_testlists(testlist_names) - moe_combos = {c for c in moe_combos if c[0] in WORLD_SIZES} - non_moe_ws = {ws for ws in non_moe_ws if ws in WORLD_SIZES} - print(f"{platform}: {len(groups)} container(s) on cluster={cluster}; " - f"{len(moe_combos)} MoE combos, {len(non_moe_ws)} non-MoE ws -> " - f"{len(groups) * (len(moe_combos) + len(non_moe_ws))} job(s)") - print() - rc = 0 - for image, backends in sorted(groups.items()): - for ws, ep, r_tp, s_tp in sorted(moe_combos): - if submit(image, sorted(backends), ws, platform, cluster, - moe_parallelism=(ep, r_tp, s_tp)) != 0: - rc = 1 - print() - for ws in sorted(non_moe_ws): - if submit(image, sorted(backends), ws, platform, cluster) != 0: - rc = 1 - print() - return rc - - -if __name__ == "__main__": - sys.exit(main(sys.argv)) diff --git a/operatorx/testlists/attention.json b/operatorx/testlists/attention.json deleted file mode 100644 index 7597df3c1d..0000000000 --- a/operatorx/testlists/attention.json +++ /dev/null @@ -1,42823 +0,0 @@ -[ - { - "type": "attention_mha", - "args": { - "batch_size": 1, - "seq_len_q": 4096, - "seq_len_kv": 4096, - "num_heads": 32, - "num_heads_kv": 8, - "head_dim": 128, - "dtype_q": "bf16", - "dtype_k": "bf16", - "dtype_v": "bf16", - "dtype_o": "bf16", - "causal": true - }, - "name": "llama3-8b" - }, - { - "type": "attention_mha", - "args": { - "batch_size": 1, - "seq_len_q": 4096, - "seq_len_kv": 4096, - "num_heads": 32, - "num_heads_kv": 8, - "head_dim": 128, - "dtype_q": "fp16", - "dtype_k": "fp16", - "dtype_v": "fp16", - "dtype_o": "fp16", - "causal": true - }, - "name": "llama3-8b" - }, - { - "type": "attention_mha", - "args": { - "batch_size": 1, - "seq_len_q": 4096, - "seq_len_kv": 4096, - "num_heads": 32, - "num_heads_kv": 8, - "head_dim": 128, - "dtype_q": "bf16", - "dtype_k": "fp8", - "dtype_v": "fp8", - "dtype_o": "bf16", - "causal": true - }, - "name": "llama3-8b" - }, - { - "type": "attention_mha", - "args": { - "batch_size": 1, - "seq_len_q": 8192, - "seq_len_kv": 8192, - "num_heads": 32, - "num_heads_kv": 8, - "head_dim": 128, - "dtype_q": "bf16", - "dtype_k": "bf16", - "dtype_v": "bf16", - "dtype_o": "bf16", - "causal": true - }, - "name": "llama3-8b" - }, - { - "type": "attention_mha", - "args": { - "batch_size": 1, - "seq_len_q": 8192, - "seq_len_kv": 8192, - "num_heads": 32, - "num_heads_kv": 8, - "head_dim": 128, - "dtype_q": "fp16", - "dtype_k": "fp16", - "dtype_v": "fp16", - "dtype_o": "fp16", - "causal": true - }, - "name": "llama3-8b" - }, - { - "type": "attention_mha", - "args": { - "batch_size": 1, - "seq_len_q": 1, - "seq_len_kv": 4096, - "num_heads": 32, - "num_heads_kv": 8, - "head_dim": 128, - "dtype_q": "bf16", - "dtype_k": "bf16", - "dtype_v": "bf16", - "dtype_o": "bf16", - "causal": true - }, - "name": "llama3-8b" - }, - { - "type": "attention_mha", - "args": { - "batch_size": 1, - "seq_len_q": 1, - "seq_len_kv": 4096, - "num_heads": 32, - "num_heads_kv": 8, - "head_dim": 128, - "dtype_q": "fp16", - "dtype_k": "fp16", - "dtype_v": "fp16", - "dtype_o": "fp16", - "causal": true - }, - "name": "llama3-8b" - }, - { - "type": "attention_mha", - "args": { - "batch_size": 1, - "seq_len_q": 1, - "seq_len_kv": 4096, - "num_heads": 32, - "num_heads_kv": 8, - "head_dim": 128, - "dtype_q": "bf16", - "dtype_k": "fp8", - "dtype_v": "fp8", - "dtype_o": "bf16", - "causal": true - }, - "name": "llama3-8b" - }, - { - "type": "attention_mha", - "args": { - "batch_size": 1, - "seq_len_q": 1, - "seq_len_kv": 16384, - "num_heads": 32, - "num_heads_kv": 8, - "head_dim": 128, - "dtype_q": "bf16", - "dtype_k": "bf16", - "dtype_v": "bf16", - "dtype_o": "bf16", - "causal": true - }, - "name": "llama3-8b" - }, - { - "type": "attention_mha", - "args": { - "batch_size": 1, - "seq_len_q": 1, - "seq_len_kv": 16384, - "num_heads": 32, - "num_heads_kv": 8, - "head_dim": 128, - "dtype_q": "fp16", - "dtype_k": "fp16", - "dtype_v": "fp16", - "dtype_o": "fp16", - "causal": true - }, - "name": "llama3-8b" - }, - { - "type": "attention_mha", - "args": { - "batch_size": 1, - "seq_len_q": 1, - "seq_len_kv": 16384, - "num_heads": 32, - "num_heads_kv": 8, - "head_dim": 128, - "dtype_q": "bf16", - "dtype_k": "fp8", - "dtype_v": "fp8", - "dtype_o": "bf16", - "causal": true - }, - "name": "llama3-8b" - }, - { - "type": "attention_mha", - "args": { - "batch_size": 1, - "seq_len_q": 1, - "seq_len_kv": 131072, - "num_heads": 32, - "num_heads_kv": 8, - "head_dim": 128, - "dtype_q": "bf16", - "dtype_k": "bf16", - "dtype_v": "bf16", - "dtype_o": "bf16", - "causal": true - }, - "name": "llama3-8b" - }, - { - "type": "attention_mha", - "args": { - "batch_size": 1, - "seq_len_q": 1, - "seq_len_kv": 131072, - "num_heads": 32, - "num_heads_kv": 8, - "head_dim": 128, - "dtype_q": "bf16", - "dtype_k": "fp8", - "dtype_v": "fp8", - "dtype_o": "bf16", - "causal": true - }, - "name": "llama3-8b" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 1, - "seq_len_q": 4096, - "seq_len_kv": 4096, - "num_heads": 128, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "bf16", - "dtype_kv": "bf16", - "dtype_o": "bf16" - }, - "name": "dsv3" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 1, - "seq_len_q": 4096, - "seq_len_kv": 4096, - "num_heads": 128, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "fp16", - "dtype_kv": "fp16", - "dtype_o": "fp16" - }, - "name": "dsv3" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 1, - "seq_len_q": 1, - "seq_len_kv": 4096, - "num_heads": 128, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "bf16", - "dtype_kv": "bf16", - "dtype_o": "bf16" - }, - "name": "dsv3" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 1, - "seq_len_q": 1, - "seq_len_kv": 4096, - "num_heads": 128, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "fp16", - "dtype_kv": "fp16", - "dtype_o": "fp16" - }, - "name": "dsv3" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 1, - "seq_len_q": 1, - "seq_len_kv": 4096, - "num_heads": 128, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "bf16", - "dtype_kv": "fp8", - "dtype_o": "bf16" - }, - "name": "dsv3" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 1, - "seq_len_q": 1, - "seq_len_kv": 16384, - "num_heads": 128, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "bf16", - "dtype_kv": "bf16", - "dtype_o": "bf16" - }, - "name": "dsv3" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 1, - "seq_len_q": 1, - "seq_len_kv": 16384, - "num_heads": 128, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "fp16", - "dtype_kv": "fp16", - "dtype_o": "fp16" - }, - "name": "dsv3" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 1, - "seq_len_q": 1, - "seq_len_kv": 16384, - "num_heads": 128, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "bf16", - "dtype_kv": "fp8", - "dtype_o": "bf16" - }, - "name": "dsv3" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 1, - "seq_len_q": 1, - "seq_len_kv": 131072, - "num_heads": 128, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "bf16", - "dtype_kv": "bf16", - "dtype_o": "bf16" - }, - "name": "dsv3" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 1, - "seq_len_q": 1, - "seq_len_kv": 131072, - "num_heads": 128, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "bf16", - "dtype_kv": "fp8", - "dtype_o": "bf16" - }, - "name": "dsv3" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 303, - "seq_len_q": 1024, - "seq_len_kv": 1024, - "num_heads": 128, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 303, - "seq_len_q": 2, - "seq_len_kv": 1536, - "num_heads": 128, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 151, - "seq_len_q": 1024, - "seq_len_kv": 1024, - "num_heads": 128, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 151, - "seq_len_q": 2, - "seq_len_kv": 1536, - "num_heads": 128, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 218, - "seq_len_q": 1024, - "seq_len_kv": 1024, - "num_heads": 128, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 218, - "seq_len_q": 2, - "seq_len_kv": 1536, - "num_heads": 128, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 109, - "seq_len_q": 1024, - "seq_len_kv": 1024, - "num_heads": 128, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 109, - "seq_len_q": 2, - "seq_len_kv": 1536, - "num_heads": 128, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 1, - "seq_len_q": 1024, - "seq_len_kv": 1024, - "num_heads": 128, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 1, - "seq_len_q": 2, - "seq_len_kv": 1536, - "num_heads": 128, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 6, - "seq_len_q": 1024, - "seq_len_kv": 1024, - "num_heads": 16, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 6, - "seq_len_q": 2, - "seq_len_kv": 1536, - "num_heads": 16, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 2, - "seq_len_q": 1024, - "seq_len_kv": 1024, - "num_heads": 128, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 2, - "seq_len_q": 2, - "seq_len_kv": 1536, - "num_heads": 128, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 3, - "seq_len_q": 1024, - "seq_len_kv": 1024, - "num_heads": 128, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 3, - "seq_len_q": 2, - "seq_len_kv": 1536, - "num_heads": 128, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 6, - "seq_len_q": 1024, - "seq_len_kv": 1024, - "num_heads": 128, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 6, - "seq_len_q": 2, - "seq_len_kv": 1536, - "num_heads": 128, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 11, - "seq_len_q": 1024, - "seq_len_kv": 1024, - "num_heads": 128, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 11, - "seq_len_q": 2, - "seq_len_kv": 1536, - "num_heads": 128, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 22, - "seq_len_q": 1024, - "seq_len_kv": 1024, - "num_heads": 128, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 22, - "seq_len_q": 2, - "seq_len_kv": 1536, - "num_heads": 128, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 45, - "seq_len_q": 1024, - "seq_len_kv": 1024, - "num_heads": 128, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 45, - "seq_len_q": 2, - "seq_len_kv": 1536, - "num_heads": 128, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 10, - "seq_len_q": 1024, - "seq_len_kv": 1024, - "num_heads": 16, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 10, - "seq_len_q": 2, - "seq_len_kv": 1536, - "num_heads": 16, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 15, - "seq_len_q": 1024, - "seq_len_kv": 1024, - "num_heads": 16, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 15, - "seq_len_q": 2, - "seq_len_kv": 1536, - "num_heads": 16, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 25, - "seq_len_q": 1024, - "seq_len_kv": 1024, - "num_heads": 16, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 25, - "seq_len_q": 2, - "seq_len_kv": 1536, - "num_heads": 16, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 45, - "seq_len_q": 1024, - "seq_len_kv": 1024, - "num_heads": 16, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 45, - "seq_len_q": 2, - "seq_len_kv": 1536, - "num_heads": 16, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 90, - "seq_len_q": 1024, - "seq_len_kv": 1024, - "num_heads": 16, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 90, - "seq_len_q": 2, - "seq_len_kv": 1536, - "num_heads": 16, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 180, - "seq_len_q": 1024, - "seq_len_kv": 1024, - "num_heads": 16, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 180, - "seq_len_q": 2, - "seq_len_kv": 1536, - "num_heads": 16, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 1242, - "seq_len_q": 1024, - "seq_len_kv": 1024, - "num_heads": 128, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 1242, - "seq_len_q": 2, - "seq_len_kv": 1536, - "num_heads": 128, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 621, - "seq_len_q": 1024, - "seq_len_kv": 1024, - "num_heads": 128, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 621, - "seq_len_q": 2, - "seq_len_kv": 1536, - "num_heads": 128, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 2715, - "seq_len_q": 1024, - "seq_len_kv": 1024, - "num_heads": 128, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 2715, - "seq_len_q": 2, - "seq_len_kv": 1536, - "num_heads": 128, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 1024, - "seq_len_q": 1024, - "seq_len_kv": 1024, - "num_heads": 128, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 1024, - "seq_len_q": 1, - "seq_len_kv": 1536, - "num_heads": 128, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 512, - "seq_len_q": 1024, - "seq_len_kv": 1024, - "num_heads": 128, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 512, - "seq_len_q": 1, - "seq_len_kv": 1536, - "num_heads": 128, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 548, - "seq_len_q": 1024, - "seq_len_kv": 1024, - "num_heads": 128, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 548, - "seq_len_q": 1, - "seq_len_kv": 1536, - "num_heads": 128, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 274, - "seq_len_q": 1024, - "seq_len_kv": 1024, - "num_heads": 128, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 274, - "seq_len_q": 1, - "seq_len_kv": 1536, - "num_heads": 128, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 341, - "seq_len_q": 1024, - "seq_len_kv": 1024, - "num_heads": 128, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 341, - "seq_len_q": 1, - "seq_len_kv": 1536, - "num_heads": 128, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 170, - "seq_len_q": 1024, - "seq_len_kv": 1024, - "num_heads": 128, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 170, - "seq_len_q": 1, - "seq_len_kv": 1536, - "num_heads": 128, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 1, - "seq_len_q": 1, - "seq_len_kv": 1536, - "num_heads": 128, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 6, - "seq_len_q": 1, - "seq_len_kv": 1536, - "num_heads": 16, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 2, - "seq_len_q": 1, - "seq_len_kv": 1536, - "num_heads": 128, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 3, - "seq_len_q": 1, - "seq_len_kv": 1536, - "num_heads": 128, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 6, - "seq_len_q": 1, - "seq_len_kv": 1536, - "num_heads": 128, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 11, - "seq_len_q": 1, - "seq_len_kv": 1536, - "num_heads": 128, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 22, - "seq_len_q": 1, - "seq_len_kv": 1536, - "num_heads": 128, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 45, - "seq_len_q": 1, - "seq_len_kv": 1536, - "num_heads": 128, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 10, - "seq_len_q": 1, - "seq_len_kv": 1536, - "num_heads": 16, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 15, - "seq_len_q": 1, - "seq_len_kv": 1536, - "num_heads": 16, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 25, - "seq_len_q": 1, - "seq_len_kv": 1536, - "num_heads": 16, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 45, - "seq_len_q": 1, - "seq_len_kv": 1536, - "num_heads": 16, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 90, - "seq_len_q": 1, - "seq_len_kv": 1536, - "num_heads": 16, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 180, - "seq_len_q": 1, - "seq_len_kv": 1536, - "num_heads": 16, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 112, - "seq_len_q": 1024, - "seq_len_kv": 1024, - "num_heads": 128, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 112, - "seq_len_q": 1, - "seq_len_kv": 1536, - "num_heads": 128, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 450, - "seq_len_q": 1024, - "seq_len_kv": 1024, - "num_heads": 16, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 450, - "seq_len_q": 1, - "seq_len_kv": 1536, - "num_heads": 16, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 22, - "seq_len_q": 8192, - "seq_len_kv": 8192, - "num_heads": 128, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 22, - "seq_len_q": 2, - "seq_len_kv": 8704, - "num_heads": 128, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 11, - "seq_len_q": 8192, - "seq_len_kv": 8192, - "num_heads": 128, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 11, - "seq_len_q": 2, - "seq_len_kv": 8704, - "num_heads": 128, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 16, - "seq_len_q": 8192, - "seq_len_kv": 8192, - "num_heads": 128, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 16, - "seq_len_q": 2, - "seq_len_kv": 8704, - "num_heads": 128, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 66, - "seq_len_q": 8192, - "seq_len_kv": 8192, - "num_heads": 16, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 66, - "seq_len_q": 2, - "seq_len_kv": 8704, - "num_heads": 16, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 1, - "seq_len_q": 8192, - "seq_len_kv": 8192, - "num_heads": 128, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 1, - "seq_len_q": 2, - "seq_len_kv": 8704, - "num_heads": 128, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 6, - "seq_len_q": 8192, - "seq_len_kv": 8192, - "num_heads": 16, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 6, - "seq_len_q": 2, - "seq_len_kv": 8704, - "num_heads": 16, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 2, - "seq_len_q": 8192, - "seq_len_kv": 8192, - "num_heads": 128, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 2, - "seq_len_q": 2, - "seq_len_kv": 8704, - "num_heads": 128, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 3, - "seq_len_q": 8192, - "seq_len_kv": 8192, - "num_heads": 128, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 3, - "seq_len_q": 2, - "seq_len_kv": 8704, - "num_heads": 128, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 7, - "seq_len_q": 8192, - "seq_len_kv": 8192, - "num_heads": 128, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 7, - "seq_len_q": 2, - "seq_len_kv": 8704, - "num_heads": 128, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 15, - "seq_len_q": 8192, - "seq_len_kv": 8192, - "num_heads": 128, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 15, - "seq_len_q": 2, - "seq_len_kv": 8704, - "num_heads": 128, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 10, - "seq_len_q": 8192, - "seq_len_kv": 8192, - "num_heads": 16, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 10, - "seq_len_q": 2, - "seq_len_kv": 8704, - "num_heads": 16, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 15, - "seq_len_q": 8192, - "seq_len_kv": 8192, - "num_heads": 16, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 15, - "seq_len_q": 2, - "seq_len_kv": 8704, - "num_heads": 16, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 30, - "seq_len_q": 8192, - "seq_len_kv": 8192, - "num_heads": 16, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 30, - "seq_len_q": 2, - "seq_len_kv": 8704, - "num_heads": 16, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 60, - "seq_len_q": 8192, - "seq_len_kv": 8192, - "num_heads": 16, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 60, - "seq_len_q": 2, - "seq_len_kv": 8704, - "num_heads": 16, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 137, - "seq_len_q": 8192, - "seq_len_kv": 8192, - "num_heads": 128, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 137, - "seq_len_q": 2, - "seq_len_kv": 8704, - "num_heads": 128, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 68, - "seq_len_q": 8192, - "seq_len_kv": 8192, - "num_heads": 128, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 68, - "seq_len_q": 2, - "seq_len_kv": 8704, - "num_heads": 128, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 274, - "seq_len_q": 8192, - "seq_len_kv": 8192, - "num_heads": 128, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 274, - "seq_len_q": 2, - "seq_len_kv": 8704, - "num_heads": 128, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 422, - "seq_len_q": 8192, - "seq_len_kv": 8192, - "num_heads": 128, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 422, - "seq_len_q": 2, - "seq_len_kv": 8704, - "num_heads": 128, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 211, - "seq_len_q": 8192, - "seq_len_kv": 8192, - "num_heads": 128, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 211, - "seq_len_q": 2, - "seq_len_kv": 8704, - "num_heads": 128, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 164, - "seq_len_q": 8192, - "seq_len_kv": 8192, - "num_heads": 128, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 164, - "seq_len_q": 2, - "seq_len_kv": 8704, - "num_heads": 128, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 82, - "seq_len_q": 8192, - "seq_len_kv": 8192, - "num_heads": 128, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 82, - "seq_len_q": 2, - "seq_len_kv": 8704, - "num_heads": 128, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 1, - "seq_len_q": 1, - "seq_len_kv": 8704, - "num_heads": 128, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 6, - "seq_len_q": 1, - "seq_len_kv": 8704, - "num_heads": 16, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 2, - "seq_len_q": 1, - "seq_len_kv": 8704, - "num_heads": 128, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 3, - "seq_len_q": 1, - "seq_len_kv": 8704, - "num_heads": 128, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 6, - "seq_len_q": 8192, - "seq_len_kv": 8192, - "num_heads": 128, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 6, - "seq_len_q": 1, - "seq_len_kv": 8704, - "num_heads": 128, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 12, - "seq_len_q": 8192, - "seq_len_kv": 8192, - "num_heads": 128, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 12, - "seq_len_q": 1, - "seq_len_kv": 8704, - "num_heads": 128, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 25, - "seq_len_q": 8192, - "seq_len_kv": 8192, - "num_heads": 128, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 25, - "seq_len_q": 1, - "seq_len_kv": 8704, - "num_heads": 128, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 10, - "seq_len_q": 1, - "seq_len_kv": 8704, - "num_heads": 16, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 15, - "seq_len_q": 1, - "seq_len_kv": 8704, - "num_heads": 16, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 25, - "seq_len_q": 8192, - "seq_len_kv": 8192, - "num_heads": 16, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 25, - "seq_len_q": 1, - "seq_len_kv": 8704, - "num_heads": 16, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 50, - "seq_len_q": 8192, - "seq_len_kv": 8192, - "num_heads": 16, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 50, - "seq_len_q": 1, - "seq_len_kv": 8704, - "num_heads": 16, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 100, - "seq_len_q": 8192, - "seq_len_kv": 8192, - "num_heads": 16, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 100, - "seq_len_q": 1, - "seq_len_kv": 8704, - "num_heads": 16, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 92, - "seq_len_q": 8192, - "seq_len_kv": 8192, - "num_heads": 128, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 92, - "seq_len_q": 1, - "seq_len_kv": 8704, - "num_heads": 128, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 370, - "seq_len_q": 8192, - "seq_len_kv": 8192, - "num_heads": 16, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 370, - "seq_len_q": 1, - "seq_len_kv": 8704, - "num_heads": 16, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 401, - "seq_len_q": 8192, - "seq_len_kv": 8192, - "num_heads": 128, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 401, - "seq_len_q": 1, - "seq_len_kv": 8704, - "num_heads": 128, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 200, - "seq_len_q": 8192, - "seq_len_kv": 8192, - "num_heads": 128, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 200, - "seq_len_q": 1, - "seq_len_kv": 8704, - "num_heads": 128, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 209, - "seq_len_q": 8192, - "seq_len_kv": 8192, - "num_heads": 128, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 209, - "seq_len_q": 1, - "seq_len_kv": 8704, - "num_heads": 128, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 104, - "seq_len_q": 8192, - "seq_len_kv": 8192, - "num_heads": 128, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 104, - "seq_len_q": 1, - "seq_len_kv": 8704, - "num_heads": 128, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 555, - "seq_len_q": 8192, - "seq_len_kv": 8192, - "num_heads": 128, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 555, - "seq_len_q": 1, - "seq_len_kv": 8704, - "num_heads": 128, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 277, - "seq_len_q": 8192, - "seq_len_kv": 8192, - "num_heads": 128, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 277, - "seq_len_q": 1, - "seq_len_kv": 8704, - "num_heads": 128, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 1, - "seq_len_q": 8192, - "seq_len_kv": 8192, - "num_heads": 16, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 1, - "seq_len_q": 1, - "seq_len_kv": 8704, - "num_heads": 16, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 2, - "seq_len_q": 8192, - "seq_len_kv": 8192, - "num_heads": 16, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 2, - "seq_len_q": 1, - "seq_len_kv": 8704, - "num_heads": 16, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 4, - "seq_len_q": 8192, - "seq_len_kv": 8192, - "num_heads": 16, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 4, - "seq_len_q": 1, - "seq_len_kv": 8704, - "num_heads": 16, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 8, - "seq_len_q": 8192, - "seq_len_kv": 8192, - "num_heads": 16, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 8, - "seq_len_q": 1, - "seq_len_kv": 8704, - "num_heads": 16, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 4, - "seq_len_q": 8192, - "seq_len_kv": 8192, - "num_heads": 128, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 4, - "seq_len_q": 1, - "seq_len_kv": 8704, - "num_heads": 128, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 16, - "seq_len_q": 8192, - "seq_len_kv": 8192, - "num_heads": 16, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 16, - "seq_len_q": 1, - "seq_len_kv": 8704, - "num_heads": 16, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 8, - "seq_len_q": 8192, - "seq_len_kv": 8192, - "num_heads": 128, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 8, - "seq_len_q": 1, - "seq_len_kv": 8704, - "num_heads": 128, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 32, - "seq_len_q": 8192, - "seq_len_kv": 8192, - "num_heads": 16, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 32, - "seq_len_q": 1, - "seq_len_kv": 8704, - "num_heads": 16, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 1, - "seq_len_q": 1024, - "seq_len_kv": 1024, - "num_heads": 128, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "fp8", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 1, - "seq_len_q": 2, - "seq_len_kv": 1536, - "num_heads": 128, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "fp8", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 8, - "seq_len_q": 1024, - "seq_len_kv": 1024, - "num_heads": 16, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "fp8", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 8, - "seq_len_q": 2, - "seq_len_kv": 1536, - "num_heads": 16, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "fp8", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 4, - "seq_len_q": 1024, - "seq_len_kv": 1024, - "num_heads": 128, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "fp8", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 4, - "seq_len_q": 2, - "seq_len_kv": 1536, - "num_heads": 128, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "fp8", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 32, - "seq_len_q": 1024, - "seq_len_kv": 1024, - "num_heads": 16, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "fp8", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 32, - "seq_len_q": 2, - "seq_len_kv": 1536, - "num_heads": 16, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "fp8", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 8, - "seq_len_q": 1024, - "seq_len_kv": 1024, - "num_heads": 128, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "fp8", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 8, - "seq_len_q": 2, - "seq_len_kv": 1536, - "num_heads": 128, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "fp8", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 64, - "seq_len_q": 1024, - "seq_len_kv": 1024, - "num_heads": 16, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "fp8", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 64, - "seq_len_q": 2, - "seq_len_kv": 1536, - "num_heads": 16, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "fp8", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 32, - "seq_len_q": 1024, - "seq_len_kv": 1024, - "num_heads": 128, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "fp8", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 32, - "seq_len_q": 2, - "seq_len_kv": 1536, - "num_heads": 128, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "fp8", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 256, - "seq_len_q": 1024, - "seq_len_kv": 1024, - "num_heads": 16, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "fp8", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 256, - "seq_len_q": 2, - "seq_len_kv": 1536, - "num_heads": 16, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "fp8", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 112, - "seq_len_q": 1024, - "seq_len_kv": 1024, - "num_heads": 128, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "fp8", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 112, - "seq_len_q": 2, - "seq_len_kv": 1536, - "num_heads": 128, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "fp8", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 128, - "seq_len_q": 1024, - "seq_len_kv": 1024, - "num_heads": 128, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "fp8", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 128, - "seq_len_q": 2, - "seq_len_kv": 1536, - "num_heads": 128, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "fp8", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 148, - "seq_len_q": 1024, - "seq_len_kv": 1024, - "num_heads": 128, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "fp8", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 148, - "seq_len_q": 2, - "seq_len_kv": 1536, - "num_heads": 128, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "fp8", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 200, - "seq_len_q": 1024, - "seq_len_kv": 1024, - "num_heads": 128, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "fp8", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 200, - "seq_len_q": 2, - "seq_len_kv": 1536, - "num_heads": 128, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "fp8", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 1, - "seq_len_q": 1, - "seq_len_kv": 1536, - "num_heads": 128, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "fp8", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 4, - "seq_len_q": 1024, - "seq_len_kv": 1024, - "num_heads": 16, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "fp8", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 4, - "seq_len_q": 1, - "seq_len_kv": 1536, - "num_heads": 16, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "fp8", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 4, - "seq_len_q": 1, - "seq_len_kv": 1536, - "num_heads": 128, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "fp8", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 32, - "seq_len_q": 1, - "seq_len_kv": 1536, - "num_heads": 16, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "fp8", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 16, - "seq_len_q": 1024, - "seq_len_kv": 1024, - "num_heads": 128, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "fp8", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 16, - "seq_len_q": 1, - "seq_len_kv": 1536, - "num_heads": 128, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "fp8", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 128, - "seq_len_q": 1024, - "seq_len_kv": 1024, - "num_heads": 16, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "fp8", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 128, - "seq_len_q": 1, - "seq_len_kv": 1536, - "num_heads": 16, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "fp8", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 240, - "seq_len_q": 1024, - "seq_len_kv": 1024, - "num_heads": 128, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "fp8", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 240, - "seq_len_q": 1, - "seq_len_kv": 1536, - "num_heads": 128, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "fp8", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 512, - "seq_len_q": 1024, - "seq_len_kv": 1024, - "num_heads": 128, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "fp8", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 512, - "seq_len_q": 1, - "seq_len_kv": 1536, - "num_heads": 128, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "fp8", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 644, - "seq_len_q": 1024, - "seq_len_kv": 1024, - "num_heads": 128, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "fp8", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 644, - "seq_len_q": 1, - "seq_len_kv": 1536, - "num_heads": 128, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "fp8", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 1, - "seq_len_q": 8192, - "seq_len_kv": 8192, - "num_heads": 128, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "fp8", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 1, - "seq_len_q": 2, - "seq_len_kv": 8704, - "num_heads": 128, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "fp8", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 8, - "seq_len_q": 8192, - "seq_len_kv": 8192, - "num_heads": 16, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "fp8", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 8, - "seq_len_q": 2, - "seq_len_kv": 8704, - "num_heads": 16, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "fp8", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 6, - "seq_len_q": 8192, - "seq_len_kv": 8192, - "num_heads": 128, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "fp8", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 6, - "seq_len_q": 2, - "seq_len_kv": 8704, - "num_heads": 128, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "fp8", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 48, - "seq_len_q": 8192, - "seq_len_kv": 8192, - "num_heads": 16, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "fp8", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 48, - "seq_len_q": 2, - "seq_len_kv": 8704, - "num_heads": 16, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "fp8", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 8, - "seq_len_q": 8192, - "seq_len_kv": 8192, - "num_heads": 128, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "fp8", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 8, - "seq_len_q": 2, - "seq_len_kv": 8704, - "num_heads": 128, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "fp8", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 64, - "seq_len_q": 8192, - "seq_len_kv": 8192, - "num_heads": 16, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "fp8", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 64, - "seq_len_q": 2, - "seq_len_kv": 8704, - "num_heads": 16, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "fp8", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 28, - "seq_len_q": 8192, - "seq_len_kv": 8192, - "num_heads": 128, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "fp8", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 28, - "seq_len_q": 2, - "seq_len_kv": 8704, - "num_heads": 128, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "fp8", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 36, - "seq_len_q": 8192, - "seq_len_kv": 8192, - "num_heads": 128, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "fp8", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 36, - "seq_len_q": 2, - "seq_len_kv": 8704, - "num_heads": 128, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "fp8", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 136, - "seq_len_q": 8192, - "seq_len_kv": 8192, - "num_heads": 128, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "fp8", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 136, - "seq_len_q": 2, - "seq_len_kv": 8704, - "num_heads": 128, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "fp8", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 1, - "seq_len_q": 8192, - "seq_len_kv": 8192, - "num_heads": 16, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "fp8", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 1, - "seq_len_q": 1, - "seq_len_kv": 8704, - "num_heads": 16, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "fp8", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 4, - "seq_len_q": 8192, - "seq_len_kv": 8192, - "num_heads": 128, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "fp8", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 4, - "seq_len_q": 1, - "seq_len_kv": 8704, - "num_heads": 128, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "fp8", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 32, - "seq_len_q": 8192, - "seq_len_kv": 8192, - "num_heads": 16, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "fp8", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 32, - "seq_len_q": 1, - "seq_len_kv": 8704, - "num_heads": 16, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "fp8", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 16, - "seq_len_q": 8192, - "seq_len_kv": 8192, - "num_heads": 128, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "fp8", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 16, - "seq_len_q": 1, - "seq_len_kv": 8704, - "num_heads": 128, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "fp8", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 128, - "seq_len_q": 8192, - "seq_len_kv": 8192, - "num_heads": 16, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "fp8", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 128, - "seq_len_q": 1, - "seq_len_kv": 8704, - "num_heads": 16, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "fp8", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 12, - "seq_len_q": 8192, - "seq_len_kv": 8192, - "num_heads": 128, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "fp8", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 12, - "seq_len_q": 1, - "seq_len_kv": 8704, - "num_heads": 128, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "fp8", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 96, - "seq_len_q": 8192, - "seq_len_kv": 8192, - "num_heads": 16, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "fp8", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 96, - "seq_len_q": 1, - "seq_len_kv": 8704, - "num_heads": 16, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "fp8", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 32, - "seq_len_q": 8192, - "seq_len_kv": 8192, - "num_heads": 128, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "fp8", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 32, - "seq_len_q": 1, - "seq_len_kv": 8704, - "num_heads": 128, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "fp8", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 80, - "seq_len_q": 8192, - "seq_len_kv": 8192, - "num_heads": 128, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "fp8", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 80, - "seq_len_q": 1, - "seq_len_kv": 8704, - "num_heads": 128, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "fp8", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 327, - "seq_len_q": 1024, - "seq_len_kv": 1024, - "num_heads": 128, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 327, - "seq_len_q": 2, - "seq_len_kv": 1536, - "num_heads": 128, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 81, - "seq_len_q": 1024, - "seq_len_kv": 1024, - "num_heads": 128, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 81, - "seq_len_q": 2, - "seq_len_kv": 1536, - "num_heads": 128, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 135, - "seq_len_q": 1024, - "seq_len_kv": 1024, - "num_heads": 128, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 135, - "seq_len_q": 2, - "seq_len_kv": 1536, - "num_heads": 128, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 33, - "seq_len_q": 1024, - "seq_len_kv": 1024, - "num_heads": 128, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 33, - "seq_len_q": 2, - "seq_len_kv": 1536, - "num_heads": 128, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 5, - "seq_len_q": 1024, - "seq_len_kv": 1024, - "num_heads": 128, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 5, - "seq_len_q": 2, - "seq_len_kv": 1536, - "num_heads": 128, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 11, - "seq_len_q": 1024, - "seq_len_kv": 1024, - "num_heads": 16, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 11, - "seq_len_q": 2, - "seq_len_kv": 1536, - "num_heads": 16, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 10, - "seq_len_q": 1024, - "seq_len_kv": 1024, - "num_heads": 128, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 10, - "seq_len_q": 2, - "seq_len_kv": 1536, - "num_heads": 128, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 12, - "seq_len_q": 1024, - "seq_len_kv": 1024, - "num_heads": 128, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 12, - "seq_len_q": 2, - "seq_len_kv": 1536, - "num_heads": 128, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 30, - "seq_len_q": 1024, - "seq_len_kv": 1024, - "num_heads": 128, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 30, - "seq_len_q": 2, - "seq_len_kv": 1536, - "num_heads": 128, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 60, - "seq_len_q": 1024, - "seq_len_kv": 1024, - "num_heads": 128, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 60, - "seq_len_q": 2, - "seq_len_kv": 1536, - "num_heads": 128, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 100, - "seq_len_q": 1024, - "seq_len_kv": 1024, - "num_heads": 128, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 100, - "seq_len_q": 2, - "seq_len_kv": 1536, - "num_heads": 128, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 20, - "seq_len_q": 1024, - "seq_len_kv": 1024, - "num_heads": 16, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 20, - "seq_len_q": 2, - "seq_len_kv": 1536, - "num_heads": 16, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 60, - "seq_len_q": 1024, - "seq_len_kv": 1024, - "num_heads": 16, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 60, - "seq_len_q": 2, - "seq_len_kv": 1536, - "num_heads": 16, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 120, - "seq_len_q": 1024, - "seq_len_kv": 1024, - "num_heads": 16, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 120, - "seq_len_q": 2, - "seq_len_kv": 1536, - "num_heads": 16, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 200, - "seq_len_q": 1024, - "seq_len_kv": 1024, - "num_heads": 16, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 200, - "seq_len_q": 2, - "seq_len_kv": 1536, - "num_heads": 16, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 1171, - "seq_len_q": 1024, - "seq_len_kv": 1024, - "num_heads": 128, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 1171, - "seq_len_q": 2, - "seq_len_kv": 1536, - "num_heads": 128, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 292, - "seq_len_q": 1024, - "seq_len_kv": 1024, - "num_heads": 128, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 292, - "seq_len_q": 2, - "seq_len_kv": 1536, - "num_heads": 128, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 4304, - "seq_len_q": 1024, - "seq_len_kv": 1024, - "num_heads": 128, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 4304, - "seq_len_q": 2, - "seq_len_kv": 1536, - "num_heads": 128, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 1076, - "seq_len_q": 1024, - "seq_len_kv": 1024, - "num_heads": 128, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 1076, - "seq_len_q": 2, - "seq_len_kv": 1536, - "num_heads": 128, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 6463, - "seq_len_q": 1024, - "seq_len_kv": 1024, - "num_heads": 128, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 6463, - "seq_len_q": 2, - "seq_len_kv": 1536, - "num_heads": 128, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 1615, - "seq_len_q": 1024, - "seq_len_kv": 1024, - "num_heads": 128, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 1615, - "seq_len_q": 2, - "seq_len_kv": 1536, - "num_heads": 128, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 588, - "seq_len_q": 1024, - "seq_len_kv": 1024, - "num_heads": 128, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 588, - "seq_len_q": 1, - "seq_len_kv": 1536, - "num_heads": 128, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 147, - "seq_len_q": 1024, - "seq_len_kv": 1024, - "num_heads": 128, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 147, - "seq_len_q": 1, - "seq_len_kv": 1536, - "num_heads": 128, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 5, - "seq_len_q": 1, - "seq_len_kv": 1536, - "num_heads": 128, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 7, - "seq_len_q": 1024, - "seq_len_kv": 1024, - "num_heads": 128, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 7, - "seq_len_q": 1, - "seq_len_kv": 1536, - "num_heads": 128, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 12, - "seq_len_q": 1, - "seq_len_kv": 1536, - "num_heads": 128, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 5, - "seq_len_q": 1024, - "seq_len_kv": 1024, - "num_heads": 32, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 5, - "seq_len_q": 1, - "seq_len_kv": 1536, - "num_heads": 32, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 10, - "seq_len_q": 1024, - "seq_len_kv": 1024, - "num_heads": 32, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 10, - "seq_len_q": 1, - "seq_len_kv": 1536, - "num_heads": 32, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 15, - "seq_len_q": 1024, - "seq_len_kv": 1024, - "num_heads": 32, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 15, - "seq_len_q": 1, - "seq_len_kv": 1536, - "num_heads": 32, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 25, - "seq_len_q": 1024, - "seq_len_kv": 1024, - "num_heads": 32, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 25, - "seq_len_q": 1, - "seq_len_kv": 1536, - "num_heads": 32, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 30, - "seq_len_q": 1, - "seq_len_kv": 1536, - "num_heads": 128, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 55, - "seq_len_q": 1024, - "seq_len_kv": 1024, - "num_heads": 128, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 55, - "seq_len_q": 1, - "seq_len_kv": 1536, - "num_heads": 128, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 97, - "seq_len_q": 1024, - "seq_len_kv": 1024, - "num_heads": 128, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 97, - "seq_len_q": 1, - "seq_len_kv": 1536, - "num_heads": 128, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 197, - "seq_len_q": 1024, - "seq_len_kv": 1024, - "num_heads": 128, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 197, - "seq_len_q": 1, - "seq_len_kv": 1536, - "num_heads": 128, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 60, - "seq_len_q": 1, - "seq_len_kv": 1536, - "num_heads": 16, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 110, - "seq_len_q": 1024, - "seq_len_kv": 1024, - "num_heads": 16, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 110, - "seq_len_q": 1, - "seq_len_kv": 1536, - "num_heads": 16, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 195, - "seq_len_q": 1024, - "seq_len_kv": 1024, - "num_heads": 16, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 195, - "seq_len_q": 1, - "seq_len_kv": 1536, - "num_heads": 16, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 395, - "seq_len_q": 1024, - "seq_len_kv": 1024, - "num_heads": 16, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 395, - "seq_len_q": 1, - "seq_len_kv": 1536, - "num_heads": 16, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 2202, - "seq_len_q": 1024, - "seq_len_kv": 1024, - "num_heads": 128, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 2202, - "seq_len_q": 1, - "seq_len_kv": 1536, - "num_heads": 128, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 550, - "seq_len_q": 1024, - "seq_len_kv": 1024, - "num_heads": 128, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 550, - "seq_len_q": 1, - "seq_len_kv": 1536, - "num_heads": 128, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 4096, - "seq_len_q": 1024, - "seq_len_kv": 1024, - "num_heads": 128, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 4096, - "seq_len_q": 1, - "seq_len_kv": 1536, - "num_heads": 128, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 2305, - "seq_len_q": 1024, - "seq_len_kv": 1024, - "num_heads": 128, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 2305, - "seq_len_q": 1, - "seq_len_kv": 1536, - "num_heads": 128, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 576, - "seq_len_q": 1024, - "seq_len_kv": 1024, - "num_heads": 128, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 576, - "seq_len_q": 1, - "seq_len_kv": 1536, - "num_heads": 128, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 1099, - "seq_len_q": 8192, - "seq_len_kv": 8192, - "num_heads": 128, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 1099, - "seq_len_q": 2, - "seq_len_kv": 8704, - "num_heads": 128, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 26, - "seq_len_q": 8192, - "seq_len_kv": 8192, - "num_heads": 128, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 26, - "seq_len_q": 2, - "seq_len_kv": 8704, - "num_heads": 128, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 52, - "seq_len_q": 8192, - "seq_len_kv": 8192, - "num_heads": 32, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 52, - "seq_len_q": 2, - "seq_len_kv": 8704, - "num_heads": 32, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 4, - "seq_len_q": 2, - "seq_len_kv": 8704, - "num_heads": 128, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 8, - "seq_len_q": 2, - "seq_len_kv": 8704, - "num_heads": 16, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 32, - "seq_len_q": 2, - "seq_len_kv": 8704, - "num_heads": 16, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 90, - "seq_len_q": 8192, - "seq_len_kv": 8192, - "num_heads": 128, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 90, - "seq_len_q": 2, - "seq_len_kv": 8704, - "num_heads": 128, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 598, - "seq_len_q": 8192, - "seq_len_kv": 8192, - "num_heads": 128, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 598, - "seq_len_q": 2, - "seq_len_kv": 8704, - "num_heads": 128, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 149, - "seq_len_q": 8192, - "seq_len_kv": 8192, - "num_heads": 128, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 149, - "seq_len_q": 2, - "seq_len_kv": 8704, - "num_heads": 128, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 52, - "seq_len_q": 8192, - "seq_len_kv": 8192, - "num_heads": 128, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 52, - "seq_len_q": 1, - "seq_len_kv": 8704, - "num_heads": 128, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 105, - "seq_len_q": 8192, - "seq_len_kv": 8192, - "num_heads": 32, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 105, - "seq_len_q": 1, - "seq_len_kv": 8704, - "num_heads": 32, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 31, - "seq_len_q": 8192, - "seq_len_kv": 8192, - "num_heads": 128, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 31, - "seq_len_q": 1, - "seq_len_kv": 8704, - "num_heads": 128, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 63, - "seq_len_q": 8192, - "seq_len_kv": 8192, - "num_heads": 16, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 63, - "seq_len_q": 1, - "seq_len_kv": 8704, - "num_heads": 16, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 12, - "seq_len_q": 8192, - "seq_len_kv": 8192, - "num_heads": 32, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 12, - "seq_len_q": 1, - "seq_len_kv": 8704, - "num_heads": 32, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 294, - "seq_len_q": 8192, - "seq_len_kv": 8192, - "num_heads": 128, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 294, - "seq_len_q": 1, - "seq_len_kv": 8704, - "num_heads": 128, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 73, - "seq_len_q": 8192, - "seq_len_kv": 8192, - "num_heads": 128, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 73, - "seq_len_q": 1, - "seq_len_kv": 8704, - "num_heads": 128, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 546, - "seq_len_q": 8192, - "seq_len_kv": 8192, - "num_heads": 128, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 546, - "seq_len_q": 1, - "seq_len_kv": 8704, - "num_heads": 128, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 136, - "seq_len_q": 8192, - "seq_len_kv": 8192, - "num_heads": 128, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 136, - "seq_len_q": 1, - "seq_len_kv": 8704, - "num_heads": 128, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 1024, - "seq_len_q": 8192, - "seq_len_kv": 8192, - "num_heads": 128, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 1024, - "seq_len_q": 1, - "seq_len_kv": 8704, - "num_heads": 128, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 256, - "seq_len_q": 8192, - "seq_len_kv": 8192, - "num_heads": 128, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 256, - "seq_len_q": 1, - "seq_len_kv": 8704, - "num_heads": 128, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 2, - "seq_len_q": 1024, - "seq_len_kv": 1024, - "num_heads": 128, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "fp8", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 2, - "seq_len_q": 2, - "seq_len_kv": 1536, - "num_heads": 128, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "fp8", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 10, - "seq_len_q": 1024, - "seq_len_kv": 1024, - "num_heads": 16, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "fp8", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 10, - "seq_len_q": 2, - "seq_len_kv": 1536, - "num_heads": 16, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "fp8", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 40, - "seq_len_q": 1024, - "seq_len_kv": 1024, - "num_heads": 128, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "fp8", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 40, - "seq_len_q": 2, - "seq_len_kv": 1536, - "num_heads": 128, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "fp8", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 160, - "seq_len_q": 1024, - "seq_len_kv": 1024, - "num_heads": 16, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "fp8", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 160, - "seq_len_q": 2, - "seq_len_kv": 1536, - "num_heads": 16, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "fp8", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 768, - "seq_len_q": 1024, - "seq_len_kv": 1024, - "num_heads": 128, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "fp8", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 768, - "seq_len_q": 2, - "seq_len_kv": 1536, - "num_heads": 128, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "fp8", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 384, - "seq_len_q": 1024, - "seq_len_kv": 1024, - "num_heads": 128, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "fp8", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 384, - "seq_len_q": 2, - "seq_len_kv": 1536, - "num_heads": 128, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "fp8", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 640, - "seq_len_q": 1024, - "seq_len_kv": 1024, - "num_heads": 128, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "fp8", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 640, - "seq_len_q": 2, - "seq_len_kv": 1536, - "num_heads": 128, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "fp8", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 320, - "seq_len_q": 1024, - "seq_len_kv": 1024, - "num_heads": 128, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "fp8", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 320, - "seq_len_q": 2, - "seq_len_kv": 1536, - "num_heads": 128, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "fp8", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 180, - "seq_len_q": 1024, - "seq_len_kv": 1024, - "num_heads": 128, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "fp8", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 180, - "seq_len_q": 2, - "seq_len_kv": 1536, - "num_heads": 128, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "fp8", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 90, - "seq_len_q": 1024, - "seq_len_kv": 1024, - "num_heads": 128, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "fp8", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 90, - "seq_len_q": 2, - "seq_len_kv": 1536, - "num_heads": 128, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "fp8", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 2816, - "seq_len_q": 1024, - "seq_len_kv": 1024, - "num_heads": 128, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "fp8", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 2816, - "seq_len_q": 2, - "seq_len_kv": 1536, - "num_heads": 128, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "fp8", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 1408, - "seq_len_q": 1024, - "seq_len_kv": 1024, - "num_heads": 128, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "fp8", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 1408, - "seq_len_q": 2, - "seq_len_kv": 1536, - "num_heads": 128, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "fp8", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 528, - "seq_len_q": 1024, - "seq_len_kv": 1024, - "num_heads": 128, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "fp8", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 528, - "seq_len_q": 1, - "seq_len_kv": 1536, - "num_heads": 128, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "fp8", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 264, - "seq_len_q": 1024, - "seq_len_kv": 1024, - "num_heads": 128, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "fp8", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 264, - "seq_len_q": 1, - "seq_len_kv": 1536, - "num_heads": 128, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "fp8", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 768, - "seq_len_q": 1, - "seq_len_kv": 1536, - "num_heads": 128, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "fp8", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 384, - "seq_len_q": 1, - "seq_len_kv": 1536, - "num_heads": 128, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "fp8", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 320, - "seq_len_q": 1, - "seq_len_kv": 1536, - "num_heads": 128, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "fp8", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 160, - "seq_len_q": 1024, - "seq_len_kv": 1024, - "num_heads": 128, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "fp8", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 160, - "seq_len_q": 1, - "seq_len_kv": 1536, - "num_heads": 128, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "fp8", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 3, - "seq_len_q": 1024, - "seq_len_kv": 1024, - "num_heads": 128, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "fp8", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 3, - "seq_len_q": 1, - "seq_len_kv": 1536, - "num_heads": 128, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "fp8", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 12, - "seq_len_q": 1024, - "seq_len_kv": 1024, - "num_heads": 16, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "fp8", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 12, - "seq_len_q": 1, - "seq_len_kv": 1536, - "num_heads": 16, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "fp8", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 32, - "seq_len_q": 1, - "seq_len_kv": 1536, - "num_heads": 128, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "fp8", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 96, - "seq_len_q": 1024, - "seq_len_kv": 1024, - "num_heads": 128, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "fp8", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 96, - "seq_len_q": 1, - "seq_len_kv": 1536, - "num_heads": 128, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "fp8", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 384, - "seq_len_q": 1024, - "seq_len_kv": 1024, - "num_heads": 16, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "fp8", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 384, - "seq_len_q": 1, - "seq_len_kv": 1536, - "num_heads": 16, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "fp8", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 4096, - "seq_len_q": 1024, - "seq_len_kv": 1024, - "num_heads": 128, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "fp8", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 4096, - "seq_len_q": 1, - "seq_len_kv": 1536, - "num_heads": 128, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "fp8", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 2048, - "seq_len_q": 1024, - "seq_len_kv": 1024, - "num_heads": 128, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "fp8", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 2048, - "seq_len_q": 1, - "seq_len_kv": 1536, - "num_heads": 128, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "fp8", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 10, - "seq_len_q": 8192, - "seq_len_kv": 8192, - "num_heads": 128, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "fp8", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 10, - "seq_len_q": 2, - "seq_len_kv": 8704, - "num_heads": 128, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "fp8", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 40, - "seq_len_q": 8192, - "seq_len_kv": 8192, - "num_heads": 16, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "fp8", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 40, - "seq_len_q": 2, - "seq_len_kv": 8704, - "num_heads": 16, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "fp8", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 2, - "seq_len_q": 8192, - "seq_len_kv": 8192, - "num_heads": 128, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "fp8", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 2, - "seq_len_q": 2, - "seq_len_kv": 8704, - "num_heads": 128, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "fp8", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 5, - "seq_len_q": 8192, - "seq_len_kv": 8192, - "num_heads": 128, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "fp8", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 5, - "seq_len_q": 2, - "seq_len_kv": 8704, - "num_heads": 128, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "fp8", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 20, - "seq_len_q": 8192, - "seq_len_kv": 8192, - "num_heads": 16, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "fp8", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 20, - "seq_len_q": 2, - "seq_len_kv": 8704, - "num_heads": 16, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "fp8", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 18, - "seq_len_q": 8192, - "seq_len_kv": 8192, - "num_heads": 128, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "fp8", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 18, - "seq_len_q": 2, - "seq_len_kv": 8704, - "num_heads": 128, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "fp8", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 9, - "seq_len_q": 8192, - "seq_len_kv": 8192, - "num_heads": 128, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "fp8", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 9, - "seq_len_q": 2, - "seq_len_kv": 8704, - "num_heads": 128, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "fp8", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 128, - "seq_len_q": 8192, - "seq_len_kv": 8192, - "num_heads": 128, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "fp8", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 128, - "seq_len_q": 2, - "seq_len_kv": 8704, - "num_heads": 128, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "fp8", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 64, - "seq_len_q": 8192, - "seq_len_kv": 8192, - "num_heads": 128, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "fp8", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 64, - "seq_len_q": 2, - "seq_len_kv": 8704, - "num_heads": 128, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "fp8", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 64, - "seq_len_q": 1, - "seq_len_kv": 8704, - "num_heads": 16, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "fp8", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 16, - "seq_len_q": 8192, - "seq_len_kv": 8192, - "num_heads": 16, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "fp8", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 16, - "seq_len_q": 1, - "seq_len_kv": 8704, - "num_heads": 16, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "fp8", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 64, - "seq_len_q": 1, - "seq_len_kv": 8704, - "num_heads": 128, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "fp8", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 128, - "seq_len_q": 1, - "seq_len_kv": 8704, - "num_heads": 128, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "fp8", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 256, - "seq_len_q": 8192, - "seq_len_kv": 8192, - "num_heads": 16, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "fp8", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 256, - "seq_len_q": 1, - "seq_len_kv": 8704, - "num_heads": 16, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "fp8", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 268, - "seq_len_q": 8192, - "seq_len_kv": 8192, - "num_heads": 128, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "fp8", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 268, - "seq_len_q": 1, - "seq_len_kv": 8704, - "num_heads": 128, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "fp8", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 134, - "seq_len_q": 8192, - "seq_len_kv": 8192, - "num_heads": 128, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "fp8", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 134, - "seq_len_q": 1, - "seq_len_kv": 8704, - "num_heads": 128, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "fp8", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 768, - "seq_len_q": 8192, - "seq_len_kv": 8192, - "num_heads": 128, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "fp8", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 768, - "seq_len_q": 1, - "seq_len_kv": 8704, - "num_heads": 128, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "fp8", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 384, - "seq_len_q": 8192, - "seq_len_kv": 8192, - "num_heads": 128, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "fp8", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 384, - "seq_len_q": 1, - "seq_len_kv": 8704, - "num_heads": 128, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "fp8", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 4, - "seq_len_q": 1024, - "seq_len_kv": 1024, - "num_heads": 32, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 4, - "seq_len_q": 1, - "seq_len_kv": 1536, - "num_heads": 32, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 8, - "seq_len_q": 1024, - "seq_len_kv": 1024, - "num_heads": 32, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 8, - "seq_len_q": 1, - "seq_len_kv": 1536, - "num_heads": 32, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 16, - "seq_len_q": 1024, - "seq_len_kv": 1024, - "num_heads": 32, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 16, - "seq_len_q": 1, - "seq_len_kv": 1536, - "num_heads": 32, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 32, - "seq_len_q": 1024, - "seq_len_kv": 1024, - "num_heads": 32, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 32, - "seq_len_q": 1, - "seq_len_kv": 1536, - "num_heads": 32, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 64, - "seq_len_q": 1024, - "seq_len_kv": 1024, - "num_heads": 32, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 64, - "seq_len_q": 1, - "seq_len_kv": 1536, - "num_heads": 32, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 128, - "seq_len_q": 1024, - "seq_len_kv": 1024, - "num_heads": 32, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 128, - "seq_len_q": 1, - "seq_len_kv": 1536, - "num_heads": 32, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 4, - "seq_len_q": 1024, - "seq_len_kv": 1024, - "num_heads": 16, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 4, - "seq_len_q": 1, - "seq_len_kv": 1536, - "num_heads": 16, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 8, - "seq_len_q": 1024, - "seq_len_kv": 1024, - "num_heads": 16, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 8, - "seq_len_q": 1, - "seq_len_kv": 1536, - "num_heads": 16, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 16, - "seq_len_q": 1024, - "seq_len_kv": 1024, - "num_heads": 16, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 16, - "seq_len_q": 1, - "seq_len_kv": 1536, - "num_heads": 16, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 32, - "seq_len_q": 1024, - "seq_len_kv": 1024, - "num_heads": 16, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 32, - "seq_len_q": 1, - "seq_len_kv": 1536, - "num_heads": 16, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 64, - "seq_len_q": 1024, - "seq_len_kv": 1024, - "num_heads": 16, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 64, - "seq_len_q": 1, - "seq_len_kv": 1536, - "num_heads": 16, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 128, - "seq_len_q": 1024, - "seq_len_kv": 1024, - "num_heads": 16, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 128, - "seq_len_q": 1, - "seq_len_kv": 1536, - "num_heads": 16, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 4, - "seq_len_q": 8192, - "seq_len_kv": 8192, - "num_heads": 32, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 4, - "seq_len_q": 1, - "seq_len_kv": 8704, - "num_heads": 32, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 8, - "seq_len_q": 8192, - "seq_len_kv": 8192, - "num_heads": 32, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 8, - "seq_len_q": 1, - "seq_len_kv": 8704, - "num_heads": 32, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 16, - "seq_len_q": 8192, - "seq_len_kv": 8192, - "num_heads": 32, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 16, - "seq_len_q": 1, - "seq_len_kv": 8704, - "num_heads": 32, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 32, - "seq_len_q": 8192, - "seq_len_kv": 8192, - "num_heads": 32, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 32, - "seq_len_q": 1, - "seq_len_kv": 8704, - "num_heads": 32, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 64, - "seq_len_q": 8192, - "seq_len_kv": 8192, - "num_heads": 32, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 64, - "seq_len_q": 1, - "seq_len_kv": 8704, - "num_heads": 32, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 128, - "seq_len_q": 8192, - "seq_len_kv": 8192, - "num_heads": 32, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 128, - "seq_len_q": 1, - "seq_len_kv": 8704, - "num_heads": 32, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 1, - "seq_len_q": 8192, - "seq_len_kv": 8192, - "num_heads": 32, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 1, - "seq_len_q": 1, - "seq_len_kv": 8704, - "num_heads": 32, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 2, - "seq_len_q": 8192, - "seq_len_kv": 8192, - "num_heads": 32, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 2, - "seq_len_q": 1, - "seq_len_kv": 8704, - "num_heads": 32, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 24, - "seq_len_q": 8192, - "seq_len_kv": 8192, - "num_heads": 32, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 24, - "seq_len_q": 1, - "seq_len_kv": 8704, - "num_heads": 32, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 48, - "seq_len_q": 8192, - "seq_len_kv": 8192, - "num_heads": 32, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 48, - "seq_len_q": 1, - "seq_len_kv": 8704, - "num_heads": 32, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 256, - "seq_len_q": 8192, - "seq_len_kv": 8192, - "num_heads": 32, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 256, - "seq_len_q": 1, - "seq_len_kv": 8704, - "num_heads": 32, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 12, - "seq_len_q": 8192, - "seq_len_kv": 8192, - "num_heads": 16, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 12, - "seq_len_q": 1, - "seq_len_kv": 8704, - "num_heads": 16, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 64, - "seq_len_q": 8192, - "seq_len_kv": 8192, - "num_heads": 16, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 64, - "seq_len_q": 1, - "seq_len_kv": 8704, - "num_heads": 16, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 128, - "seq_len_q": 8192, - "seq_len_kv": 8192, - "num_heads": 16, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 128, - "seq_len_q": 1, - "seq_len_kv": 8704, - "num_heads": 16, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 256, - "seq_len_q": 8192, - "seq_len_kv": 8192, - "num_heads": 16, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 256, - "seq_len_q": 1, - "seq_len_kv": 8704, - "num_heads": 16, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 512, - "seq_len_q": 8192, - "seq_len_kv": 8192, - "num_heads": 16, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 512, - "seq_len_q": 1, - "seq_len_kv": 8704, - "num_heads": 16, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 1, - "seq_len_q": 1024, - "seq_len_kv": 1024, - "num_heads": 16, - "head_dim_qk_nope": 448, - "head_dim_qk_rope": 64, - "head_dim_v": 512, - "kv_lora_rank": 1024, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsv4" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 1, - "seq_len_q": 1, - "seq_len_kv": 1536, - "num_heads": 16, - "head_dim_qk_nope": 448, - "head_dim_qk_rope": 64, - "head_dim_v": 512, - "kv_lora_rank": 1024, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsv4" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 2, - "seq_len_q": 1024, - "seq_len_kv": 1024, - "num_heads": 16, - "head_dim_qk_nope": 448, - "head_dim_qk_rope": 64, - "head_dim_v": 512, - "kv_lora_rank": 1024, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsv4" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 2, - "seq_len_q": 1, - "seq_len_kv": 1536, - "num_heads": 16, - "head_dim_qk_nope": 448, - "head_dim_qk_rope": 64, - "head_dim_v": 512, - "kv_lora_rank": 1024, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsv4" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 4, - "seq_len_q": 1024, - "seq_len_kv": 1024, - "num_heads": 16, - "head_dim_qk_nope": 448, - "head_dim_qk_rope": 64, - "head_dim_v": 512, - "kv_lora_rank": 1024, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsv4" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 4, - "seq_len_q": 1, - "seq_len_kv": 1536, - "num_heads": 16, - "head_dim_qk_nope": 448, - "head_dim_qk_rope": 64, - "head_dim_v": 512, - "kv_lora_rank": 1024, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsv4" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 8, - "seq_len_q": 1024, - "seq_len_kv": 1024, - "num_heads": 16, - "head_dim_qk_nope": 448, - "head_dim_qk_rope": 64, - "head_dim_v": 512, - "kv_lora_rank": 1024, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsv4" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 8, - "seq_len_q": 1, - "seq_len_kv": 1536, - "num_heads": 16, - "head_dim_qk_nope": 448, - "head_dim_qk_rope": 64, - "head_dim_v": 512, - "kv_lora_rank": 1024, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsv4" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 16, - "seq_len_q": 1024, - "seq_len_kv": 1024, - "num_heads": 16, - "head_dim_qk_nope": 448, - "head_dim_qk_rope": 64, - "head_dim_v": 512, - "kv_lora_rank": 1024, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsv4" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 16, - "seq_len_q": 1, - "seq_len_kv": 1536, - "num_heads": 16, - "head_dim_qk_nope": 448, - "head_dim_qk_rope": 64, - "head_dim_v": 512, - "kv_lora_rank": 1024, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsv4" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 32, - "seq_len_q": 1024, - "seq_len_kv": 1024, - "num_heads": 16, - "head_dim_qk_nope": 448, - "head_dim_qk_rope": 64, - "head_dim_v": 512, - "kv_lora_rank": 1024, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsv4" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 32, - "seq_len_q": 1, - "seq_len_kv": 1536, - "num_heads": 16, - "head_dim_qk_nope": 448, - "head_dim_qk_rope": 64, - "head_dim_v": 512, - "kv_lora_rank": 1024, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsv4" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 8, - "seq_len_q": 1024, - "seq_len_kv": 1024, - "num_heads": 128, - "head_dim_qk_nope": 448, - "head_dim_qk_rope": 64, - "head_dim_v": 512, - "kv_lora_rank": 1024, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsv4" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 8, - "seq_len_q": 1, - "seq_len_kv": 1536, - "num_heads": 128, - "head_dim_qk_nope": 448, - "head_dim_qk_rope": 64, - "head_dim_v": 512, - "kv_lora_rank": 1024, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsv4" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 16, - "seq_len_q": 1024, - "seq_len_kv": 1024, - "num_heads": 128, - "head_dim_qk_nope": 448, - "head_dim_qk_rope": 64, - "head_dim_v": 512, - "kv_lora_rank": 1024, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsv4" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 16, - "seq_len_q": 1, - "seq_len_kv": 1536, - "num_heads": 128, - "head_dim_qk_nope": 448, - "head_dim_qk_rope": 64, - "head_dim_v": 512, - "kv_lora_rank": 1024, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsv4" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 32, - "seq_len_q": 1024, - "seq_len_kv": 1024, - "num_heads": 128, - "head_dim_qk_nope": 448, - "head_dim_qk_rope": 64, - "head_dim_v": 512, - "kv_lora_rank": 1024, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsv4" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 32, - "seq_len_q": 1, - "seq_len_kv": 1536, - "num_heads": 128, - "head_dim_qk_nope": 448, - "head_dim_qk_rope": 64, - "head_dim_v": 512, - "kv_lora_rank": 1024, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsv4" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 64, - "seq_len_q": 1024, - "seq_len_kv": 1024, - "num_heads": 128, - "head_dim_qk_nope": 448, - "head_dim_qk_rope": 64, - "head_dim_v": 512, - "kv_lora_rank": 1024, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsv4" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 64, - "seq_len_q": 1, - "seq_len_kv": 1536, - "num_heads": 128, - "head_dim_qk_nope": 448, - "head_dim_qk_rope": 64, - "head_dim_v": 512, - "kv_lora_rank": 1024, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsv4" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 128, - "seq_len_q": 1024, - "seq_len_kv": 1024, - "num_heads": 128, - "head_dim_qk_nope": 448, - "head_dim_qk_rope": 64, - "head_dim_v": 512, - "kv_lora_rank": 1024, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsv4" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 128, - "seq_len_q": 1, - "seq_len_kv": 1536, - "num_heads": 128, - "head_dim_qk_nope": 448, - "head_dim_qk_rope": 64, - "head_dim_v": 512, - "kv_lora_rank": 1024, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsv4" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 1, - "seq_len_q": 8192, - "seq_len_kv": 8192, - "num_heads": 16, - "head_dim_qk_nope": 448, - "head_dim_qk_rope": 64, - "head_dim_v": 512, - "kv_lora_rank": 1024, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsv4" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 1, - "seq_len_q": 1, - "seq_len_kv": 8704, - "num_heads": 16, - "head_dim_qk_nope": 448, - "head_dim_qk_rope": 64, - "head_dim_v": 512, - "kv_lora_rank": 1024, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsv4" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 2, - "seq_len_q": 8192, - "seq_len_kv": 8192, - "num_heads": 16, - "head_dim_qk_nope": 448, - "head_dim_qk_rope": 64, - "head_dim_v": 512, - "kv_lora_rank": 1024, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsv4" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 2, - "seq_len_q": 1, - "seq_len_kv": 8704, - "num_heads": 16, - "head_dim_qk_nope": 448, - "head_dim_qk_rope": 64, - "head_dim_v": 512, - "kv_lora_rank": 1024, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsv4" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 4, - "seq_len_q": 8192, - "seq_len_kv": 8192, - "num_heads": 16, - "head_dim_qk_nope": 448, - "head_dim_qk_rope": 64, - "head_dim_v": 512, - "kv_lora_rank": 1024, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsv4" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 4, - "seq_len_q": 1, - "seq_len_kv": 8704, - "num_heads": 16, - "head_dim_qk_nope": 448, - "head_dim_qk_rope": 64, - "head_dim_v": 512, - "kv_lora_rank": 1024, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsv4" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 8, - "seq_len_q": 8192, - "seq_len_kv": 8192, - "num_heads": 16, - "head_dim_qk_nope": 448, - "head_dim_qk_rope": 64, - "head_dim_v": 512, - "kv_lora_rank": 1024, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsv4" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 8, - "seq_len_q": 1, - "seq_len_kv": 8704, - "num_heads": 16, - "head_dim_qk_nope": 448, - "head_dim_qk_rope": 64, - "head_dim_v": 512, - "kv_lora_rank": 1024, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsv4" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 16, - "seq_len_q": 8192, - "seq_len_kv": 8192, - "num_heads": 16, - "head_dim_qk_nope": 448, - "head_dim_qk_rope": 64, - "head_dim_v": 512, - "kv_lora_rank": 1024, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsv4" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 16, - "seq_len_q": 1, - "seq_len_kv": 8704, - "num_heads": 16, - "head_dim_qk_nope": 448, - "head_dim_qk_rope": 64, - "head_dim_v": 512, - "kv_lora_rank": 1024, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsv4" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 32, - "seq_len_q": 8192, - "seq_len_kv": 8192, - "num_heads": 16, - "head_dim_qk_nope": 448, - "head_dim_qk_rope": 64, - "head_dim_v": 512, - "kv_lora_rank": 1024, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsv4" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 32, - "seq_len_q": 1, - "seq_len_kv": 8704, - "num_heads": 16, - "head_dim_qk_nope": 448, - "head_dim_qk_rope": 64, - "head_dim_v": 512, - "kv_lora_rank": 1024, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsv4" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 8, - "seq_len_q": 8192, - "seq_len_kv": 8192, - "num_heads": 128, - "head_dim_qk_nope": 448, - "head_dim_qk_rope": 64, - "head_dim_v": 512, - "kv_lora_rank": 1024, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsv4" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 8, - "seq_len_q": 1, - "seq_len_kv": 8704, - "num_heads": 128, - "head_dim_qk_nope": 448, - "head_dim_qk_rope": 64, - "head_dim_v": 512, - "kv_lora_rank": 1024, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsv4" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 16, - "seq_len_q": 8192, - "seq_len_kv": 8192, - "num_heads": 128, - "head_dim_qk_nope": 448, - "head_dim_qk_rope": 64, - "head_dim_v": 512, - "kv_lora_rank": 1024, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsv4" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 16, - "seq_len_q": 1, - "seq_len_kv": 8704, - "num_heads": 128, - "head_dim_qk_nope": 448, - "head_dim_qk_rope": 64, - "head_dim_v": 512, - "kv_lora_rank": 1024, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsv4" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 32, - "seq_len_q": 8192, - "seq_len_kv": 8192, - "num_heads": 128, - "head_dim_qk_nope": 448, - "head_dim_qk_rope": 64, - "head_dim_v": 512, - "kv_lora_rank": 1024, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsv4" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 32, - "seq_len_q": 1, - "seq_len_kv": 8704, - "num_heads": 128, - "head_dim_qk_nope": 448, - "head_dim_qk_rope": 64, - "head_dim_v": 512, - "kv_lora_rank": 1024, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsv4" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 64, - "seq_len_q": 8192, - "seq_len_kv": 8192, - "num_heads": 128, - "head_dim_qk_nope": 448, - "head_dim_qk_rope": 64, - "head_dim_v": 512, - "kv_lora_rank": 1024, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsv4" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 64, - "seq_len_q": 1, - "seq_len_kv": 8704, - "num_heads": 128, - "head_dim_qk_nope": 448, - "head_dim_qk_rope": 64, - "head_dim_v": 512, - "kv_lora_rank": 1024, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsv4" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 64, - "seq_len_q": 1024, - "seq_len_kv": 1024, - "num_heads": 16, - "head_dim_qk_nope": 448, - "head_dim_qk_rope": 64, - "head_dim_v": 512, - "kv_lora_rank": 1024, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsv4" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 64, - "seq_len_q": 1, - "seq_len_kv": 1536, - "num_heads": 16, - "head_dim_qk_nope": 448, - "head_dim_qk_rope": 64, - "head_dim_v": 512, - "kv_lora_rank": 1024, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsv4" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 128, - "seq_len_q": 1024, - "seq_len_kv": 1024, - "num_heads": 16, - "head_dim_qk_nope": 448, - "head_dim_qk_rope": 64, - "head_dim_v": 512, - "kv_lora_rank": 1024, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsv4" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 128, - "seq_len_q": 1, - "seq_len_kv": 1536, - "num_heads": 16, - "head_dim_qk_nope": 448, - "head_dim_qk_rope": 64, - "head_dim_v": 512, - "kv_lora_rank": 1024, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsv4" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 256, - "seq_len_q": 1024, - "seq_len_kv": 1024, - "num_heads": 128, - "head_dim_qk_nope": 448, - "head_dim_qk_rope": 64, - "head_dim_v": 512, - "kv_lora_rank": 1024, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsv4" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 256, - "seq_len_q": 1, - "seq_len_kv": 1536, - "num_heads": 128, - "head_dim_qk_nope": 448, - "head_dim_qk_rope": 64, - "head_dim_v": 512, - "kv_lora_rank": 1024, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsv4" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 512, - "seq_len_q": 1024, - "seq_len_kv": 1024, - "num_heads": 128, - "head_dim_qk_nope": 448, - "head_dim_qk_rope": 64, - "head_dim_v": 512, - "kv_lora_rank": 1024, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsv4" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 512, - "seq_len_q": 1, - "seq_len_kv": 1536, - "num_heads": 128, - "head_dim_qk_nope": 448, - "head_dim_qk_rope": 64, - "head_dim_v": 512, - "kv_lora_rank": 1024, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsv4" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 128, - "seq_len_q": 8192, - "seq_len_kv": 8192, - "num_heads": 128, - "head_dim_qk_nope": 448, - "head_dim_qk_rope": 64, - "head_dim_v": 512, - "kv_lora_rank": 1024, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsv4" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 128, - "seq_len_q": 1, - "seq_len_kv": 8704, - "num_heads": 128, - "head_dim_qk_nope": 448, - "head_dim_qk_rope": 64, - "head_dim_v": 512, - "kv_lora_rank": 1024, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsv4" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 4, - "seq_len_q": 1024, - "seq_len_kv": 1024, - "num_heads": 128, - "head_dim_qk_nope": 448, - "head_dim_qk_rope": 64, - "head_dim_v": 512, - "kv_lora_rank": 1024, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsv4" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 4, - "seq_len_q": 1, - "seq_len_kv": 1536, - "num_heads": 128, - "head_dim_qk_nope": 448, - "head_dim_qk_rope": 64, - "head_dim_v": 512, - "kv_lora_rank": 1024, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsv4" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 4, - "seq_len_q": 8192, - "seq_len_kv": 8192, - "num_heads": 128, - "head_dim_qk_nope": 448, - "head_dim_qk_rope": 64, - "head_dim_v": 512, - "kv_lora_rank": 1024, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsv4" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 4, - "seq_len_q": 1, - "seq_len_kv": 8704, - "num_heads": 128, - "head_dim_qk_nope": 448, - "head_dim_qk_rope": 64, - "head_dim_v": 512, - "kv_lora_rank": 1024, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsv4" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 1, - "seq_len_q": 2, - "seq_len_kv": 1536, - "num_heads": 16, - "head_dim_qk_nope": 448, - "head_dim_qk_rope": 64, - "head_dim_v": 512, - "kv_lora_rank": 1024, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsv4" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 2, - "seq_len_q": 2, - "seq_len_kv": 1536, - "num_heads": 16, - "head_dim_qk_nope": 448, - "head_dim_qk_rope": 64, - "head_dim_v": 512, - "kv_lora_rank": 1024, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsv4" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 4, - "seq_len_q": 2, - "seq_len_kv": 1536, - "num_heads": 16, - "head_dim_qk_nope": 448, - "head_dim_qk_rope": 64, - "head_dim_v": 512, - "kv_lora_rank": 1024, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsv4" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 8, - "seq_len_q": 2, - "seq_len_kv": 1536, - "num_heads": 16, - "head_dim_qk_nope": 448, - "head_dim_qk_rope": 64, - "head_dim_v": 512, - "kv_lora_rank": 1024, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsv4" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 16, - "seq_len_q": 2, - "seq_len_kv": 1536, - "num_heads": 16, - "head_dim_qk_nope": 448, - "head_dim_qk_rope": 64, - "head_dim_v": 512, - "kv_lora_rank": 1024, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsv4" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 32, - "seq_len_q": 2, - "seq_len_kv": 1536, - "num_heads": 16, - "head_dim_qk_nope": 448, - "head_dim_qk_rope": 64, - "head_dim_v": 512, - "kv_lora_rank": 1024, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsv4" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 4, - "seq_len_q": 2, - "seq_len_kv": 1536, - "num_heads": 128, - "head_dim_qk_nope": 448, - "head_dim_qk_rope": 64, - "head_dim_v": 512, - "kv_lora_rank": 1024, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsv4" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 8, - "seq_len_q": 2, - "seq_len_kv": 1536, - "num_heads": 128, - "head_dim_qk_nope": 448, - "head_dim_qk_rope": 64, - "head_dim_v": 512, - "kv_lora_rank": 1024, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsv4" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 16, - "seq_len_q": 2, - "seq_len_kv": 1536, - "num_heads": 128, - "head_dim_qk_nope": 448, - "head_dim_qk_rope": 64, - "head_dim_v": 512, - "kv_lora_rank": 1024, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsv4" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 32, - "seq_len_q": 2, - "seq_len_kv": 1536, - "num_heads": 128, - "head_dim_qk_nope": 448, - "head_dim_qk_rope": 64, - "head_dim_v": 512, - "kv_lora_rank": 1024, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsv4" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 64, - "seq_len_q": 2, - "seq_len_kv": 1536, - "num_heads": 128, - "head_dim_qk_nope": 448, - "head_dim_qk_rope": 64, - "head_dim_v": 512, - "kv_lora_rank": 1024, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsv4" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 1, - "seq_len_q": 2, - "seq_len_kv": 8704, - "num_heads": 16, - "head_dim_qk_nope": 448, - "head_dim_qk_rope": 64, - "head_dim_v": 512, - "kv_lora_rank": 1024, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsv4" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 2, - "seq_len_q": 2, - "seq_len_kv": 8704, - "num_heads": 16, - "head_dim_qk_nope": 448, - "head_dim_qk_rope": 64, - "head_dim_v": 512, - "kv_lora_rank": 1024, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsv4" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 4, - "seq_len_q": 2, - "seq_len_kv": 8704, - "num_heads": 16, - "head_dim_qk_nope": 448, - "head_dim_qk_rope": 64, - "head_dim_v": 512, - "kv_lora_rank": 1024, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsv4" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 8, - "seq_len_q": 2, - "seq_len_kv": 8704, - "num_heads": 16, - "head_dim_qk_nope": 448, - "head_dim_qk_rope": 64, - "head_dim_v": 512, - "kv_lora_rank": 1024, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsv4" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 16, - "seq_len_q": 2, - "seq_len_kv": 8704, - "num_heads": 16, - "head_dim_qk_nope": 448, - "head_dim_qk_rope": 64, - "head_dim_v": 512, - "kv_lora_rank": 1024, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsv4" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 32, - "seq_len_q": 2, - "seq_len_kv": 8704, - "num_heads": 16, - "head_dim_qk_nope": 448, - "head_dim_qk_rope": 64, - "head_dim_v": 512, - "kv_lora_rank": 1024, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsv4" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 4, - "seq_len_q": 2, - "seq_len_kv": 8704, - "num_heads": 128, - "head_dim_qk_nope": 448, - "head_dim_qk_rope": 64, - "head_dim_v": 512, - "kv_lora_rank": 1024, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsv4" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 8, - "seq_len_q": 2, - "seq_len_kv": 8704, - "num_heads": 128, - "head_dim_qk_nope": 448, - "head_dim_qk_rope": 64, - "head_dim_v": 512, - "kv_lora_rank": 1024, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsv4" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 16, - "seq_len_q": 2, - "seq_len_kv": 8704, - "num_heads": 128, - "head_dim_qk_nope": 448, - "head_dim_qk_rope": 64, - "head_dim_v": 512, - "kv_lora_rank": 1024, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsv4" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 64, - "seq_len_q": 2, - "seq_len_kv": 1536, - "num_heads": 16, - "head_dim_qk_nope": 448, - "head_dim_qk_rope": 64, - "head_dim_v": 512, - "kv_lora_rank": 1024, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsv4" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 128, - "seq_len_q": 2, - "seq_len_kv": 1536, - "num_heads": 16, - "head_dim_qk_nope": 448, - "head_dim_qk_rope": 64, - "head_dim_v": 512, - "kv_lora_rank": 1024, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsv4" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 128, - "seq_len_q": 2, - "seq_len_kv": 1536, - "num_heads": 128, - "head_dim_qk_nope": 448, - "head_dim_qk_rope": 64, - "head_dim_v": 512, - "kv_lora_rank": 1024, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsv4" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 256, - "seq_len_q": 2, - "seq_len_kv": 1536, - "num_heads": 128, - "head_dim_qk_nope": 448, - "head_dim_qk_rope": 64, - "head_dim_v": 512, - "kv_lora_rank": 1024, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsv4" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 512, - "seq_len_q": 2, - "seq_len_kv": 1536, - "num_heads": 128, - "head_dim_qk_nope": 448, - "head_dim_qk_rope": 64, - "head_dim_v": 512, - "kv_lora_rank": 1024, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsv4" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 32, - "seq_len_q": 2, - "seq_len_kv": 8704, - "num_heads": 128, - "head_dim_qk_nope": 448, - "head_dim_qk_rope": 64, - "head_dim_v": 512, - "kv_lora_rank": 1024, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsv4" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 64, - "seq_len_q": 1024, - "seq_len_kv": 1024, - "num_heads": 128, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 64, - "seq_len_q": 1, - "seq_len_kv": 1536, - "num_heads": 128, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 16, - "seq_len_q": 1024, - "seq_len_kv": 1024, - "num_heads": 128, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 16, - "seq_len_q": 1, - "seq_len_kv": 1536, - "num_heads": 128, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 32, - "seq_len_q": 1024, - "seq_len_kv": 1024, - "num_heads": 128, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 32, - "seq_len_q": 1, - "seq_len_kv": 1536, - "num_heads": 128, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 64, - "seq_len_q": 8192, - "seq_len_kv": 8192, - "num_heads": 128, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 64, - "seq_len_q": 1, - "seq_len_kv": 8704, - "num_heads": 128, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 16, - "seq_len_q": 1, - "seq_len_kv": 8704, - "num_heads": 128, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 32, - "seq_len_q": 8192, - "seq_len_kv": 8192, - "num_heads": 128, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 32, - "seq_len_q": 1, - "seq_len_kv": 8704, - "num_heads": 128, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 4, - "seq_len_q": 2, - "seq_len_kv": 1536, - "num_heads": 32, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 8, - "seq_len_q": 2, - "seq_len_kv": 1536, - "num_heads": 32, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 64, - "seq_len_q": 2, - "seq_len_kv": 1536, - "num_heads": 128, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 4, - "seq_len_q": 2, - "seq_len_kv": 1536, - "num_heads": 16, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 128, - "seq_len_q": 2, - "seq_len_kv": 1536, - "num_heads": 16, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 32, - "seq_len_q": 2, - "seq_len_kv": 1536, - "num_heads": 16, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 64, - "seq_len_q": 2, - "seq_len_kv": 1536, - "num_heads": 16, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 4, - "seq_len_q": 1024, - "seq_len_kv": 1024, - "num_heads": 128, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 4, - "seq_len_q": 2, - "seq_len_kv": 1536, - "num_heads": 128, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 8, - "seq_len_q": 1024, - "seq_len_kv": 1024, - "num_heads": 128, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 8, - "seq_len_q": 2, - "seq_len_kv": 1536, - "num_heads": 128, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 4, - "seq_len_q": 2, - "seq_len_kv": 8704, - "num_heads": 32, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 8, - "seq_len_q": 2, - "seq_len_kv": 8704, - "num_heads": 32, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 16, - "seq_len_q": 2, - "seq_len_kv": 8704, - "num_heads": 32, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 32, - "seq_len_q": 2, - "seq_len_kv": 8704, - "num_heads": 32, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 64, - "seq_len_q": 2, - "seq_len_kv": 8704, - "num_heads": 128, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 4, - "seq_len_q": 2, - "seq_len_kv": 8704, - "num_heads": 16, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 8, - "seq_len_q": 2, - "seq_len_kv": 8704, - "num_heads": 128, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 32, - "seq_len_q": 2, - "seq_len_kv": 8704, - "num_heads": 128, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 8, - "seq_len_q": 1, - "seq_len_kv": 1536, - "num_heads": 16, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "fp8", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 16, - "seq_len_q": 1024, - "seq_len_kv": 1024, - "num_heads": 16, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "fp8", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 16, - "seq_len_q": 1, - "seq_len_kv": 1536, - "num_heads": 16, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "fp8", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 64, - "seq_len_q": 1, - "seq_len_kv": 1536, - "num_heads": 16, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "fp8", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 4, - "seq_len_q": 8192, - "seq_len_kv": 8192, - "num_heads": 16, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "fp8", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 4, - "seq_len_q": 1, - "seq_len_kv": 8704, - "num_heads": 16, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "fp8", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 4, - "seq_len_q": 8192, - "seq_len_kv": 8192, - "num_heads": 32, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "fp8", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 4, - "seq_len_q": 1, - "seq_len_kv": 8704, - "num_heads": 32, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "fp8", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 8, - "seq_len_q": 8192, - "seq_len_kv": 8192, - "num_heads": 32, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "fp8", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 8, - "seq_len_q": 1, - "seq_len_kv": 8704, - "num_heads": 32, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "fp8", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 16, - "seq_len_q": 8192, - "seq_len_kv": 8192, - "num_heads": 32, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "fp8", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 16, - "seq_len_q": 1, - "seq_len_kv": 8704, - "num_heads": 32, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "fp8", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 32, - "seq_len_q": 8192, - "seq_len_kv": 8192, - "num_heads": 32, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "fp8", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 32, - "seq_len_q": 1, - "seq_len_kv": 8704, - "num_heads": 32, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "fp8", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 32, - "seq_len_q": 1024, - "seq_len_kv": 1024, - "num_heads": 32, - "head_dim_qk_nope": 448, - "head_dim_qk_rope": 64, - "head_dim_v": 512, - "kv_lora_rank": 1024, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsv4" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 32, - "seq_len_q": 1, - "seq_len_kv": 1536, - "num_heads": 32, - "head_dim_qk_nope": 448, - "head_dim_qk_rope": 64, - "head_dim_v": 512, - "kv_lora_rank": 1024, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsv4" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 1024, - "seq_len_q": 1024, - "seq_len_kv": 1024, - "num_heads": 128, - "head_dim_qk_nope": 448, - "head_dim_qk_rope": 64, - "head_dim_v": 512, - "kv_lora_rank": 1024, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsv4" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 1024, - "seq_len_q": 1, - "seq_len_kv": 1536, - "num_heads": 128, - "head_dim_qk_nope": 448, - "head_dim_qk_rope": 64, - "head_dim_v": 512, - "kv_lora_rank": 1024, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsv4" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 32, - "seq_len_q": 8192, - "seq_len_kv": 8192, - "num_heads": 32, - "head_dim_qk_nope": 448, - "head_dim_qk_rope": 64, - "head_dim_v": 512, - "kv_lora_rank": 1024, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsv4" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 32, - "seq_len_q": 1, - "seq_len_kv": 8704, - "num_heads": 32, - "head_dim_qk_nope": 448, - "head_dim_qk_rope": 64, - "head_dim_v": 512, - "kv_lora_rank": 1024, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsv4" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 256, - "seq_len_q": 8192, - "seq_len_kv": 8192, - "num_heads": 128, - "head_dim_qk_nope": 448, - "head_dim_qk_rope": 64, - "head_dim_v": 512, - "kv_lora_rank": 1024, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsv4" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 256, - "seq_len_q": 1, - "seq_len_kv": 8704, - "num_heads": 128, - "head_dim_qk_nope": 448, - "head_dim_qk_rope": 64, - "head_dim_v": 512, - "kv_lora_rank": 1024, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsv4" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 512, - "seq_len_q": 8192, - "seq_len_kv": 8192, - "num_heads": 128, - "head_dim_qk_nope": 448, - "head_dim_qk_rope": 64, - "head_dim_v": 512, - "kv_lora_rank": 1024, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsv4" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 512, - "seq_len_q": 1, - "seq_len_kv": 8704, - "num_heads": 128, - "head_dim_qk_nope": 448, - "head_dim_qk_rope": 64, - "head_dim_v": 512, - "kv_lora_rank": 1024, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsv4" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 4, - "seq_len_q": 1024, - "seq_len_kv": 1024, - "num_heads": 32, - "head_dim_qk_nope": 448, - "head_dim_qk_rope": 64, - "head_dim_v": 512, - "kv_lora_rank": 1024, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsv4" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 4, - "seq_len_q": 2, - "seq_len_kv": 1536, - "num_heads": 32, - "head_dim_qk_nope": 448, - "head_dim_qk_rope": 64, - "head_dim_v": 512, - "kv_lora_rank": 1024, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsv4" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 8, - "seq_len_q": 1024, - "seq_len_kv": 1024, - "num_heads": 32, - "head_dim_qk_nope": 448, - "head_dim_qk_rope": 64, - "head_dim_v": 512, - "kv_lora_rank": 1024, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsv4" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 8, - "seq_len_q": 2, - "seq_len_kv": 1536, - "num_heads": 32, - "head_dim_qk_nope": 448, - "head_dim_qk_rope": 64, - "head_dim_v": 512, - "kv_lora_rank": 1024, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsv4" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 16, - "seq_len_q": 1024, - "seq_len_kv": 1024, - "num_heads": 32, - "head_dim_qk_nope": 448, - "head_dim_qk_rope": 64, - "head_dim_v": 512, - "kv_lora_rank": 1024, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsv4" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 16, - "seq_len_q": 2, - "seq_len_kv": 1536, - "num_heads": 32, - "head_dim_qk_nope": 448, - "head_dim_qk_rope": 64, - "head_dim_v": 512, - "kv_lora_rank": 1024, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsv4" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 32, - "seq_len_q": 2, - "seq_len_kv": 1536, - "num_heads": 32, - "head_dim_qk_nope": 448, - "head_dim_qk_rope": 64, - "head_dim_v": 512, - "kv_lora_rank": 1024, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsv4" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 4, - "seq_len_q": 8192, - "seq_len_kv": 8192, - "num_heads": 32, - "head_dim_qk_nope": 448, - "head_dim_qk_rope": 64, - "head_dim_v": 512, - "kv_lora_rank": 1024, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsv4" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 4, - "seq_len_q": 2, - "seq_len_kv": 8704, - "num_heads": 32, - "head_dim_qk_nope": 448, - "head_dim_qk_rope": 64, - "head_dim_v": 512, - "kv_lora_rank": 1024, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsv4" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 8, - "seq_len_q": 8192, - "seq_len_kv": 8192, - "num_heads": 32, - "head_dim_qk_nope": 448, - "head_dim_qk_rope": 64, - "head_dim_v": 512, - "kv_lora_rank": 1024, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsv4" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 8, - "seq_len_q": 2, - "seq_len_kv": 8704, - "num_heads": 32, - "head_dim_qk_nope": 448, - "head_dim_qk_rope": 64, - "head_dim_v": 512, - "kv_lora_rank": 1024, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsv4" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 16, - "seq_len_q": 8192, - "seq_len_kv": 8192, - "num_heads": 32, - "head_dim_qk_nope": 448, - "head_dim_qk_rope": 64, - "head_dim_v": 512, - "kv_lora_rank": 1024, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsv4" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 16, - "seq_len_q": 2, - "seq_len_kv": 8704, - "num_heads": 32, - "head_dim_qk_nope": 448, - "head_dim_qk_rope": 64, - "head_dim_v": 512, - "kv_lora_rank": 1024, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsv4" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 32, - "seq_len_q": 2, - "seq_len_kv": 8704, - "num_heads": 32, - "head_dim_qk_nope": 448, - "head_dim_qk_rope": 64, - "head_dim_v": 512, - "kv_lora_rank": 1024, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsv4" - }, - { - "type": "attention_mha", - "args": { - "batch_size": 4, - "seq_len_q": 1024, - "seq_len_kv": 1024, - "num_heads": 4, - "num_heads_kv": 1, - "head_dim": 256, - "dtype_q": "bf16", - "dtype_k": "bf16", - "dtype_v": "bf16", - "dtype_o": "bf16", - "causal": true - }, - "name": "qwen3.5" - }, - { - "type": "attention_mha", - "args": { - "batch_size": 4, - "seq_len_q": 1, - "seq_len_kv": 1536, - "num_heads": 4, - "num_heads_kv": 1, - "head_dim": 256, - "dtype_q": "bf16", - "dtype_k": "bf16", - "dtype_v": "bf16", - "dtype_o": "bf16", - "causal": true - }, - "name": "qwen3.5" - }, - { - "type": "attention_mha", - "args": { - "batch_size": 8, - "seq_len_q": 1024, - "seq_len_kv": 1024, - "num_heads": 4, - "num_heads_kv": 1, - "head_dim": 256, - "dtype_q": "bf16", - "dtype_k": "bf16", - "dtype_v": "bf16", - "dtype_o": "bf16", - "causal": true - }, - "name": "qwen3.5" - }, - { - "type": "attention_mha", - "args": { - "batch_size": 8, - "seq_len_q": 1, - "seq_len_kv": 1536, - "num_heads": 4, - "num_heads_kv": 1, - "head_dim": 256, - "dtype_q": "bf16", - "dtype_k": "bf16", - "dtype_v": "bf16", - "dtype_o": "bf16", - "causal": true - }, - "name": "qwen3.5" - }, - { - "type": "attention_mha", - "args": { - "batch_size": 16, - "seq_len_q": 1024, - "seq_len_kv": 1024, - "num_heads": 4, - "num_heads_kv": 1, - "head_dim": 256, - "dtype_q": "bf16", - "dtype_k": "bf16", - "dtype_v": "bf16", - "dtype_o": "bf16", - "causal": true - }, - "name": "qwen3.5" - }, - { - "type": "attention_mha", - "args": { - "batch_size": 16, - "seq_len_q": 1, - "seq_len_kv": 1536, - "num_heads": 4, - "num_heads_kv": 1, - "head_dim": 256, - "dtype_q": "bf16", - "dtype_k": "bf16", - "dtype_v": "bf16", - "dtype_o": "bf16", - "causal": true - }, - "name": "qwen3.5" - }, - { - "type": "attention_mha", - "args": { - "batch_size": 32, - "seq_len_q": 1024, - "seq_len_kv": 1024, - "num_heads": 4, - "num_heads_kv": 1, - "head_dim": 256, - "dtype_q": "bf16", - "dtype_k": "bf16", - "dtype_v": "bf16", - "dtype_o": "bf16", - "causal": true - }, - "name": "qwen3.5" - }, - { - "type": "attention_mha", - "args": { - "batch_size": 32, - "seq_len_q": 1, - "seq_len_kv": 1536, - "num_heads": 4, - "num_heads_kv": 1, - "head_dim": 256, - "dtype_q": "bf16", - "dtype_k": "bf16", - "dtype_v": "bf16", - "dtype_o": "bf16", - "causal": true - }, - "name": "qwen3.5" - }, - { - "type": "attention_mha", - "args": { - "batch_size": 64, - "seq_len_q": 1024, - "seq_len_kv": 1024, - "num_heads": 4, - "num_heads_kv": 1, - "head_dim": 256, - "dtype_q": "bf16", - "dtype_k": "bf16", - "dtype_v": "bf16", - "dtype_o": "bf16", - "causal": true - }, - "name": "qwen3.5" - }, - { - "type": "attention_mha", - "args": { - "batch_size": 64, - "seq_len_q": 1, - "seq_len_kv": 1536, - "num_heads": 4, - "num_heads_kv": 1, - "head_dim": 256, - "dtype_q": "bf16", - "dtype_k": "bf16", - "dtype_v": "bf16", - "dtype_o": "bf16", - "causal": true - }, - "name": "qwen3.5" - }, - { - "type": "attention_mha", - "args": { - "batch_size": 4, - "seq_len_q": 8192, - "seq_len_kv": 8192, - "num_heads": 4, - "num_heads_kv": 1, - "head_dim": 256, - "dtype_q": "bf16", - "dtype_k": "bf16", - "dtype_v": "bf16", - "dtype_o": "bf16", - "causal": true - }, - "name": "qwen3.5" - }, - { - "type": "attention_mha", - "args": { - "batch_size": 4, - "seq_len_q": 1, - "seq_len_kv": 8704, - "num_heads": 4, - "num_heads_kv": 1, - "head_dim": 256, - "dtype_q": "bf16", - "dtype_k": "bf16", - "dtype_v": "bf16", - "dtype_o": "bf16", - "causal": true - }, - "name": "qwen3.5" - }, - { - "type": "attention_mha", - "args": { - "batch_size": 8, - "seq_len_q": 8192, - "seq_len_kv": 8192, - "num_heads": 4, - "num_heads_kv": 1, - "head_dim": 256, - "dtype_q": "bf16", - "dtype_k": "bf16", - "dtype_v": "bf16", - "dtype_o": "bf16", - "causal": true - }, - "name": "qwen3.5" - }, - { - "type": "attention_mha", - "args": { - "batch_size": 8, - "seq_len_q": 1, - "seq_len_kv": 8704, - "num_heads": 4, - "num_heads_kv": 1, - "head_dim": 256, - "dtype_q": "bf16", - "dtype_k": "bf16", - "dtype_v": "bf16", - "dtype_o": "bf16", - "causal": true - }, - "name": "qwen3.5" - }, - { - "type": "attention_mha", - "args": { - "batch_size": 16, - "seq_len_q": 8192, - "seq_len_kv": 8192, - "num_heads": 4, - "num_heads_kv": 1, - "head_dim": 256, - "dtype_q": "bf16", - "dtype_k": "bf16", - "dtype_v": "bf16", - "dtype_o": "bf16", - "causal": true - }, - "name": "qwen3.5" - }, - { - "type": "attention_mha", - "args": { - "batch_size": 16, - "seq_len_q": 1, - "seq_len_kv": 8704, - "num_heads": 4, - "num_heads_kv": 1, - "head_dim": 256, - "dtype_q": "bf16", - "dtype_k": "bf16", - "dtype_v": "bf16", - "dtype_o": "bf16", - "causal": true - }, - "name": "qwen3.5" - }, - { - "type": "attention_mha", - "args": { - "batch_size": 32, - "seq_len_q": 8192, - "seq_len_kv": 8192, - "num_heads": 4, - "num_heads_kv": 1, - "head_dim": 256, - "dtype_q": "bf16", - "dtype_k": "bf16", - "dtype_v": "bf16", - "dtype_o": "bf16", - "causal": true - }, - "name": "qwen3.5" - }, - { - "type": "attention_mha", - "args": { - "batch_size": 32, - "seq_len_q": 1, - "seq_len_kv": 8704, - "num_heads": 4, - "num_heads_kv": 1, - "head_dim": 256, - "dtype_q": "bf16", - "dtype_k": "bf16", - "dtype_v": "bf16", - "dtype_o": "bf16", - "causal": true - }, - "name": "qwen3.5" - }, - { - "type": "attention_mha", - "args": { - "batch_size": 64, - "seq_len_q": 8192, - "seq_len_kv": 8192, - "num_heads": 4, - "num_heads_kv": 1, - "head_dim": 256, - "dtype_q": "bf16", - "dtype_k": "bf16", - "dtype_v": "bf16", - "dtype_o": "bf16", - "causal": true - }, - "name": "qwen3.5" - }, - { - "type": "attention_mha", - "args": { - "batch_size": 64, - "seq_len_q": 1, - "seq_len_kv": 8704, - "num_heads": 4, - "num_heads_kv": 1, - "head_dim": 256, - "dtype_q": "bf16", - "dtype_k": "bf16", - "dtype_v": "bf16", - "dtype_o": "bf16", - "causal": true - }, - "name": "qwen3.5" - }, - { - "type": "attention_mha", - "args": { - "batch_size": 4, - "seq_len_q": 2, - "seq_len_kv": 1536, - "num_heads": 4, - "num_heads_kv": 1, - "head_dim": 256, - "dtype_q": "bf16", - "dtype_k": "bf16", - "dtype_v": "bf16", - "dtype_o": "bf16", - "causal": true - }, - "name": "qwen3.5" - }, - { - "type": "attention_mha", - "args": { - "batch_size": 8, - "seq_len_q": 2, - "seq_len_kv": 1536, - "num_heads": 4, - "num_heads_kv": 1, - "head_dim": 256, - "dtype_q": "bf16", - "dtype_k": "bf16", - "dtype_v": "bf16", - "dtype_o": "bf16", - "causal": true - }, - "name": "qwen3.5" - }, - { - "type": "attention_mha", - "args": { - "batch_size": 16, - "seq_len_q": 2, - "seq_len_kv": 1536, - "num_heads": 4, - "num_heads_kv": 1, - "head_dim": 256, - "dtype_q": "bf16", - "dtype_k": "bf16", - "dtype_v": "bf16", - "dtype_o": "bf16", - "causal": true - }, - "name": "qwen3.5" - }, - { - "type": "attention_mha", - "args": { - "batch_size": 32, - "seq_len_q": 2, - "seq_len_kv": 1536, - "num_heads": 4, - "num_heads_kv": 1, - "head_dim": 256, - "dtype_q": "bf16", - "dtype_k": "bf16", - "dtype_v": "bf16", - "dtype_o": "bf16", - "causal": true - }, - "name": "qwen3.5" - }, - { - "type": "attention_mha", - "args": { - "batch_size": 64, - "seq_len_q": 2, - "seq_len_kv": 1536, - "num_heads": 4, - "num_heads_kv": 1, - "head_dim": 256, - "dtype_q": "bf16", - "dtype_k": "bf16", - "dtype_v": "bf16", - "dtype_o": "bf16", - "causal": true - }, - "name": "qwen3.5" - }, - { - "type": "attention_mha", - "args": { - "batch_size": 4, - "seq_len_q": 2, - "seq_len_kv": 8704, - "num_heads": 4, - "num_heads_kv": 1, - "head_dim": 256, - "dtype_q": "bf16", - "dtype_k": "bf16", - "dtype_v": "bf16", - "dtype_o": "bf16", - "causal": true - }, - "name": "qwen3.5" - }, - { - "type": "attention_mha", - "args": { - "batch_size": 8, - "seq_len_q": 2, - "seq_len_kv": 8704, - "num_heads": 4, - "num_heads_kv": 1, - "head_dim": 256, - "dtype_q": "bf16", - "dtype_k": "bf16", - "dtype_v": "bf16", - "dtype_o": "bf16", - "causal": true - }, - "name": "qwen3.5" - }, - { - "type": "attention_mha", - "args": { - "batch_size": 16, - "seq_len_q": 2, - "seq_len_kv": 8704, - "num_heads": 4, - "num_heads_kv": 1, - "head_dim": 256, - "dtype_q": "bf16", - "dtype_k": "bf16", - "dtype_v": "bf16", - "dtype_o": "bf16", - "causal": true - }, - "name": "qwen3.5" - }, - { - "type": "attention_mha", - "args": { - "batch_size": 32, - "seq_len_q": 2, - "seq_len_kv": 8704, - "num_heads": 4, - "num_heads_kv": 1, - "head_dim": 256, - "dtype_q": "bf16", - "dtype_k": "bf16", - "dtype_v": "bf16", - "dtype_o": "bf16", - "causal": true - }, - "name": "qwen3.5" - }, - { - "type": "attention_mha", - "args": { - "batch_size": 64, - "seq_len_q": 2, - "seq_len_kv": 8704, - "num_heads": 4, - "num_heads_kv": 1, - "head_dim": 256, - "dtype_q": "bf16", - "dtype_k": "bf16", - "dtype_v": "bf16", - "dtype_o": "bf16", - "causal": true - }, - "name": "qwen3.5" - }, - { - "type": "attention_mha", - "args": { - "batch_size": 4, - "seq_len_q": 1024, - "seq_len_kv": 1024, - "num_heads": 4, - "num_heads_kv": 1, - "head_dim": 256, - "dtype_q": "fp8", - "dtype_k": "fp8", - "dtype_v": "fp8", - "dtype_o": "bf16", - "causal": true - }, - "name": "qwen3.5" - }, - { - "type": "attention_mha", - "args": { - "batch_size": 4, - "seq_len_q": 1, - "seq_len_kv": 1536, - "num_heads": 4, - "num_heads_kv": 1, - "head_dim": 256, - "dtype_q": "fp8", - "dtype_k": "fp8", - "dtype_v": "fp8", - "dtype_o": "bf16", - "causal": true - }, - "name": "qwen3.5" - }, - { - "type": "attention_mha", - "args": { - "batch_size": 4, - "seq_len_q": 1024, - "seq_len_kv": 1024, - "num_heads": 8, - "num_heads_kv": 1, - "head_dim": 256, - "dtype_q": "fp8", - "dtype_k": "fp8", - "dtype_v": "fp8", - "dtype_o": "bf16", - "causal": true - }, - "name": "qwen3.5" - }, - { - "type": "attention_mha", - "args": { - "batch_size": 4, - "seq_len_q": 1, - "seq_len_kv": 1536, - "num_heads": 8, - "num_heads_kv": 1, - "head_dim": 256, - "dtype_q": "fp8", - "dtype_k": "fp8", - "dtype_v": "fp8", - "dtype_o": "bf16", - "causal": true - }, - "name": "qwen3.5" - }, - { - "type": "attention_mha", - "args": { - "batch_size": 8, - "seq_len_q": 1024, - "seq_len_kv": 1024, - "num_heads": 8, - "num_heads_kv": 1, - "head_dim": 256, - "dtype_q": "fp8", - "dtype_k": "fp8", - "dtype_v": "fp8", - "dtype_o": "bf16", - "causal": true - }, - "name": "qwen3.5" - }, - { - "type": "attention_mha", - "args": { - "batch_size": 8, - "seq_len_q": 1, - "seq_len_kv": 1536, - "num_heads": 8, - "num_heads_kv": 1, - "head_dim": 256, - "dtype_q": "fp8", - "dtype_k": "fp8", - "dtype_v": "fp8", - "dtype_o": "bf16", - "causal": true - }, - "name": "qwen3.5" - }, - { - "type": "attention_mha", - "args": { - "batch_size": 16, - "seq_len_q": 1024, - "seq_len_kv": 1024, - "num_heads": 8, - "num_heads_kv": 1, - "head_dim": 256, - "dtype_q": "fp8", - "dtype_k": "fp8", - "dtype_v": "fp8", - "dtype_o": "bf16", - "causal": true - }, - "name": "qwen3.5" - }, - { - "type": "attention_mha", - "args": { - "batch_size": 16, - "seq_len_q": 1, - "seq_len_kv": 1536, - "num_heads": 8, - "num_heads_kv": 1, - "head_dim": 256, - "dtype_q": "fp8", - "dtype_k": "fp8", - "dtype_v": "fp8", - "dtype_o": "bf16", - "causal": true - }, - "name": "qwen3.5" - }, - { - "type": "attention_mha", - "args": { - "batch_size": 32, - "seq_len_q": 1024, - "seq_len_kv": 1024, - "num_heads": 8, - "num_heads_kv": 1, - "head_dim": 256, - "dtype_q": "fp8", - "dtype_k": "fp8", - "dtype_v": "fp8", - "dtype_o": "bf16", - "causal": true - }, - "name": "qwen3.5" - }, - { - "type": "attention_mha", - "args": { - "batch_size": 32, - "seq_len_q": 1, - "seq_len_kv": 1536, - "num_heads": 8, - "num_heads_kv": 1, - "head_dim": 256, - "dtype_q": "fp8", - "dtype_k": "fp8", - "dtype_v": "fp8", - "dtype_o": "bf16", - "causal": true - }, - "name": "qwen3.5" - }, - { - "type": "attention_mha", - "args": { - "batch_size": 64, - "seq_len_q": 1024, - "seq_len_kv": 1024, - "num_heads": 8, - "num_heads_kv": 1, - "head_dim": 256, - "dtype_q": "fp8", - "dtype_k": "fp8", - "dtype_v": "fp8", - "dtype_o": "bf16", - "causal": true - }, - "name": "qwen3.5" - }, - { - "type": "attention_mha", - "args": { - "batch_size": 64, - "seq_len_q": 1, - "seq_len_kv": 1536, - "num_heads": 8, - "num_heads_kv": 1, - "head_dim": 256, - "dtype_q": "fp8", - "dtype_k": "fp8", - "dtype_v": "fp8", - "dtype_o": "bf16", - "causal": true - }, - "name": "qwen3.5" - }, - { - "type": "attention_mha", - "args": { - "batch_size": 128, - "seq_len_q": 1024, - "seq_len_kv": 1024, - "num_heads": 8, - "num_heads_kv": 1, - "head_dim": 256, - "dtype_q": "fp8", - "dtype_k": "fp8", - "dtype_v": "fp8", - "dtype_o": "bf16", - "causal": true - }, - "name": "qwen3.5" - }, - { - "type": "attention_mha", - "args": { - "batch_size": 128, - "seq_len_q": 1, - "seq_len_kv": 1536, - "num_heads": 8, - "num_heads_kv": 1, - "head_dim": 256, - "dtype_q": "fp8", - "dtype_k": "fp8", - "dtype_v": "fp8", - "dtype_o": "bf16", - "causal": true - }, - "name": "qwen3.5" - }, - { - "type": "attention_mha", - "args": { - "batch_size": 256, - "seq_len_q": 1024, - "seq_len_kv": 1024, - "num_heads": 8, - "num_heads_kv": 1, - "head_dim": 256, - "dtype_q": "fp8", - "dtype_k": "fp8", - "dtype_v": "fp8", - "dtype_o": "bf16", - "causal": true - }, - "name": "qwen3.5" - }, - { - "type": "attention_mha", - "args": { - "batch_size": 256, - "seq_len_q": 1, - "seq_len_kv": 1536, - "num_heads": 8, - "num_heads_kv": 1, - "head_dim": 256, - "dtype_q": "fp8", - "dtype_k": "fp8", - "dtype_v": "fp8", - "dtype_o": "bf16", - "causal": true - }, - "name": "qwen3.5" - }, - { - "type": "attention_mha", - "args": { - "batch_size": 4, - "seq_len_q": 8192, - "seq_len_kv": 8192, - "num_heads": 4, - "num_heads_kv": 1, - "head_dim": 256, - "dtype_q": "fp8", - "dtype_k": "fp8", - "dtype_v": "fp8", - "dtype_o": "bf16", - "causal": true - }, - "name": "qwen3.5" - }, - { - "type": "attention_mha", - "args": { - "batch_size": 4, - "seq_len_q": 1, - "seq_len_kv": 8704, - "num_heads": 4, - "num_heads_kv": 1, - "head_dim": 256, - "dtype_q": "fp8", - "dtype_k": "fp8", - "dtype_v": "fp8", - "dtype_o": "bf16", - "causal": true - }, - "name": "qwen3.5" - }, - { - "type": "attention_mha", - "args": { - "batch_size": 4, - "seq_len_q": 8192, - "seq_len_kv": 8192, - "num_heads": 8, - "num_heads_kv": 1, - "head_dim": 256, - "dtype_q": "fp8", - "dtype_k": "fp8", - "dtype_v": "fp8", - "dtype_o": "bf16", - "causal": true - }, - "name": "qwen3.5" - }, - { - "type": "attention_mha", - "args": { - "batch_size": 4, - "seq_len_q": 1, - "seq_len_kv": 8704, - "num_heads": 8, - "num_heads_kv": 1, - "head_dim": 256, - "dtype_q": "fp8", - "dtype_k": "fp8", - "dtype_v": "fp8", - "dtype_o": "bf16", - "causal": true - }, - "name": "qwen3.5" - }, - { - "type": "attention_mha", - "args": { - "batch_size": 8, - "seq_len_q": 8192, - "seq_len_kv": 8192, - "num_heads": 8, - "num_heads_kv": 1, - "head_dim": 256, - "dtype_q": "fp8", - "dtype_k": "fp8", - "dtype_v": "fp8", - "dtype_o": "bf16", - "causal": true - }, - "name": "qwen3.5" - }, - { - "type": "attention_mha", - "args": { - "batch_size": 8, - "seq_len_q": 1, - "seq_len_kv": 8704, - "num_heads": 8, - "num_heads_kv": 1, - "head_dim": 256, - "dtype_q": "fp8", - "dtype_k": "fp8", - "dtype_v": "fp8", - "dtype_o": "bf16", - "causal": true - }, - "name": "qwen3.5" - }, - { - "type": "attention_mha", - "args": { - "batch_size": 16, - "seq_len_q": 8192, - "seq_len_kv": 8192, - "num_heads": 8, - "num_heads_kv": 1, - "head_dim": 256, - "dtype_q": "fp8", - "dtype_k": "fp8", - "dtype_v": "fp8", - "dtype_o": "bf16", - "causal": true - }, - "name": "qwen3.5" - }, - { - "type": "attention_mha", - "args": { - "batch_size": 16, - "seq_len_q": 1, - "seq_len_kv": 8704, - "num_heads": 8, - "num_heads_kv": 1, - "head_dim": 256, - "dtype_q": "fp8", - "dtype_k": "fp8", - "dtype_v": "fp8", - "dtype_o": "bf16", - "causal": true - }, - "name": "qwen3.5" - }, - { - "type": "attention_mha", - "args": { - "batch_size": 32, - "seq_len_q": 8192, - "seq_len_kv": 8192, - "num_heads": 8, - "num_heads_kv": 1, - "head_dim": 256, - "dtype_q": "fp8", - "dtype_k": "fp8", - "dtype_v": "fp8", - "dtype_o": "bf16", - "causal": true - }, - "name": "qwen3.5" - }, - { - "type": "attention_mha", - "args": { - "batch_size": 32, - "seq_len_q": 1, - "seq_len_kv": 8704, - "num_heads": 8, - "num_heads_kv": 1, - "head_dim": 256, - "dtype_q": "fp8", - "dtype_k": "fp8", - "dtype_v": "fp8", - "dtype_o": "bf16", - "causal": true - }, - "name": "qwen3.5" - }, - { - "type": "attention_mha", - "args": { - "batch_size": 64, - "seq_len_q": 8192, - "seq_len_kv": 8192, - "num_heads": 8, - "num_heads_kv": 1, - "head_dim": 256, - "dtype_q": "fp8", - "dtype_k": "fp8", - "dtype_v": "fp8", - "dtype_o": "bf16", - "causal": true - }, - "name": "qwen3.5" - }, - { - "type": "attention_mha", - "args": { - "batch_size": 64, - "seq_len_q": 1, - "seq_len_kv": 8704, - "num_heads": 8, - "num_heads_kv": 1, - "head_dim": 256, - "dtype_q": "fp8", - "dtype_k": "fp8", - "dtype_v": "fp8", - "dtype_o": "bf16", - "causal": true - }, - "name": "qwen3.5" - }, - { - "type": "attention_mha", - "args": { - "batch_size": 128, - "seq_len_q": 8192, - "seq_len_kv": 8192, - "num_heads": 8, - "num_heads_kv": 1, - "head_dim": 256, - "dtype_q": "fp8", - "dtype_k": "fp8", - "dtype_v": "fp8", - "dtype_o": "bf16", - "causal": true - }, - "name": "qwen3.5" - }, - { - "type": "attention_mha", - "args": { - "batch_size": 128, - "seq_len_q": 1, - "seq_len_kv": 8704, - "num_heads": 8, - "num_heads_kv": 1, - "head_dim": 256, - "dtype_q": "fp8", - "dtype_k": "fp8", - "dtype_v": "fp8", - "dtype_o": "bf16", - "causal": true - }, - "name": "qwen3.5" - }, - { - "type": "attention_mha", - "args": { - "batch_size": 256, - "seq_len_q": 8192, - "seq_len_kv": 8192, - "num_heads": 8, - "num_heads_kv": 1, - "head_dim": 256, - "dtype_q": "fp8", - "dtype_k": "fp8", - "dtype_v": "fp8", - "dtype_o": "bf16", - "causal": true - }, - "name": "qwen3.5" - }, - { - "type": "attention_mha", - "args": { - "batch_size": 256, - "seq_len_q": 1, - "seq_len_kv": 8704, - "num_heads": 8, - "num_heads_kv": 1, - "head_dim": 256, - "dtype_q": "fp8", - "dtype_k": "fp8", - "dtype_v": "fp8", - "dtype_o": "bf16", - "causal": true - }, - "name": "qwen3.5" - }, - { - "type": "attention_mha", - "args": { - "batch_size": 4, - "seq_len_q": 1024, - "seq_len_kv": 1024, - "num_heads": 8, - "num_heads_kv": 1, - "head_dim": 256, - "dtype_q": "nvfp4", - "dtype_k": "fp8", - "dtype_v": "fp8", - "dtype_o": "bf16", - "causal": true - }, - "name": "qwen3.5" - }, - { - "type": "attention_mha", - "args": { - "batch_size": 4, - "seq_len_q": 1, - "seq_len_kv": 1536, - "num_heads": 8, - "num_heads_kv": 1, - "head_dim": 256, - "dtype_q": "nvfp4", - "dtype_k": "fp8", - "dtype_v": "fp8", - "dtype_o": "bf16", - "causal": true - }, - "name": "qwen3.5" - }, - { - "type": "attention_mha", - "args": { - "batch_size": 4, - "seq_len_q": 1024, - "seq_len_kv": 1024, - "num_heads": 16, - "num_heads_kv": 1, - "head_dim": 256, - "dtype_q": "nvfp4", - "dtype_k": "fp8", - "dtype_v": "fp8", - "dtype_o": "bf16", - "causal": true - }, - "name": "qwen3.5" - }, - { - "type": "attention_mha", - "args": { - "batch_size": 4, - "seq_len_q": 1, - "seq_len_kv": 1536, - "num_heads": 16, - "num_heads_kv": 1, - "head_dim": 256, - "dtype_q": "nvfp4", - "dtype_k": "fp8", - "dtype_v": "fp8", - "dtype_o": "bf16", - "causal": true - }, - "name": "qwen3.5" - }, - { - "type": "attention_mha", - "args": { - "batch_size": 8, - "seq_len_q": 1024, - "seq_len_kv": 1024, - "num_heads": 16, - "num_heads_kv": 1, - "head_dim": 256, - "dtype_q": "nvfp4", - "dtype_k": "fp8", - "dtype_v": "fp8", - "dtype_o": "bf16", - "causal": true - }, - "name": "qwen3.5" - }, - { - "type": "attention_mha", - "args": { - "batch_size": 8, - "seq_len_q": 1, - "seq_len_kv": 1536, - "num_heads": 16, - "num_heads_kv": 1, - "head_dim": 256, - "dtype_q": "nvfp4", - "dtype_k": "fp8", - "dtype_v": "fp8", - "dtype_o": "bf16", - "causal": true - }, - "name": "qwen3.5" - }, - { - "type": "attention_mha", - "args": { - "batch_size": 16, - "seq_len_q": 1024, - "seq_len_kv": 1024, - "num_heads": 16, - "num_heads_kv": 1, - "head_dim": 256, - "dtype_q": "nvfp4", - "dtype_k": "fp8", - "dtype_v": "fp8", - "dtype_o": "bf16", - "causal": true - }, - "name": "qwen3.5" - }, - { - "type": "attention_mha", - "args": { - "batch_size": 16, - "seq_len_q": 1, - "seq_len_kv": 1536, - "num_heads": 16, - "num_heads_kv": 1, - "head_dim": 256, - "dtype_q": "nvfp4", - "dtype_k": "fp8", - "dtype_v": "fp8", - "dtype_o": "bf16", - "causal": true - }, - "name": "qwen3.5" - }, - { - "type": "attention_mha", - "args": { - "batch_size": 32, - "seq_len_q": 1024, - "seq_len_kv": 1024, - "num_heads": 16, - "num_heads_kv": 1, - "head_dim": 256, - "dtype_q": "nvfp4", - "dtype_k": "fp8", - "dtype_v": "fp8", - "dtype_o": "bf16", - "causal": true - }, - "name": "qwen3.5" - }, - { - "type": "attention_mha", - "args": { - "batch_size": 32, - "seq_len_q": 1, - "seq_len_kv": 1536, - "num_heads": 16, - "num_heads_kv": 1, - "head_dim": 256, - "dtype_q": "nvfp4", - "dtype_k": "fp8", - "dtype_v": "fp8", - "dtype_o": "bf16", - "causal": true - }, - "name": "qwen3.5" - }, - { - "type": "attention_mha", - "args": { - "batch_size": 64, - "seq_len_q": 1024, - "seq_len_kv": 1024, - "num_heads": 16, - "num_heads_kv": 1, - "head_dim": 256, - "dtype_q": "nvfp4", - "dtype_k": "fp8", - "dtype_v": "fp8", - "dtype_o": "bf16", - "causal": true - }, - "name": "qwen3.5" - }, - { - "type": "attention_mha", - "args": { - "batch_size": 64, - "seq_len_q": 1, - "seq_len_kv": 1536, - "num_heads": 16, - "num_heads_kv": 1, - "head_dim": 256, - "dtype_q": "nvfp4", - "dtype_k": "fp8", - "dtype_v": "fp8", - "dtype_o": "bf16", - "causal": true - }, - "name": "qwen3.5" - }, - { - "type": "attention_mha", - "args": { - "batch_size": 128, - "seq_len_q": 1024, - "seq_len_kv": 1024, - "num_heads": 16, - "num_heads_kv": 1, - "head_dim": 256, - "dtype_q": "nvfp4", - "dtype_k": "fp8", - "dtype_v": "fp8", - "dtype_o": "bf16", - "causal": true - }, - "name": "qwen3.5" - }, - { - "type": "attention_mha", - "args": { - "batch_size": 128, - "seq_len_q": 1, - "seq_len_kv": 1536, - "num_heads": 16, - "num_heads_kv": 1, - "head_dim": 256, - "dtype_q": "nvfp4", - "dtype_k": "fp8", - "dtype_v": "fp8", - "dtype_o": "bf16", - "causal": true - }, - "name": "qwen3.5" - }, - { - "type": "attention_mha", - "args": { - "batch_size": 4, - "seq_len_q": 8192, - "seq_len_kv": 8192, - "num_heads": 8, - "num_heads_kv": 1, - "head_dim": 256, - "dtype_q": "nvfp4", - "dtype_k": "fp8", - "dtype_v": "fp8", - "dtype_o": "bf16", - "causal": true - }, - "name": "qwen3.5" - }, - { - "type": "attention_mha", - "args": { - "batch_size": 4, - "seq_len_q": 1, - "seq_len_kv": 8704, - "num_heads": 8, - "num_heads_kv": 1, - "head_dim": 256, - "dtype_q": "nvfp4", - "dtype_k": "fp8", - "dtype_v": "fp8", - "dtype_o": "bf16", - "causal": true - }, - "name": "qwen3.5" - }, - { - "type": "attention_mha", - "args": { - "batch_size": 4, - "seq_len_q": 8192, - "seq_len_kv": 8192, - "num_heads": 16, - "num_heads_kv": 1, - "head_dim": 256, - "dtype_q": "nvfp4", - "dtype_k": "fp8", - "dtype_v": "fp8", - "dtype_o": "bf16", - "causal": true - }, - "name": "qwen3.5" - }, - { - "type": "attention_mha", - "args": { - "batch_size": 4, - "seq_len_q": 1, - "seq_len_kv": 8704, - "num_heads": 16, - "num_heads_kv": 1, - "head_dim": 256, - "dtype_q": "nvfp4", - "dtype_k": "fp8", - "dtype_v": "fp8", - "dtype_o": "bf16", - "causal": true - }, - "name": "qwen3.5" - }, - { - "type": "attention_mha", - "args": { - "batch_size": 8, - "seq_len_q": 8192, - "seq_len_kv": 8192, - "num_heads": 16, - "num_heads_kv": 1, - "head_dim": 256, - "dtype_q": "nvfp4", - "dtype_k": "fp8", - "dtype_v": "fp8", - "dtype_o": "bf16", - "causal": true - }, - "name": "qwen3.5" - }, - { - "type": "attention_mha", - "args": { - "batch_size": 8, - "seq_len_q": 1, - "seq_len_kv": 8704, - "num_heads": 16, - "num_heads_kv": 1, - "head_dim": 256, - "dtype_q": "nvfp4", - "dtype_k": "fp8", - "dtype_v": "fp8", - "dtype_o": "bf16", - "causal": true - }, - "name": "qwen3.5" - }, - { - "type": "attention_mha", - "args": { - "batch_size": 16, - "seq_len_q": 8192, - "seq_len_kv": 8192, - "num_heads": 16, - "num_heads_kv": 1, - "head_dim": 256, - "dtype_q": "nvfp4", - "dtype_k": "fp8", - "dtype_v": "fp8", - "dtype_o": "bf16", - "causal": true - }, - "name": "qwen3.5" - }, - { - "type": "attention_mha", - "args": { - "batch_size": 16, - "seq_len_q": 1, - "seq_len_kv": 8704, - "num_heads": 16, - "num_heads_kv": 1, - "head_dim": 256, - "dtype_q": "nvfp4", - "dtype_k": "fp8", - "dtype_v": "fp8", - "dtype_o": "bf16", - "causal": true - }, - "name": "qwen3.5" - }, - { - "type": "attention_mha", - "args": { - "batch_size": 32, - "seq_len_q": 8192, - "seq_len_kv": 8192, - "num_heads": 16, - "num_heads_kv": 1, - "head_dim": 256, - "dtype_q": "nvfp4", - "dtype_k": "fp8", - "dtype_v": "fp8", - "dtype_o": "bf16", - "causal": true - }, - "name": "qwen3.5" - }, - { - "type": "attention_mha", - "args": { - "batch_size": 32, - "seq_len_q": 1, - "seq_len_kv": 8704, - "num_heads": 16, - "num_heads_kv": 1, - "head_dim": 256, - "dtype_q": "nvfp4", - "dtype_k": "fp8", - "dtype_v": "fp8", - "dtype_o": "bf16", - "causal": true - }, - "name": "qwen3.5" - }, - { - "type": "attention_mha", - "args": { - "batch_size": 64, - "seq_len_q": 8192, - "seq_len_kv": 8192, - "num_heads": 16, - "num_heads_kv": 1, - "head_dim": 256, - "dtype_q": "nvfp4", - "dtype_k": "fp8", - "dtype_v": "fp8", - "dtype_o": "bf16", - "causal": true - }, - "name": "qwen3.5" - }, - { - "type": "attention_mha", - "args": { - "batch_size": 64, - "seq_len_q": 1, - "seq_len_kv": 8704, - "num_heads": 16, - "num_heads_kv": 1, - "head_dim": 256, - "dtype_q": "nvfp4", - "dtype_k": "fp8", - "dtype_v": "fp8", - "dtype_o": "bf16", - "causal": true - }, - "name": "qwen3.5" - }, - { - "type": "attention_mha", - "args": { - "batch_size": 128, - "seq_len_q": 8192, - "seq_len_kv": 8192, - "num_heads": 16, - "num_heads_kv": 1, - "head_dim": 256, - "dtype_q": "nvfp4", - "dtype_k": "fp8", - "dtype_v": "fp8", - "dtype_o": "bf16", - "causal": true - }, - "name": "qwen3.5" - }, - { - "type": "attention_mha", - "args": { - "batch_size": 128, - "seq_len_q": 1, - "seq_len_kv": 8704, - "num_heads": 16, - "num_heads_kv": 1, - "head_dim": 256, - "dtype_q": "nvfp4", - "dtype_k": "fp8", - "dtype_v": "fp8", - "dtype_o": "bf16", - "causal": true - }, - "name": "qwen3.5" - }, - { - "type": "attention_mha", - "args": { - "batch_size": 4, - "seq_len_q": 2, - "seq_len_kv": 1536, - "num_heads": 8, - "num_heads_kv": 1, - "head_dim": 256, - "dtype_q": "nvfp4", - "dtype_k": "fp8", - "dtype_v": "fp8", - "dtype_o": "bf16", - "causal": true - }, - "name": "qwen3.5" - }, - { - "type": "attention_mha", - "args": { - "batch_size": 4, - "seq_len_q": 2, - "seq_len_kv": 1536, - "num_heads": 16, - "num_heads_kv": 1, - "head_dim": 256, - "dtype_q": "nvfp4", - "dtype_k": "fp8", - "dtype_v": "fp8", - "dtype_o": "bf16", - "causal": true - }, - "name": "qwen3.5" - }, - { - "type": "attention_mha", - "args": { - "batch_size": 8, - "seq_len_q": 2, - "seq_len_kv": 1536, - "num_heads": 16, - "num_heads_kv": 1, - "head_dim": 256, - "dtype_q": "nvfp4", - "dtype_k": "fp8", - "dtype_v": "fp8", - "dtype_o": "bf16", - "causal": true - }, - "name": "qwen3.5" - }, - { - "type": "attention_mha", - "args": { - "batch_size": 16, - "seq_len_q": 2, - "seq_len_kv": 1536, - "num_heads": 16, - "num_heads_kv": 1, - "head_dim": 256, - "dtype_q": "nvfp4", - "dtype_k": "fp8", - "dtype_v": "fp8", - "dtype_o": "bf16", - "causal": true - }, - "name": "qwen3.5" - }, - { - "type": "attention_mha", - "args": { - "batch_size": 32, - "seq_len_q": 2, - "seq_len_kv": 1536, - "num_heads": 16, - "num_heads_kv": 1, - "head_dim": 256, - "dtype_q": "nvfp4", - "dtype_k": "fp8", - "dtype_v": "fp8", - "dtype_o": "bf16", - "causal": true - }, - "name": "qwen3.5" - }, - { - "type": "attention_mha", - "args": { - "batch_size": 64, - "seq_len_q": 2, - "seq_len_kv": 1536, - "num_heads": 16, - "num_heads_kv": 1, - "head_dim": 256, - "dtype_q": "nvfp4", - "dtype_k": "fp8", - "dtype_v": "fp8", - "dtype_o": "bf16", - "causal": true - }, - "name": "qwen3.5" - }, - { - "type": "attention_mha", - "args": { - "batch_size": 4, - "seq_len_q": 2, - "seq_len_kv": 8704, - "num_heads": 8, - "num_heads_kv": 1, - "head_dim": 256, - "dtype_q": "nvfp4", - "dtype_k": "fp8", - "dtype_v": "fp8", - "dtype_o": "bf16", - "causal": true - }, - "name": "qwen3.5" - }, - { - "type": "attention_mha", - "args": { - "batch_size": 4, - "seq_len_q": 2, - "seq_len_kv": 8704, - "num_heads": 16, - "num_heads_kv": 1, - "head_dim": 256, - "dtype_q": "nvfp4", - "dtype_k": "fp8", - "dtype_v": "fp8", - "dtype_o": "bf16", - "causal": true - }, - "name": "qwen3.5" - }, - { - "type": "attention_mha", - "args": { - "batch_size": 8, - "seq_len_q": 2, - "seq_len_kv": 8704, - "num_heads": 16, - "num_heads_kv": 1, - "head_dim": 256, - "dtype_q": "nvfp4", - "dtype_k": "fp8", - "dtype_v": "fp8", - "dtype_o": "bf16", - "causal": true - }, - "name": "qwen3.5" - }, - { - "type": "attention_mha", - "args": { - "batch_size": 16, - "seq_len_q": 2, - "seq_len_kv": 8704, - "num_heads": 16, - "num_heads_kv": 1, - "head_dim": 256, - "dtype_q": "nvfp4", - "dtype_k": "fp8", - "dtype_v": "fp8", - "dtype_o": "bf16", - "causal": true - }, - "name": "qwen3.5" - }, - { - "type": "attention_mha", - "args": { - "batch_size": 32, - "seq_len_q": 2, - "seq_len_kv": 8704, - "num_heads": 16, - "num_heads_kv": 1, - "head_dim": 256, - "dtype_q": "nvfp4", - "dtype_k": "fp8", - "dtype_v": "fp8", - "dtype_o": "bf16", - "causal": true - }, - "name": "qwen3.5" - }, - { - "type": "attention_mha", - "args": { - "batch_size": 64, - "seq_len_q": 2, - "seq_len_kv": 8704, - "num_heads": 16, - "num_heads_kv": 1, - "head_dim": 256, - "dtype_q": "nvfp4", - "dtype_k": "fp8", - "dtype_v": "fp8", - "dtype_o": "bf16", - "causal": true - }, - "name": "qwen3.5" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 4, - "seq_len_q": 1024, - "seq_len_kv": 1024, - "num_heads": 8, - "head_dim_qk_nope": 192, - "head_dim_qk_rope": 64, - "head_dim_v": 256, - "kv_lora_rank": 512, - "dtype_q": "fp8", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "glm5" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 4, - "seq_len_q": 1, - "seq_len_kv": 1536, - "num_heads": 8, - "head_dim_qk_nope": 192, - "head_dim_qk_rope": 64, - "head_dim_v": 256, - "kv_lora_rank": 512, - "dtype_q": "fp8", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "glm5" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 8, - "seq_len_q": 1024, - "seq_len_kv": 1024, - "num_heads": 8, - "head_dim_qk_nope": 192, - "head_dim_qk_rope": 64, - "head_dim_v": 256, - "kv_lora_rank": 512, - "dtype_q": "fp8", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "glm5" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 8, - "seq_len_q": 1, - "seq_len_kv": 1536, - "num_heads": 8, - "head_dim_qk_nope": 192, - "head_dim_qk_rope": 64, - "head_dim_v": 256, - "kv_lora_rank": 512, - "dtype_q": "fp8", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "glm5" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 16, - "seq_len_q": 1024, - "seq_len_kv": 1024, - "num_heads": 8, - "head_dim_qk_nope": 192, - "head_dim_qk_rope": 64, - "head_dim_v": 256, - "kv_lora_rank": 512, - "dtype_q": "fp8", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "glm5" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 16, - "seq_len_q": 1, - "seq_len_kv": 1536, - "num_heads": 8, - "head_dim_qk_nope": 192, - "head_dim_qk_rope": 64, - "head_dim_v": 256, - "kv_lora_rank": 512, - "dtype_q": "fp8", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "glm5" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 32, - "seq_len_q": 1024, - "seq_len_kv": 1024, - "num_heads": 8, - "head_dim_qk_nope": 192, - "head_dim_qk_rope": 64, - "head_dim_v": 256, - "kv_lora_rank": 512, - "dtype_q": "fp8", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "glm5" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 32, - "seq_len_q": 1, - "seq_len_kv": 1536, - "num_heads": 8, - "head_dim_qk_nope": 192, - "head_dim_qk_rope": 64, - "head_dim_v": 256, - "kv_lora_rank": 512, - "dtype_q": "fp8", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "glm5" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 64, - "seq_len_q": 1024, - "seq_len_kv": 1024, - "num_heads": 8, - "head_dim_qk_nope": 192, - "head_dim_qk_rope": 64, - "head_dim_v": 256, - "kv_lora_rank": 512, - "dtype_q": "fp8", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "glm5" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 64, - "seq_len_q": 1, - "seq_len_kv": 1536, - "num_heads": 8, - "head_dim_qk_nope": 192, - "head_dim_qk_rope": 64, - "head_dim_v": 256, - "kv_lora_rank": 512, - "dtype_q": "fp8", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "glm5" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 128, - "seq_len_q": 1024, - "seq_len_kv": 1024, - "num_heads": 8, - "head_dim_qk_nope": 192, - "head_dim_qk_rope": 64, - "head_dim_v": 256, - "kv_lora_rank": 512, - "dtype_q": "fp8", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "glm5" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 128, - "seq_len_q": 1, - "seq_len_kv": 1536, - "num_heads": 8, - "head_dim_qk_nope": 192, - "head_dim_qk_rope": 64, - "head_dim_v": 256, - "kv_lora_rank": 512, - "dtype_q": "fp8", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "glm5" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 256, - "seq_len_q": 1024, - "seq_len_kv": 1024, - "num_heads": 8, - "head_dim_qk_nope": 192, - "head_dim_qk_rope": 64, - "head_dim_v": 256, - "kv_lora_rank": 512, - "dtype_q": "fp8", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "glm5" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 256, - "seq_len_q": 1, - "seq_len_kv": 1536, - "num_heads": 8, - "head_dim_qk_nope": 192, - "head_dim_qk_rope": 64, - "head_dim_v": 256, - "kv_lora_rank": 512, - "dtype_q": "fp8", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "glm5" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 4, - "seq_len_q": 8192, - "seq_len_kv": 8192, - "num_heads": 8, - "head_dim_qk_nope": 192, - "head_dim_qk_rope": 64, - "head_dim_v": 256, - "kv_lora_rank": 512, - "dtype_q": "fp8", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "glm5" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 4, - "seq_len_q": 1, - "seq_len_kv": 8704, - "num_heads": 8, - "head_dim_qk_nope": 192, - "head_dim_qk_rope": 64, - "head_dim_v": 256, - "kv_lora_rank": 512, - "dtype_q": "fp8", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "glm5" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 8, - "seq_len_q": 8192, - "seq_len_kv": 8192, - "num_heads": 8, - "head_dim_qk_nope": 192, - "head_dim_qk_rope": 64, - "head_dim_v": 256, - "kv_lora_rank": 512, - "dtype_q": "fp8", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "glm5" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 8, - "seq_len_q": 1, - "seq_len_kv": 8704, - "num_heads": 8, - "head_dim_qk_nope": 192, - "head_dim_qk_rope": 64, - "head_dim_v": 256, - "kv_lora_rank": 512, - "dtype_q": "fp8", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "glm5" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 16, - "seq_len_q": 8192, - "seq_len_kv": 8192, - "num_heads": 8, - "head_dim_qk_nope": 192, - "head_dim_qk_rope": 64, - "head_dim_v": 256, - "kv_lora_rank": 512, - "dtype_q": "fp8", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "glm5" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 16, - "seq_len_q": 1, - "seq_len_kv": 8704, - "num_heads": 8, - "head_dim_qk_nope": 192, - "head_dim_qk_rope": 64, - "head_dim_v": 256, - "kv_lora_rank": 512, - "dtype_q": "fp8", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "glm5" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 32, - "seq_len_q": 8192, - "seq_len_kv": 8192, - "num_heads": 8, - "head_dim_qk_nope": 192, - "head_dim_qk_rope": 64, - "head_dim_v": 256, - "kv_lora_rank": 512, - "dtype_q": "fp8", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "glm5" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 32, - "seq_len_q": 1, - "seq_len_kv": 8704, - "num_heads": 8, - "head_dim_qk_nope": 192, - "head_dim_qk_rope": 64, - "head_dim_v": 256, - "kv_lora_rank": 512, - "dtype_q": "fp8", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "glm5" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 64, - "seq_len_q": 8192, - "seq_len_kv": 8192, - "num_heads": 8, - "head_dim_qk_nope": 192, - "head_dim_qk_rope": 64, - "head_dim_v": 256, - "kv_lora_rank": 512, - "dtype_q": "fp8", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "glm5" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 64, - "seq_len_q": 1, - "seq_len_kv": 8704, - "num_heads": 8, - "head_dim_qk_nope": 192, - "head_dim_qk_rope": 64, - "head_dim_v": 256, - "kv_lora_rank": 512, - "dtype_q": "fp8", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "glm5" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 128, - "seq_len_q": 8192, - "seq_len_kv": 8192, - "num_heads": 8, - "head_dim_qk_nope": 192, - "head_dim_qk_rope": 64, - "head_dim_v": 256, - "kv_lora_rank": 512, - "dtype_q": "fp8", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "glm5" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 128, - "seq_len_q": 1, - "seq_len_kv": 8704, - "num_heads": 8, - "head_dim_qk_nope": 192, - "head_dim_qk_rope": 64, - "head_dim_v": 256, - "kv_lora_rank": 512, - "dtype_q": "fp8", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "glm5" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 256, - "seq_len_q": 8192, - "seq_len_kv": 8192, - "num_heads": 8, - "head_dim_qk_nope": 192, - "head_dim_qk_rope": 64, - "head_dim_v": 256, - "kv_lora_rank": 512, - "dtype_q": "fp8", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "glm5" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 256, - "seq_len_q": 1, - "seq_len_kv": 8704, - "num_heads": 8, - "head_dim_qk_nope": 192, - "head_dim_qk_rope": 64, - "head_dim_v": 256, - "kv_lora_rank": 512, - "dtype_q": "fp8", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "glm5" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 4, - "seq_len_q": 2, - "seq_len_kv": 1536, - "num_heads": 8, - "head_dim_qk_nope": 192, - "head_dim_qk_rope": 64, - "head_dim_v": 256, - "kv_lora_rank": 512, - "dtype_q": "fp8", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "glm5" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 8, - "seq_len_q": 2, - "seq_len_kv": 1536, - "num_heads": 8, - "head_dim_qk_nope": 192, - "head_dim_qk_rope": 64, - "head_dim_v": 256, - "kv_lora_rank": 512, - "dtype_q": "fp8", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "glm5" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 16, - "seq_len_q": 2, - "seq_len_kv": 1536, - "num_heads": 8, - "head_dim_qk_nope": 192, - "head_dim_qk_rope": 64, - "head_dim_v": 256, - "kv_lora_rank": 512, - "dtype_q": "fp8", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "glm5" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 32, - "seq_len_q": 2, - "seq_len_kv": 1536, - "num_heads": 8, - "head_dim_qk_nope": 192, - "head_dim_qk_rope": 64, - "head_dim_v": 256, - "kv_lora_rank": 512, - "dtype_q": "fp8", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "glm5" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 64, - "seq_len_q": 2, - "seq_len_kv": 1536, - "num_heads": 8, - "head_dim_qk_nope": 192, - "head_dim_qk_rope": 64, - "head_dim_v": 256, - "kv_lora_rank": 512, - "dtype_q": "fp8", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "glm5" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 128, - "seq_len_q": 2, - "seq_len_kv": 1536, - "num_heads": 8, - "head_dim_qk_nope": 192, - "head_dim_qk_rope": 64, - "head_dim_v": 256, - "kv_lora_rank": 512, - "dtype_q": "fp8", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "glm5" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 256, - "seq_len_q": 2, - "seq_len_kv": 1536, - "num_heads": 8, - "head_dim_qk_nope": 192, - "head_dim_qk_rope": 64, - "head_dim_v": 256, - "kv_lora_rank": 512, - "dtype_q": "fp8", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "glm5" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 4, - "seq_len_q": 2, - "seq_len_kv": 8704, - "num_heads": 8, - "head_dim_qk_nope": 192, - "head_dim_qk_rope": 64, - "head_dim_v": 256, - "kv_lora_rank": 512, - "dtype_q": "fp8", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "glm5" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 8, - "seq_len_q": 2, - "seq_len_kv": 8704, - "num_heads": 8, - "head_dim_qk_nope": 192, - "head_dim_qk_rope": 64, - "head_dim_v": 256, - "kv_lora_rank": 512, - "dtype_q": "fp8", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "glm5" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 16, - "seq_len_q": 2, - "seq_len_kv": 8704, - "num_heads": 8, - "head_dim_qk_nope": 192, - "head_dim_qk_rope": 64, - "head_dim_v": 256, - "kv_lora_rank": 512, - "dtype_q": "fp8", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "glm5" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 32, - "seq_len_q": 2, - "seq_len_kv": 8704, - "num_heads": 8, - "head_dim_qk_nope": 192, - "head_dim_qk_rope": 64, - "head_dim_v": 256, - "kv_lora_rank": 512, - "dtype_q": "fp8", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "glm5" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 64, - "seq_len_q": 2, - "seq_len_kv": 8704, - "num_heads": 8, - "head_dim_qk_nope": 192, - "head_dim_qk_rope": 64, - "head_dim_v": 256, - "kv_lora_rank": 512, - "dtype_q": "fp8", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "glm5" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 128, - "seq_len_q": 2, - "seq_len_kv": 8704, - "num_heads": 8, - "head_dim_qk_nope": 192, - "head_dim_qk_rope": 64, - "head_dim_v": 256, - "kv_lora_rank": 512, - "dtype_q": "fp8", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "glm5" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 256, - "seq_len_q": 2, - "seq_len_kv": 8704, - "num_heads": 8, - "head_dim_qk_nope": 192, - "head_dim_qk_rope": 64, - "head_dim_v": 256, - "kv_lora_rank": 512, - "dtype_q": "fp8", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "glm5" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 4, - "seq_len_q": 1024, - "seq_len_kv": 1024, - "num_heads": 8, - "head_dim_qk_nope": 192, - "head_dim_qk_rope": 64, - "head_dim_v": 256, - "kv_lora_rank": 512, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "glm5" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 4, - "seq_len_q": 1, - "seq_len_kv": 1536, - "num_heads": 8, - "head_dim_qk_nope": 192, - "head_dim_qk_rope": 64, - "head_dim_v": 256, - "kv_lora_rank": 512, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "glm5" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 4, - "seq_len_q": 1024, - "seq_len_kv": 1024, - "num_heads": 16, - "head_dim_qk_nope": 192, - "head_dim_qk_rope": 64, - "head_dim_v": 256, - "kv_lora_rank": 512, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "glm5" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 4, - "seq_len_q": 1, - "seq_len_kv": 1536, - "num_heads": 16, - "head_dim_qk_nope": 192, - "head_dim_qk_rope": 64, - "head_dim_v": 256, - "kv_lora_rank": 512, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "glm5" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 8, - "seq_len_q": 1024, - "seq_len_kv": 1024, - "num_heads": 16, - "head_dim_qk_nope": 192, - "head_dim_qk_rope": 64, - "head_dim_v": 256, - "kv_lora_rank": 512, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "glm5" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 8, - "seq_len_q": 1, - "seq_len_kv": 1536, - "num_heads": 16, - "head_dim_qk_nope": 192, - "head_dim_qk_rope": 64, - "head_dim_v": 256, - "kv_lora_rank": 512, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "glm5" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 16, - "seq_len_q": 1024, - "seq_len_kv": 1024, - "num_heads": 16, - "head_dim_qk_nope": 192, - "head_dim_qk_rope": 64, - "head_dim_v": 256, - "kv_lora_rank": 512, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "glm5" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 16, - "seq_len_q": 1, - "seq_len_kv": 1536, - "num_heads": 16, - "head_dim_qk_nope": 192, - "head_dim_qk_rope": 64, - "head_dim_v": 256, - "kv_lora_rank": 512, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "glm5" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 32, - "seq_len_q": 1024, - "seq_len_kv": 1024, - "num_heads": 16, - "head_dim_qk_nope": 192, - "head_dim_qk_rope": 64, - "head_dim_v": 256, - "kv_lora_rank": 512, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "glm5" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 32, - "seq_len_q": 1, - "seq_len_kv": 1536, - "num_heads": 16, - "head_dim_qk_nope": 192, - "head_dim_qk_rope": 64, - "head_dim_v": 256, - "kv_lora_rank": 512, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "glm5" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 64, - "seq_len_q": 1024, - "seq_len_kv": 1024, - "num_heads": 16, - "head_dim_qk_nope": 192, - "head_dim_qk_rope": 64, - "head_dim_v": 256, - "kv_lora_rank": 512, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "glm5" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 64, - "seq_len_q": 1, - "seq_len_kv": 1536, - "num_heads": 16, - "head_dim_qk_nope": 192, - "head_dim_qk_rope": 64, - "head_dim_v": 256, - "kv_lora_rank": 512, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "glm5" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 128, - "seq_len_q": 1024, - "seq_len_kv": 1024, - "num_heads": 16, - "head_dim_qk_nope": 192, - "head_dim_qk_rope": 64, - "head_dim_v": 256, - "kv_lora_rank": 512, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "glm5" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 128, - "seq_len_q": 1, - "seq_len_kv": 1536, - "num_heads": 16, - "head_dim_qk_nope": 192, - "head_dim_qk_rope": 64, - "head_dim_v": 256, - "kv_lora_rank": 512, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "glm5" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 256, - "seq_len_q": 1024, - "seq_len_kv": 1024, - "num_heads": 16, - "head_dim_qk_nope": 192, - "head_dim_qk_rope": 64, - "head_dim_v": 256, - "kv_lora_rank": 512, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "glm5" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 256, - "seq_len_q": 1, - "seq_len_kv": 1536, - "num_heads": 16, - "head_dim_qk_nope": 192, - "head_dim_qk_rope": 64, - "head_dim_v": 256, - "kv_lora_rank": 512, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "glm5" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 4, - "seq_len_q": 8192, - "seq_len_kv": 8192, - "num_heads": 8, - "head_dim_qk_nope": 192, - "head_dim_qk_rope": 64, - "head_dim_v": 256, - "kv_lora_rank": 512, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "glm5" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 4, - "seq_len_q": 1, - "seq_len_kv": 8704, - "num_heads": 8, - "head_dim_qk_nope": 192, - "head_dim_qk_rope": 64, - "head_dim_v": 256, - "kv_lora_rank": 512, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "glm5" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 4, - "seq_len_q": 8192, - "seq_len_kv": 8192, - "num_heads": 16, - "head_dim_qk_nope": 192, - "head_dim_qk_rope": 64, - "head_dim_v": 256, - "kv_lora_rank": 512, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "glm5" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 4, - "seq_len_q": 1, - "seq_len_kv": 8704, - "num_heads": 16, - "head_dim_qk_nope": 192, - "head_dim_qk_rope": 64, - "head_dim_v": 256, - "kv_lora_rank": 512, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "glm5" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 8, - "seq_len_q": 8192, - "seq_len_kv": 8192, - "num_heads": 16, - "head_dim_qk_nope": 192, - "head_dim_qk_rope": 64, - "head_dim_v": 256, - "kv_lora_rank": 512, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "glm5" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 8, - "seq_len_q": 1, - "seq_len_kv": 8704, - "num_heads": 16, - "head_dim_qk_nope": 192, - "head_dim_qk_rope": 64, - "head_dim_v": 256, - "kv_lora_rank": 512, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "glm5" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 16, - "seq_len_q": 8192, - "seq_len_kv": 8192, - "num_heads": 16, - "head_dim_qk_nope": 192, - "head_dim_qk_rope": 64, - "head_dim_v": 256, - "kv_lora_rank": 512, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "glm5" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 16, - "seq_len_q": 1, - "seq_len_kv": 8704, - "num_heads": 16, - "head_dim_qk_nope": 192, - "head_dim_qk_rope": 64, - "head_dim_v": 256, - "kv_lora_rank": 512, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "glm5" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 32, - "seq_len_q": 8192, - "seq_len_kv": 8192, - "num_heads": 16, - "head_dim_qk_nope": 192, - "head_dim_qk_rope": 64, - "head_dim_v": 256, - "kv_lora_rank": 512, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "glm5" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 32, - "seq_len_q": 1, - "seq_len_kv": 8704, - "num_heads": 16, - "head_dim_qk_nope": 192, - "head_dim_qk_rope": 64, - "head_dim_v": 256, - "kv_lora_rank": 512, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "glm5" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 64, - "seq_len_q": 8192, - "seq_len_kv": 8192, - "num_heads": 16, - "head_dim_qk_nope": 192, - "head_dim_qk_rope": 64, - "head_dim_v": 256, - "kv_lora_rank": 512, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "glm5" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 64, - "seq_len_q": 1, - "seq_len_kv": 8704, - "num_heads": 16, - "head_dim_qk_nope": 192, - "head_dim_qk_rope": 64, - "head_dim_v": 256, - "kv_lora_rank": 512, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "glm5" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 128, - "seq_len_q": 8192, - "seq_len_kv": 8192, - "num_heads": 16, - "head_dim_qk_nope": 192, - "head_dim_qk_rope": 64, - "head_dim_v": 256, - "kv_lora_rank": 512, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "glm5" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 128, - "seq_len_q": 1, - "seq_len_kv": 8704, - "num_heads": 16, - "head_dim_qk_nope": 192, - "head_dim_qk_rope": 64, - "head_dim_v": 256, - "kv_lora_rank": 512, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "glm5" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 256, - "seq_len_q": 8192, - "seq_len_kv": 8192, - "num_heads": 16, - "head_dim_qk_nope": 192, - "head_dim_qk_rope": 64, - "head_dim_v": 256, - "kv_lora_rank": 512, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "glm5" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 256, - "seq_len_q": 1, - "seq_len_kv": 8704, - "num_heads": 16, - "head_dim_qk_nope": 192, - "head_dim_qk_rope": 64, - "head_dim_v": 256, - "kv_lora_rank": 512, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "glm5" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 4, - "seq_len_q": 2, - "seq_len_kv": 1536, - "num_heads": 8, - "head_dim_qk_nope": 192, - "head_dim_qk_rope": 64, - "head_dim_v": 256, - "kv_lora_rank": 512, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "glm5" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 4, - "seq_len_q": 2, - "seq_len_kv": 1536, - "num_heads": 16, - "head_dim_qk_nope": 192, - "head_dim_qk_rope": 64, - "head_dim_v": 256, - "kv_lora_rank": 512, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "glm5" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 8, - "seq_len_q": 2, - "seq_len_kv": 1536, - "num_heads": 16, - "head_dim_qk_nope": 192, - "head_dim_qk_rope": 64, - "head_dim_v": 256, - "kv_lora_rank": 512, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "glm5" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 16, - "seq_len_q": 2, - "seq_len_kv": 1536, - "num_heads": 16, - "head_dim_qk_nope": 192, - "head_dim_qk_rope": 64, - "head_dim_v": 256, - "kv_lora_rank": 512, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "glm5" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 32, - "seq_len_q": 2, - "seq_len_kv": 1536, - "num_heads": 16, - "head_dim_qk_nope": 192, - "head_dim_qk_rope": 64, - "head_dim_v": 256, - "kv_lora_rank": 512, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "glm5" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 64, - "seq_len_q": 2, - "seq_len_kv": 1536, - "num_heads": 16, - "head_dim_qk_nope": 192, - "head_dim_qk_rope": 64, - "head_dim_v": 256, - "kv_lora_rank": 512, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "glm5" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 128, - "seq_len_q": 2, - "seq_len_kv": 1536, - "num_heads": 16, - "head_dim_qk_nope": 192, - "head_dim_qk_rope": 64, - "head_dim_v": 256, - "kv_lora_rank": 512, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "glm5" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 256, - "seq_len_q": 2, - "seq_len_kv": 1536, - "num_heads": 16, - "head_dim_qk_nope": 192, - "head_dim_qk_rope": 64, - "head_dim_v": 256, - "kv_lora_rank": 512, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "glm5" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 4, - "seq_len_q": 2, - "seq_len_kv": 8704, - "num_heads": 8, - "head_dim_qk_nope": 192, - "head_dim_qk_rope": 64, - "head_dim_v": 256, - "kv_lora_rank": 512, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "glm5" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 4, - "seq_len_q": 2, - "seq_len_kv": 8704, - "num_heads": 16, - "head_dim_qk_nope": 192, - "head_dim_qk_rope": 64, - "head_dim_v": 256, - "kv_lora_rank": 512, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "glm5" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 8, - "seq_len_q": 2, - "seq_len_kv": 8704, - "num_heads": 16, - "head_dim_qk_nope": 192, - "head_dim_qk_rope": 64, - "head_dim_v": 256, - "kv_lora_rank": 512, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "glm5" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 16, - "seq_len_q": 2, - "seq_len_kv": 8704, - "num_heads": 16, - "head_dim_qk_nope": 192, - "head_dim_qk_rope": 64, - "head_dim_v": 256, - "kv_lora_rank": 512, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "glm5" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 32, - "seq_len_q": 2, - "seq_len_kv": 8704, - "num_heads": 16, - "head_dim_qk_nope": 192, - "head_dim_qk_rope": 64, - "head_dim_v": 256, - "kv_lora_rank": 512, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "glm5" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 64, - "seq_len_q": 2, - "seq_len_kv": 8704, - "num_heads": 16, - "head_dim_qk_nope": 192, - "head_dim_qk_rope": 64, - "head_dim_v": 256, - "kv_lora_rank": 512, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "glm5" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 128, - "seq_len_q": 2, - "seq_len_kv": 8704, - "num_heads": 16, - "head_dim_qk_nope": 192, - "head_dim_qk_rope": 64, - "head_dim_v": 256, - "kv_lora_rank": 512, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "glm5" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 256, - "seq_len_q": 2, - "seq_len_kv": 8704, - "num_heads": 16, - "head_dim_qk_nope": 192, - "head_dim_qk_rope": 64, - "head_dim_v": 256, - "kv_lora_rank": 512, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "glm5" - }, - { - "type": "attention_mha", - "args": { - "batch_size": 4, - "seq_len_q": 2, - "seq_len_kv": 1536, - "num_heads": 4, - "num_heads_kv": 1, - "head_dim": 256, - "dtype_q": "fp8", - "dtype_k": "fp8", - "dtype_v": "fp8", - "dtype_o": "bf16", - "causal": true - }, - "name": "qwen3.5" - }, - { - "type": "attention_mha", - "args": { - "batch_size": 4, - "seq_len_q": 2, - "seq_len_kv": 1536, - "num_heads": 8, - "num_heads_kv": 1, - "head_dim": 256, - "dtype_q": "fp8", - "dtype_k": "fp8", - "dtype_v": "fp8", - "dtype_o": "bf16", - "causal": true - }, - "name": "qwen3.5" - }, - { - "type": "attention_mha", - "args": { - "batch_size": 8, - "seq_len_q": 2, - "seq_len_kv": 1536, - "num_heads": 8, - "num_heads_kv": 1, - "head_dim": 256, - "dtype_q": "fp8", - "dtype_k": "fp8", - "dtype_v": "fp8", - "dtype_o": "bf16", - "causal": true - }, - "name": "qwen3.5" - }, - { - "type": "attention_mha", - "args": { - "batch_size": 16, - "seq_len_q": 2, - "seq_len_kv": 1536, - "num_heads": 8, - "num_heads_kv": 1, - "head_dim": 256, - "dtype_q": "fp8", - "dtype_k": "fp8", - "dtype_v": "fp8", - "dtype_o": "bf16", - "causal": true - }, - "name": "qwen3.5" - }, - { - "type": "attention_mha", - "args": { - "batch_size": 32, - "seq_len_q": 2, - "seq_len_kv": 1536, - "num_heads": 8, - "num_heads_kv": 1, - "head_dim": 256, - "dtype_q": "fp8", - "dtype_k": "fp8", - "dtype_v": "fp8", - "dtype_o": "bf16", - "causal": true - }, - "name": "qwen3.5" - }, - { - "type": "attention_mha", - "args": { - "batch_size": 64, - "seq_len_q": 2, - "seq_len_kv": 1536, - "num_heads": 8, - "num_heads_kv": 1, - "head_dim": 256, - "dtype_q": "fp8", - "dtype_k": "fp8", - "dtype_v": "fp8", - "dtype_o": "bf16", - "causal": true - }, - "name": "qwen3.5" - }, - { - "type": "attention_mha", - "args": { - "batch_size": 128, - "seq_len_q": 2, - "seq_len_kv": 1536, - "num_heads": 8, - "num_heads_kv": 1, - "head_dim": 256, - "dtype_q": "fp8", - "dtype_k": "fp8", - "dtype_v": "fp8", - "dtype_o": "bf16", - "causal": true - }, - "name": "qwen3.5" - }, - { - "type": "attention_mha", - "args": { - "batch_size": 256, - "seq_len_q": 2, - "seq_len_kv": 1536, - "num_heads": 8, - "num_heads_kv": 1, - "head_dim": 256, - "dtype_q": "fp8", - "dtype_k": "fp8", - "dtype_v": "fp8", - "dtype_o": "bf16", - "causal": true - }, - "name": "qwen3.5" - }, - { - "type": "attention_mha", - "args": { - "batch_size": 4, - "seq_len_q": 2, - "seq_len_kv": 8704, - "num_heads": 4, - "num_heads_kv": 1, - "head_dim": 256, - "dtype_q": "fp8", - "dtype_k": "fp8", - "dtype_v": "fp8", - "dtype_o": "bf16", - "causal": true - }, - "name": "qwen3.5" - }, - { - "type": "attention_mha", - "args": { - "batch_size": 4, - "seq_len_q": 2, - "seq_len_kv": 8704, - "num_heads": 8, - "num_heads_kv": 1, - "head_dim": 256, - "dtype_q": "fp8", - "dtype_k": "fp8", - "dtype_v": "fp8", - "dtype_o": "bf16", - "causal": true - }, - "name": "qwen3.5" - }, - { - "type": "attention_mha", - "args": { - "batch_size": 8, - "seq_len_q": 2, - "seq_len_kv": 8704, - "num_heads": 8, - "num_heads_kv": 1, - "head_dim": 256, - "dtype_q": "fp8", - "dtype_k": "fp8", - "dtype_v": "fp8", - "dtype_o": "bf16", - "causal": true - }, - "name": "qwen3.5" - }, - { - "type": "attention_mha", - "args": { - "batch_size": 16, - "seq_len_q": 2, - "seq_len_kv": 8704, - "num_heads": 8, - "num_heads_kv": 1, - "head_dim": 256, - "dtype_q": "fp8", - "dtype_k": "fp8", - "dtype_v": "fp8", - "dtype_o": "bf16", - "causal": true - }, - "name": "qwen3.5" - }, - { - "type": "attention_mha", - "args": { - "batch_size": 32, - "seq_len_q": 2, - "seq_len_kv": 8704, - "num_heads": 8, - "num_heads_kv": 1, - "head_dim": 256, - "dtype_q": "fp8", - "dtype_k": "fp8", - "dtype_v": "fp8", - "dtype_o": "bf16", - "causal": true - }, - "name": "qwen3.5" - }, - { - "type": "attention_mha", - "args": { - "batch_size": 64, - "seq_len_q": 2, - "seq_len_kv": 8704, - "num_heads": 8, - "num_heads_kv": 1, - "head_dim": 256, - "dtype_q": "fp8", - "dtype_k": "fp8", - "dtype_v": "fp8", - "dtype_o": "bf16", - "causal": true - }, - "name": "qwen3.5" - }, - { - "type": "attention_mha", - "args": { - "batch_size": 128, - "seq_len_q": 2, - "seq_len_kv": 8704, - "num_heads": 8, - "num_heads_kv": 1, - "head_dim": 256, - "dtype_q": "fp8", - "dtype_k": "fp8", - "dtype_v": "fp8", - "dtype_o": "bf16", - "causal": true - }, - "name": "qwen3.5" - }, - { - "type": "attention_mha", - "args": { - "batch_size": 256, - "seq_len_q": 2, - "seq_len_kv": 8704, - "num_heads": 8, - "num_heads_kv": 1, - "head_dim": 256, - "dtype_q": "fp8", - "dtype_k": "fp8", - "dtype_v": "fp8", - "dtype_o": "bf16", - "causal": true - }, - "name": "qwen3.5" - }, - { - "type": "attention_mha", - "args": { - "batch_size": 8, - "seq_len_q": 1024, - "seq_len_kv": 1024, - "num_heads": 8, - "num_heads_kv": 1, - "head_dim": 256, - "dtype_q": "nvfp4", - "dtype_k": "fp8", - "dtype_v": "fp8", - "dtype_o": "bf16", - "causal": true - }, - "name": "qwen3.5" - }, - { - "type": "attention_mha", - "args": { - "batch_size": 8, - "seq_len_q": 1, - "seq_len_kv": 1536, - "num_heads": 8, - "num_heads_kv": 1, - "head_dim": 256, - "dtype_q": "nvfp4", - "dtype_k": "fp8", - "dtype_v": "fp8", - "dtype_o": "bf16", - "causal": true - }, - "name": "qwen3.5" - }, - { - "type": "attention_mha", - "args": { - "batch_size": 16, - "seq_len_q": 1024, - "seq_len_kv": 1024, - "num_heads": 8, - "num_heads_kv": 1, - "head_dim": 256, - "dtype_q": "nvfp4", - "dtype_k": "fp8", - "dtype_v": "fp8", - "dtype_o": "bf16", - "causal": true - }, - "name": "qwen3.5" - }, - { - "type": "attention_mha", - "args": { - "batch_size": 16, - "seq_len_q": 1, - "seq_len_kv": 1536, - "num_heads": 8, - "num_heads_kv": 1, - "head_dim": 256, - "dtype_q": "nvfp4", - "dtype_k": "fp8", - "dtype_v": "fp8", - "dtype_o": "bf16", - "causal": true - }, - "name": "qwen3.5" - }, - { - "type": "attention_mha", - "args": { - "batch_size": 32, - "seq_len_q": 1024, - "seq_len_kv": 1024, - "num_heads": 8, - "num_heads_kv": 1, - "head_dim": 256, - "dtype_q": "nvfp4", - "dtype_k": "fp8", - "dtype_v": "fp8", - "dtype_o": "bf16", - "causal": true - }, - "name": "qwen3.5" - }, - { - "type": "attention_mha", - "args": { - "batch_size": 32, - "seq_len_q": 1, - "seq_len_kv": 1536, - "num_heads": 8, - "num_heads_kv": 1, - "head_dim": 256, - "dtype_q": "nvfp4", - "dtype_k": "fp8", - "dtype_v": "fp8", - "dtype_o": "bf16", - "causal": true - }, - "name": "qwen3.5" - }, - { - "type": "attention_mha", - "args": { - "batch_size": 64, - "seq_len_q": 1024, - "seq_len_kv": 1024, - "num_heads": 8, - "num_heads_kv": 1, - "head_dim": 256, - "dtype_q": "nvfp4", - "dtype_k": "fp8", - "dtype_v": "fp8", - "dtype_o": "bf16", - "causal": true - }, - "name": "qwen3.5" - }, - { - "type": "attention_mha", - "args": { - "batch_size": 64, - "seq_len_q": 1, - "seq_len_kv": 1536, - "num_heads": 8, - "num_heads_kv": 1, - "head_dim": 256, - "dtype_q": "nvfp4", - "dtype_k": "fp8", - "dtype_v": "fp8", - "dtype_o": "bf16", - "causal": true - }, - "name": "qwen3.5" - }, - { - "type": "attention_mha", - "args": { - "batch_size": 128, - "seq_len_q": 1024, - "seq_len_kv": 1024, - "num_heads": 8, - "num_heads_kv": 1, - "head_dim": 256, - "dtype_q": "nvfp4", - "dtype_k": "fp8", - "dtype_v": "fp8", - "dtype_o": "bf16", - "causal": true - }, - "name": "qwen3.5" - }, - { - "type": "attention_mha", - "args": { - "batch_size": 128, - "seq_len_q": 1, - "seq_len_kv": 1536, - "num_heads": 8, - "num_heads_kv": 1, - "head_dim": 256, - "dtype_q": "nvfp4", - "dtype_k": "fp8", - "dtype_v": "fp8", - "dtype_o": "bf16", - "causal": true - }, - "name": "qwen3.5" - }, - { - "type": "attention_mha", - "args": { - "batch_size": 8, - "seq_len_q": 8192, - "seq_len_kv": 8192, - "num_heads": 8, - "num_heads_kv": 1, - "head_dim": 256, - "dtype_q": "nvfp4", - "dtype_k": "fp8", - "dtype_v": "fp8", - "dtype_o": "bf16", - "causal": true - }, - "name": "qwen3.5" - }, - { - "type": "attention_mha", - "args": { - "batch_size": 8, - "seq_len_q": 1, - "seq_len_kv": 8704, - "num_heads": 8, - "num_heads_kv": 1, - "head_dim": 256, - "dtype_q": "nvfp4", - "dtype_k": "fp8", - "dtype_v": "fp8", - "dtype_o": "bf16", - "causal": true - }, - "name": "qwen3.5" - }, - { - "type": "attention_mha", - "args": { - "batch_size": 16, - "seq_len_q": 8192, - "seq_len_kv": 8192, - "num_heads": 8, - "num_heads_kv": 1, - "head_dim": 256, - "dtype_q": "nvfp4", - "dtype_k": "fp8", - "dtype_v": "fp8", - "dtype_o": "bf16", - "causal": true - }, - "name": "qwen3.5" - }, - { - "type": "attention_mha", - "args": { - "batch_size": 16, - "seq_len_q": 1, - "seq_len_kv": 8704, - "num_heads": 8, - "num_heads_kv": 1, - "head_dim": 256, - "dtype_q": "nvfp4", - "dtype_k": "fp8", - "dtype_v": "fp8", - "dtype_o": "bf16", - "causal": true - }, - "name": "qwen3.5" - }, - { - "type": "attention_mha", - "args": { - "batch_size": 32, - "seq_len_q": 8192, - "seq_len_kv": 8192, - "num_heads": 8, - "num_heads_kv": 1, - "head_dim": 256, - "dtype_q": "nvfp4", - "dtype_k": "fp8", - "dtype_v": "fp8", - "dtype_o": "bf16", - "causal": true - }, - "name": "qwen3.5" - }, - { - "type": "attention_mha", - "args": { - "batch_size": 32, - "seq_len_q": 1, - "seq_len_kv": 8704, - "num_heads": 8, - "num_heads_kv": 1, - "head_dim": 256, - "dtype_q": "nvfp4", - "dtype_k": "fp8", - "dtype_v": "fp8", - "dtype_o": "bf16", - "causal": true - }, - "name": "qwen3.5" - }, - { - "type": "attention_mha", - "args": { - "batch_size": 64, - "seq_len_q": 8192, - "seq_len_kv": 8192, - "num_heads": 8, - "num_heads_kv": 1, - "head_dim": 256, - "dtype_q": "nvfp4", - "dtype_k": "fp8", - "dtype_v": "fp8", - "dtype_o": "bf16", - "causal": true - }, - "name": "qwen3.5" - }, - { - "type": "attention_mha", - "args": { - "batch_size": 64, - "seq_len_q": 1, - "seq_len_kv": 8704, - "num_heads": 8, - "num_heads_kv": 1, - "head_dim": 256, - "dtype_q": "nvfp4", - "dtype_k": "fp8", - "dtype_v": "fp8", - "dtype_o": "bf16", - "causal": true - }, - "name": "qwen3.5" - }, - { - "type": "attention_mha", - "args": { - "batch_size": 128, - "seq_len_q": 8192, - "seq_len_kv": 8192, - "num_heads": 8, - "num_heads_kv": 1, - "head_dim": 256, - "dtype_q": "nvfp4", - "dtype_k": "fp8", - "dtype_v": "fp8", - "dtype_o": "bf16", - "causal": true - }, - "name": "qwen3.5" - }, - { - "type": "attention_mha", - "args": { - "batch_size": 128, - "seq_len_q": 1, - "seq_len_kv": 8704, - "num_heads": 8, - "num_heads_kv": 1, - "head_dim": 256, - "dtype_q": "nvfp4", - "dtype_k": "fp8", - "dtype_v": "fp8", - "dtype_o": "bf16", - "causal": true - }, - "name": "qwen3.5" - }, - { - "type": "attention_mha", - "args": { - "batch_size": 8, - "seq_len_q": 2, - "seq_len_kv": 1536, - "num_heads": 8, - "num_heads_kv": 1, - "head_dim": 256, - "dtype_q": "nvfp4", - "dtype_k": "fp8", - "dtype_v": "fp8", - "dtype_o": "bf16", - "causal": true - }, - "name": "qwen3.5" - }, - { - "type": "attention_mha", - "args": { - "batch_size": 16, - "seq_len_q": 2, - "seq_len_kv": 1536, - "num_heads": 8, - "num_heads_kv": 1, - "head_dim": 256, - "dtype_q": "nvfp4", - "dtype_k": "fp8", - "dtype_v": "fp8", - "dtype_o": "bf16", - "causal": true - }, - "name": "qwen3.5" - }, - { - "type": "attention_mha", - "args": { - "batch_size": 32, - "seq_len_q": 2, - "seq_len_kv": 1536, - "num_heads": 8, - "num_heads_kv": 1, - "head_dim": 256, - "dtype_q": "nvfp4", - "dtype_k": "fp8", - "dtype_v": "fp8", - "dtype_o": "bf16", - "causal": true - }, - "name": "qwen3.5" - }, - { - "type": "attention_mha", - "args": { - "batch_size": 64, - "seq_len_q": 2, - "seq_len_kv": 1536, - "num_heads": 8, - "num_heads_kv": 1, - "head_dim": 256, - "dtype_q": "nvfp4", - "dtype_k": "fp8", - "dtype_v": "fp8", - "dtype_o": "bf16", - "causal": true - }, - "name": "qwen3.5" - }, - { - "type": "attention_mha", - "args": { - "batch_size": 128, - "seq_len_q": 2, - "seq_len_kv": 1536, - "num_heads": 8, - "num_heads_kv": 1, - "head_dim": 256, - "dtype_q": "nvfp4", - "dtype_k": "fp8", - "dtype_v": "fp8", - "dtype_o": "bf16", - "causal": true - }, - "name": "qwen3.5" - }, - { - "type": "attention_mha", - "args": { - "batch_size": 128, - "seq_len_q": 2, - "seq_len_kv": 1536, - "num_heads": 16, - "num_heads_kv": 1, - "head_dim": 256, - "dtype_q": "nvfp4", - "dtype_k": "fp8", - "dtype_v": "fp8", - "dtype_o": "bf16", - "causal": true - }, - "name": "qwen3.5" - }, - { - "type": "attention_mha", - "args": { - "batch_size": 8, - "seq_len_q": 2, - "seq_len_kv": 8704, - "num_heads": 8, - "num_heads_kv": 1, - "head_dim": 256, - "dtype_q": "nvfp4", - "dtype_k": "fp8", - "dtype_v": "fp8", - "dtype_o": "bf16", - "causal": true - }, - "name": "qwen3.5" - }, - { - "type": "attention_mha", - "args": { - "batch_size": 16, - "seq_len_q": 2, - "seq_len_kv": 8704, - "num_heads": 8, - "num_heads_kv": 1, - "head_dim": 256, - "dtype_q": "nvfp4", - "dtype_k": "fp8", - "dtype_v": "fp8", - "dtype_o": "bf16", - "causal": true - }, - "name": "qwen3.5" - }, - { - "type": "attention_mha", - "args": { - "batch_size": 32, - "seq_len_q": 2, - "seq_len_kv": 8704, - "num_heads": 8, - "num_heads_kv": 1, - "head_dim": 256, - "dtype_q": "nvfp4", - "dtype_k": "fp8", - "dtype_v": "fp8", - "dtype_o": "bf16", - "causal": true - }, - "name": "qwen3.5" - }, - { - "type": "attention_mha", - "args": { - "batch_size": 64, - "seq_len_q": 2, - "seq_len_kv": 8704, - "num_heads": 8, - "num_heads_kv": 1, - "head_dim": 256, - "dtype_q": "nvfp4", - "dtype_k": "fp8", - "dtype_v": "fp8", - "dtype_o": "bf16", - "causal": true - }, - "name": "qwen3.5" - }, - { - "type": "attention_mha", - "args": { - "batch_size": 128, - "seq_len_q": 2, - "seq_len_kv": 8704, - "num_heads": 8, - "num_heads_kv": 1, - "head_dim": 256, - "dtype_q": "nvfp4", - "dtype_k": "fp8", - "dtype_v": "fp8", - "dtype_o": "bf16", - "causal": true - }, - "name": "qwen3.5" - }, - { - "type": "attention_mha", - "args": { - "batch_size": 128, - "seq_len_q": 2, - "seq_len_kv": 8704, - "num_heads": 16, - "num_heads_kv": 1, - "head_dim": 256, - "dtype_q": "nvfp4", - "dtype_k": "fp8", - "dtype_v": "fp8", - "dtype_o": "bf16", - "causal": true - }, - "name": "qwen3.5" - }, - { - "type": "attention_mha", - "args": { - "batch_size": 4, - "seq_len_q": 1024, - "seq_len_kv": 1024, - "num_heads": 8, - "num_heads_kv": 1, - "head_dim": 256, - "dtype_q": "bf16", - "dtype_k": "bf16", - "dtype_v": "bf16", - "dtype_o": "bf16", - "causal": true - }, - "name": "qwen3.5" - }, - { - "type": "attention_mha", - "args": { - "batch_size": 4, - "seq_len_q": 1, - "seq_len_kv": 1536, - "num_heads": 8, - "num_heads_kv": 1, - "head_dim": 256, - "dtype_q": "bf16", - "dtype_k": "bf16", - "dtype_v": "bf16", - "dtype_o": "bf16", - "causal": true - }, - "name": "qwen3.5" - }, - { - "type": "attention_mha", - "args": { - "batch_size": 8, - "seq_len_q": 1024, - "seq_len_kv": 1024, - "num_heads": 8, - "num_heads_kv": 1, - "head_dim": 256, - "dtype_q": "bf16", - "dtype_k": "bf16", - "dtype_v": "bf16", - "dtype_o": "bf16", - "causal": true - }, - "name": "qwen3.5" - }, - { - "type": "attention_mha", - "args": { - "batch_size": 8, - "seq_len_q": 1, - "seq_len_kv": 1536, - "num_heads": 8, - "num_heads_kv": 1, - "head_dim": 256, - "dtype_q": "bf16", - "dtype_k": "bf16", - "dtype_v": "bf16", - "dtype_o": "bf16", - "causal": true - }, - "name": "qwen3.5" - }, - { - "type": "attention_mha", - "args": { - "batch_size": 16, - "seq_len_q": 1024, - "seq_len_kv": 1024, - "num_heads": 8, - "num_heads_kv": 1, - "head_dim": 256, - "dtype_q": "bf16", - "dtype_k": "bf16", - "dtype_v": "bf16", - "dtype_o": "bf16", - "causal": true - }, - "name": "qwen3.5" - }, - { - "type": "attention_mha", - "args": { - "batch_size": 16, - "seq_len_q": 1, - "seq_len_kv": 1536, - "num_heads": 8, - "num_heads_kv": 1, - "head_dim": 256, - "dtype_q": "bf16", - "dtype_k": "bf16", - "dtype_v": "bf16", - "dtype_o": "bf16", - "causal": true - }, - "name": "qwen3.5" - }, - { - "type": "attention_mha", - "args": { - "batch_size": 32, - "seq_len_q": 1024, - "seq_len_kv": 1024, - "num_heads": 8, - "num_heads_kv": 1, - "head_dim": 256, - "dtype_q": "bf16", - "dtype_k": "bf16", - "dtype_v": "bf16", - "dtype_o": "bf16", - "causal": true - }, - "name": "qwen3.5" - }, - { - "type": "attention_mha", - "args": { - "batch_size": 32, - "seq_len_q": 1, - "seq_len_kv": 1536, - "num_heads": 8, - "num_heads_kv": 1, - "head_dim": 256, - "dtype_q": "bf16", - "dtype_k": "bf16", - "dtype_v": "bf16", - "dtype_o": "bf16", - "causal": true - }, - "name": "qwen3.5" - }, - { - "type": "attention_mha", - "args": { - "batch_size": 64, - "seq_len_q": 1024, - "seq_len_kv": 1024, - "num_heads": 8, - "num_heads_kv": 1, - "head_dim": 256, - "dtype_q": "bf16", - "dtype_k": "bf16", - "dtype_v": "bf16", - "dtype_o": "bf16", - "causal": true - }, - "name": "qwen3.5" - }, - { - "type": "attention_mha", - "args": { - "batch_size": 64, - "seq_len_q": 1, - "seq_len_kv": 1536, - "num_heads": 8, - "num_heads_kv": 1, - "head_dim": 256, - "dtype_q": "bf16", - "dtype_k": "bf16", - "dtype_v": "bf16", - "dtype_o": "bf16", - "causal": true - }, - "name": "qwen3.5" - }, - { - "type": "attention_mha", - "args": { - "batch_size": 4, - "seq_len_q": 8192, - "seq_len_kv": 8192, - "num_heads": 8, - "num_heads_kv": 1, - "head_dim": 256, - "dtype_q": "bf16", - "dtype_k": "bf16", - "dtype_v": "bf16", - "dtype_o": "bf16", - "causal": true - }, - "name": "qwen3.5" - }, - { - "type": "attention_mha", - "args": { - "batch_size": 4, - "seq_len_q": 1, - "seq_len_kv": 8704, - "num_heads": 8, - "num_heads_kv": 1, - "head_dim": 256, - "dtype_q": "bf16", - "dtype_k": "bf16", - "dtype_v": "bf16", - "dtype_o": "bf16", - "causal": true - }, - "name": "qwen3.5" - }, - { - "type": "attention_mha", - "args": { - "batch_size": 8, - "seq_len_q": 8192, - "seq_len_kv": 8192, - "num_heads": 8, - "num_heads_kv": 1, - "head_dim": 256, - "dtype_q": "bf16", - "dtype_k": "bf16", - "dtype_v": "bf16", - "dtype_o": "bf16", - "causal": true - }, - "name": "qwen3.5" - }, - { - "type": "attention_mha", - "args": { - "batch_size": 8, - "seq_len_q": 1, - "seq_len_kv": 8704, - "num_heads": 8, - "num_heads_kv": 1, - "head_dim": 256, - "dtype_q": "bf16", - "dtype_k": "bf16", - "dtype_v": "bf16", - "dtype_o": "bf16", - "causal": true - }, - "name": "qwen3.5" - }, - { - "type": "attention_mha", - "args": { - "batch_size": 16, - "seq_len_q": 8192, - "seq_len_kv": 8192, - "num_heads": 8, - "num_heads_kv": 1, - "head_dim": 256, - "dtype_q": "bf16", - "dtype_k": "bf16", - "dtype_v": "bf16", - "dtype_o": "bf16", - "causal": true - }, - "name": "qwen3.5" - }, - { - "type": "attention_mha", - "args": { - "batch_size": 16, - "seq_len_q": 1, - "seq_len_kv": 8704, - "num_heads": 8, - "num_heads_kv": 1, - "head_dim": 256, - "dtype_q": "bf16", - "dtype_k": "bf16", - "dtype_v": "bf16", - "dtype_o": "bf16", - "causal": true - }, - "name": "qwen3.5" - }, - { - "type": "attention_mha", - "args": { - "batch_size": 32, - "seq_len_q": 8192, - "seq_len_kv": 8192, - "num_heads": 8, - "num_heads_kv": 1, - "head_dim": 256, - "dtype_q": "bf16", - "dtype_k": "bf16", - "dtype_v": "bf16", - "dtype_o": "bf16", - "causal": true - }, - "name": "qwen3.5" - }, - { - "type": "attention_mha", - "args": { - "batch_size": 32, - "seq_len_q": 1, - "seq_len_kv": 8704, - "num_heads": 8, - "num_heads_kv": 1, - "head_dim": 256, - "dtype_q": "bf16", - "dtype_k": "bf16", - "dtype_v": "bf16", - "dtype_o": "bf16", - "causal": true - }, - "name": "qwen3.5" - }, - { - "type": "attention_mha", - "args": { - "batch_size": 64, - "seq_len_q": 8192, - "seq_len_kv": 8192, - "num_heads": 8, - "num_heads_kv": 1, - "head_dim": 256, - "dtype_q": "bf16", - "dtype_k": "bf16", - "dtype_v": "bf16", - "dtype_o": "bf16", - "causal": true - }, - "name": "qwen3.5" - }, - { - "type": "attention_mha", - "args": { - "batch_size": 64, - "seq_len_q": 1, - "seq_len_kv": 8704, - "num_heads": 8, - "num_heads_kv": 1, - "head_dim": 256, - "dtype_q": "bf16", - "dtype_k": "bf16", - "dtype_v": "bf16", - "dtype_o": "bf16", - "causal": true - }, - "name": "qwen3.5" - }, - { - "type": "attention_mha", - "args": { - "batch_size": 4, - "seq_len_q": 2, - "seq_len_kv": 1536, - "num_heads": 8, - "num_heads_kv": 1, - "head_dim": 256, - "dtype_q": "bf16", - "dtype_k": "bf16", - "dtype_v": "bf16", - "dtype_o": "bf16", - "causal": true - }, - "name": "qwen3.5" - }, - { - "type": "attention_mha", - "args": { - "batch_size": 8, - "seq_len_q": 2, - "seq_len_kv": 1536, - "num_heads": 8, - "num_heads_kv": 1, - "head_dim": 256, - "dtype_q": "bf16", - "dtype_k": "bf16", - "dtype_v": "bf16", - "dtype_o": "bf16", - "causal": true - }, - "name": "qwen3.5" - }, - { - "type": "attention_mha", - "args": { - "batch_size": 16, - "seq_len_q": 2, - "seq_len_kv": 1536, - "num_heads": 8, - "num_heads_kv": 1, - "head_dim": 256, - "dtype_q": "bf16", - "dtype_k": "bf16", - "dtype_v": "bf16", - "dtype_o": "bf16", - "causal": true - }, - "name": "qwen3.5" - }, - { - "type": "attention_mha", - "args": { - "batch_size": 32, - "seq_len_q": 2, - "seq_len_kv": 1536, - "num_heads": 8, - "num_heads_kv": 1, - "head_dim": 256, - "dtype_q": "bf16", - "dtype_k": "bf16", - "dtype_v": "bf16", - "dtype_o": "bf16", - "causal": true - }, - "name": "qwen3.5" - }, - { - "type": "attention_mha", - "args": { - "batch_size": 64, - "seq_len_q": 2, - "seq_len_kv": 1536, - "num_heads": 8, - "num_heads_kv": 1, - "head_dim": 256, - "dtype_q": "bf16", - "dtype_k": "bf16", - "dtype_v": "bf16", - "dtype_o": "bf16", - "causal": true - }, - "name": "qwen3.5" - }, - { - "type": "attention_mha", - "args": { - "batch_size": 4, - "seq_len_q": 2, - "seq_len_kv": 8704, - "num_heads": 8, - "num_heads_kv": 1, - "head_dim": 256, - "dtype_q": "bf16", - "dtype_k": "bf16", - "dtype_v": "bf16", - "dtype_o": "bf16", - "causal": true - }, - "name": "qwen3.5" - }, - { - "type": "attention_mha", - "args": { - "batch_size": 8, - "seq_len_q": 2, - "seq_len_kv": 8704, - "num_heads": 8, - "num_heads_kv": 1, - "head_dim": 256, - "dtype_q": "bf16", - "dtype_k": "bf16", - "dtype_v": "bf16", - "dtype_o": "bf16", - "causal": true - }, - "name": "qwen3.5" - }, - { - "type": "attention_mha", - "args": { - "batch_size": 16, - "seq_len_q": 2, - "seq_len_kv": 8704, - "num_heads": 8, - "num_heads_kv": 1, - "head_dim": 256, - "dtype_q": "bf16", - "dtype_k": "bf16", - "dtype_v": "bf16", - "dtype_o": "bf16", - "causal": true - }, - "name": "qwen3.5" - }, - { - "type": "attention_mha", - "args": { - "batch_size": 32, - "seq_len_q": 2, - "seq_len_kv": 8704, - "num_heads": 8, - "num_heads_kv": 1, - "head_dim": 256, - "dtype_q": "bf16", - "dtype_k": "bf16", - "dtype_v": "bf16", - "dtype_o": "bf16", - "causal": true - }, - "name": "qwen3.5" - }, - { - "type": "attention_mha", - "args": { - "batch_size": 64, - "seq_len_q": 2, - "seq_len_kv": 8704, - "num_heads": 8, - "num_heads_kv": 1, - "head_dim": 256, - "dtype_q": "bf16", - "dtype_k": "bf16", - "dtype_v": "bf16", - "dtype_o": "bf16", - "causal": true - }, - "name": "qwen3.5" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 4, - "seq_len_q": 1024, - "seq_len_kv": 1024, - "num_heads": 8, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "int4", - "dtype_kv": "int4", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "kimik2.5" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 4, - "seq_len_q": 1, - "seq_len_kv": 1536, - "num_heads": 8, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "int4", - "dtype_kv": "int4", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "kimik2.5" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 8, - "seq_len_q": 1024, - "seq_len_kv": 1024, - "num_heads": 8, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "int4", - "dtype_kv": "int4", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "kimik2.5" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 8, - "seq_len_q": 1, - "seq_len_kv": 1536, - "num_heads": 8, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "int4", - "dtype_kv": "int4", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "kimik2.5" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 16, - "seq_len_q": 1024, - "seq_len_kv": 1024, - "num_heads": 8, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "int4", - "dtype_kv": "int4", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "kimik2.5" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 16, - "seq_len_q": 1, - "seq_len_kv": 1536, - "num_heads": 8, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "int4", - "dtype_kv": "int4", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "kimik2.5" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 32, - "seq_len_q": 1024, - "seq_len_kv": 1024, - "num_heads": 8, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "int4", - "dtype_kv": "int4", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "kimik2.5" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 32, - "seq_len_q": 1, - "seq_len_kv": 1536, - "num_heads": 8, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "int4", - "dtype_kv": "int4", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "kimik2.5" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 64, - "seq_len_q": 1024, - "seq_len_kv": 1024, - "num_heads": 8, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "int4", - "dtype_kv": "int4", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "kimik2.5" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 64, - "seq_len_q": 1, - "seq_len_kv": 1536, - "num_heads": 8, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "int4", - "dtype_kv": "int4", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "kimik2.5" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 4, - "seq_len_q": 8192, - "seq_len_kv": 8192, - "num_heads": 8, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "int4", - "dtype_kv": "int4", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "kimik2.5" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 4, - "seq_len_q": 1, - "seq_len_kv": 8704, - "num_heads": 8, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "int4", - "dtype_kv": "int4", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "kimik2.5" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 8, - "seq_len_q": 8192, - "seq_len_kv": 8192, - "num_heads": 8, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "int4", - "dtype_kv": "int4", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "kimik2.5" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 8, - "seq_len_q": 1, - "seq_len_kv": 8704, - "num_heads": 8, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "int4", - "dtype_kv": "int4", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "kimik2.5" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 16, - "seq_len_q": 8192, - "seq_len_kv": 8192, - "num_heads": 8, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "int4", - "dtype_kv": "int4", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "kimik2.5" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 16, - "seq_len_q": 1, - "seq_len_kv": 8704, - "num_heads": 8, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "int4", - "dtype_kv": "int4", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "kimik2.5" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 32, - "seq_len_q": 8192, - "seq_len_kv": 8192, - "num_heads": 8, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "int4", - "dtype_kv": "int4", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "kimik2.5" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 32, - "seq_len_q": 1, - "seq_len_kv": 8704, - "num_heads": 8, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "int4", - "dtype_kv": "int4", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "kimik2.5" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 64, - "seq_len_q": 8192, - "seq_len_kv": 8192, - "num_heads": 8, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "int4", - "dtype_kv": "int4", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "kimik2.5" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 64, - "seq_len_q": 1, - "seq_len_kv": 8704, - "num_heads": 8, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "int4", - "dtype_kv": "int4", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "kimik2.5" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 4, - "seq_len_q": 1024, - "seq_len_kv": 1024, - "num_heads": 16, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "int4", - "dtype_kv": "int4", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "kimik2.5" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 4, - "seq_len_q": 1, - "seq_len_kv": 1536, - "num_heads": 16, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "int4", - "dtype_kv": "int4", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "kimik2.5" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 8, - "seq_len_q": 1024, - "seq_len_kv": 1024, - "num_heads": 16, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "int4", - "dtype_kv": "int4", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "kimik2.5" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 8, - "seq_len_q": 1, - "seq_len_kv": 1536, - "num_heads": 16, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "int4", - "dtype_kv": "int4", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "kimik2.5" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 16, - "seq_len_q": 1024, - "seq_len_kv": 1024, - "num_heads": 16, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "int4", - "dtype_kv": "int4", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "kimik2.5" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 16, - "seq_len_q": 1, - "seq_len_kv": 1536, - "num_heads": 16, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "int4", - "dtype_kv": "int4", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "kimik2.5" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 32, - "seq_len_q": 1024, - "seq_len_kv": 1024, - "num_heads": 16, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "int4", - "dtype_kv": "int4", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "kimik2.5" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 32, - "seq_len_q": 1, - "seq_len_kv": 1536, - "num_heads": 16, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "int4", - "dtype_kv": "int4", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "kimik2.5" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 64, - "seq_len_q": 1024, - "seq_len_kv": 1024, - "num_heads": 16, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "int4", - "dtype_kv": "int4", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "kimik2.5" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 64, - "seq_len_q": 1, - "seq_len_kv": 1536, - "num_heads": 16, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "int4", - "dtype_kv": "int4", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "kimik2.5" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 4, - "seq_len_q": 8192, - "seq_len_kv": 8192, - "num_heads": 16, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "int4", - "dtype_kv": "int4", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "kimik2.5" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 4, - "seq_len_q": 1, - "seq_len_kv": 8704, - "num_heads": 16, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "int4", - "dtype_kv": "int4", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "kimik2.5" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 8, - "seq_len_q": 8192, - "seq_len_kv": 8192, - "num_heads": 16, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "int4", - "dtype_kv": "int4", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "kimik2.5" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 8, - "seq_len_q": 1, - "seq_len_kv": 8704, - "num_heads": 16, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "int4", - "dtype_kv": "int4", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "kimik2.5" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 16, - "seq_len_q": 8192, - "seq_len_kv": 8192, - "num_heads": 16, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "int4", - "dtype_kv": "int4", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "kimik2.5" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 16, - "seq_len_q": 1, - "seq_len_kv": 8704, - "num_heads": 16, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "int4", - "dtype_kv": "int4", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "kimik2.5" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 32, - "seq_len_q": 8192, - "seq_len_kv": 8192, - "num_heads": 16, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "int4", - "dtype_kv": "int4", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "kimik2.5" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 32, - "seq_len_q": 1, - "seq_len_kv": 8704, - "num_heads": 16, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "int4", - "dtype_kv": "int4", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "kimik2.5" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 64, - "seq_len_q": 8192, - "seq_len_kv": 8192, - "num_heads": 16, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "int4", - "dtype_kv": "int4", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "kimik2.5" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 64, - "seq_len_q": 1, - "seq_len_kv": 8704, - "num_heads": 16, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "int4", - "dtype_kv": "int4", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "kimik2.5" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 4, - "seq_len_q": 1024, - "seq_len_kv": 1024, - "num_heads": 8, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "kimik2.5" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 4, - "seq_len_q": 1, - "seq_len_kv": 1536, - "num_heads": 8, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "kimik2.5" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 4, - "seq_len_q": 8192, - "seq_len_kv": 8192, - "num_heads": 8, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "kimik2.5" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 4, - "seq_len_q": 1, - "seq_len_kv": 8704, - "num_heads": 8, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "kimik2.5" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 4, - "seq_len_q": 2, - "seq_len_kv": 1536, - "num_heads": 16, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "fp8", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 16, - "seq_len_q": 2, - "seq_len_kv": 1536, - "num_heads": 16, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "fp8", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 128, - "seq_len_q": 2, - "seq_len_kv": 1536, - "num_heads": 16, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "fp8", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 512, - "seq_len_q": 1024, - "seq_len_kv": 1024, - "num_heads": 16, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "fp8", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 512, - "seq_len_q": 2, - "seq_len_kv": 1536, - "num_heads": 16, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "fp8", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 4, - "seq_len_q": 2, - "seq_len_kv": 8704, - "num_heads": 16, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "fp8", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 16, - "seq_len_q": 2, - "seq_len_kv": 8704, - "num_heads": 16, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "fp8", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 32, - "seq_len_q": 2, - "seq_len_kv": 8704, - "num_heads": 16, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "fp8", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 128, - "seq_len_q": 2, - "seq_len_kv": 8704, - "num_heads": 16, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "fp8", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 256, - "seq_len_q": 2, - "seq_len_kv": 8704, - "num_heads": 16, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "fp8", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 512, - "seq_len_q": 8192, - "seq_len_kv": 8192, - "num_heads": 16, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "fp8", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 512, - "seq_len_q": 2, - "seq_len_kv": 8704, - "num_heads": 16, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "fp8", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 8, - "seq_len_q": 1024, - "seq_len_kv": 1024, - "num_heads": 32, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "fp8", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 8, - "seq_len_q": 1, - "seq_len_kv": 1536, - "num_heads": 32, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "fp8", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 16, - "seq_len_q": 1024, - "seq_len_kv": 1024, - "num_heads": 32, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "fp8", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 16, - "seq_len_q": 1, - "seq_len_kv": 1536, - "num_heads": 32, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "fp8", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 8, - "seq_len_q": 1, - "seq_len_kv": 8704, - "num_heads": 16, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "fp8", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 1, - "seq_len_q": 1024, - "seq_len_kv": 1024, - "num_heads": 16, - "head_dim_qk_nope": 448, - "head_dim_qk_rope": 64, - "head_dim_v": 512, - "kv_lora_rank": 1024, - "dtype_q": "fp8", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsv4" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 1, - "seq_len_q": 1, - "seq_len_kv": 1536, - "num_heads": 16, - "head_dim_qk_nope": 448, - "head_dim_qk_rope": 64, - "head_dim_v": 512, - "kv_lora_rank": 1024, - "dtype_q": "fp8", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsv4" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 2, - "seq_len_q": 1024, - "seq_len_kv": 1024, - "num_heads": 16, - "head_dim_qk_nope": 448, - "head_dim_qk_rope": 64, - "head_dim_v": 512, - "kv_lora_rank": 1024, - "dtype_q": "fp8", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsv4" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 2, - "seq_len_q": 1, - "seq_len_kv": 1536, - "num_heads": 16, - "head_dim_qk_nope": 448, - "head_dim_qk_rope": 64, - "head_dim_v": 512, - "kv_lora_rank": 1024, - "dtype_q": "fp8", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsv4" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 4, - "seq_len_q": 1024, - "seq_len_kv": 1024, - "num_heads": 16, - "head_dim_qk_nope": 448, - "head_dim_qk_rope": 64, - "head_dim_v": 512, - "kv_lora_rank": 1024, - "dtype_q": "fp8", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsv4" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 4, - "seq_len_q": 1, - "seq_len_kv": 1536, - "num_heads": 16, - "head_dim_qk_nope": 448, - "head_dim_qk_rope": 64, - "head_dim_v": 512, - "kv_lora_rank": 1024, - "dtype_q": "fp8", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsv4" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 8, - "seq_len_q": 1024, - "seq_len_kv": 1024, - "num_heads": 16, - "head_dim_qk_nope": 448, - "head_dim_qk_rope": 64, - "head_dim_v": 512, - "kv_lora_rank": 1024, - "dtype_q": "fp8", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsv4" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 8, - "seq_len_q": 1, - "seq_len_kv": 1536, - "num_heads": 16, - "head_dim_qk_nope": 448, - "head_dim_qk_rope": 64, - "head_dim_v": 512, - "kv_lora_rank": 1024, - "dtype_q": "fp8", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsv4" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 16, - "seq_len_q": 1024, - "seq_len_kv": 1024, - "num_heads": 16, - "head_dim_qk_nope": 448, - "head_dim_qk_rope": 64, - "head_dim_v": 512, - "kv_lora_rank": 1024, - "dtype_q": "fp8", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsv4" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 16, - "seq_len_q": 1, - "seq_len_kv": 1536, - "num_heads": 16, - "head_dim_qk_nope": 448, - "head_dim_qk_rope": 64, - "head_dim_v": 512, - "kv_lora_rank": 1024, - "dtype_q": "fp8", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsv4" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 32, - "seq_len_q": 1024, - "seq_len_kv": 1024, - "num_heads": 16, - "head_dim_qk_nope": 448, - "head_dim_qk_rope": 64, - "head_dim_v": 512, - "kv_lora_rank": 1024, - "dtype_q": "fp8", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsv4" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 32, - "seq_len_q": 1, - "seq_len_kv": 1536, - "num_heads": 16, - "head_dim_qk_nope": 448, - "head_dim_qk_rope": 64, - "head_dim_v": 512, - "kv_lora_rank": 1024, - "dtype_q": "fp8", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsv4" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 64, - "seq_len_q": 1024, - "seq_len_kv": 1024, - "num_heads": 16, - "head_dim_qk_nope": 448, - "head_dim_qk_rope": 64, - "head_dim_v": 512, - "kv_lora_rank": 1024, - "dtype_q": "fp8", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsv4" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 64, - "seq_len_q": 1, - "seq_len_kv": 1536, - "num_heads": 16, - "head_dim_qk_nope": 448, - "head_dim_qk_rope": 64, - "head_dim_v": 512, - "kv_lora_rank": 1024, - "dtype_q": "fp8", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsv4" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 128, - "seq_len_q": 1024, - "seq_len_kv": 1024, - "num_heads": 16, - "head_dim_qk_nope": 448, - "head_dim_qk_rope": 64, - "head_dim_v": 512, - "kv_lora_rank": 1024, - "dtype_q": "fp8", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsv4" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 128, - "seq_len_q": 1, - "seq_len_kv": 1536, - "num_heads": 16, - "head_dim_qk_nope": 448, - "head_dim_qk_rope": 64, - "head_dim_v": 512, - "kv_lora_rank": 1024, - "dtype_q": "fp8", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsv4" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 256, - "seq_len_q": 1024, - "seq_len_kv": 1024, - "num_heads": 16, - "head_dim_qk_nope": 448, - "head_dim_qk_rope": 64, - "head_dim_v": 512, - "kv_lora_rank": 1024, - "dtype_q": "fp8", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsv4" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 256, - "seq_len_q": 1, - "seq_len_kv": 1536, - "num_heads": 16, - "head_dim_qk_nope": 448, - "head_dim_qk_rope": 64, - "head_dim_v": 512, - "kv_lora_rank": 1024, - "dtype_q": "fp8", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsv4" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 1, - "seq_len_q": 1024, - "seq_len_kv": 1024, - "num_heads": 128, - "head_dim_qk_nope": 448, - "head_dim_qk_rope": 64, - "head_dim_v": 512, - "kv_lora_rank": 1024, - "dtype_q": "fp8", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsv4" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 1, - "seq_len_q": 1, - "seq_len_kv": 1536, - "num_heads": 128, - "head_dim_qk_nope": 448, - "head_dim_qk_rope": 64, - "head_dim_v": 512, - "kv_lora_rank": 1024, - "dtype_q": "fp8", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsv4" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 2, - "seq_len_q": 1024, - "seq_len_kv": 1024, - "num_heads": 128, - "head_dim_qk_nope": 448, - "head_dim_qk_rope": 64, - "head_dim_v": 512, - "kv_lora_rank": 1024, - "dtype_q": "fp8", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsv4" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 2, - "seq_len_q": 1, - "seq_len_kv": 1536, - "num_heads": 128, - "head_dim_qk_nope": 448, - "head_dim_qk_rope": 64, - "head_dim_v": 512, - "kv_lora_rank": 1024, - "dtype_q": "fp8", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsv4" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 4, - "seq_len_q": 1024, - "seq_len_kv": 1024, - "num_heads": 128, - "head_dim_qk_nope": 448, - "head_dim_qk_rope": 64, - "head_dim_v": 512, - "kv_lora_rank": 1024, - "dtype_q": "fp8", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsv4" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 4, - "seq_len_q": 1, - "seq_len_kv": 1536, - "num_heads": 128, - "head_dim_qk_nope": 448, - "head_dim_qk_rope": 64, - "head_dim_v": 512, - "kv_lora_rank": 1024, - "dtype_q": "fp8", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsv4" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 8, - "seq_len_q": 1024, - "seq_len_kv": 1024, - "num_heads": 128, - "head_dim_qk_nope": 448, - "head_dim_qk_rope": 64, - "head_dim_v": 512, - "kv_lora_rank": 1024, - "dtype_q": "fp8", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsv4" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 8, - "seq_len_q": 1, - "seq_len_kv": 1536, - "num_heads": 128, - "head_dim_qk_nope": 448, - "head_dim_qk_rope": 64, - "head_dim_v": 512, - "kv_lora_rank": 1024, - "dtype_q": "fp8", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsv4" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 16, - "seq_len_q": 1024, - "seq_len_kv": 1024, - "num_heads": 128, - "head_dim_qk_nope": 448, - "head_dim_qk_rope": 64, - "head_dim_v": 512, - "kv_lora_rank": 1024, - "dtype_q": "fp8", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsv4" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 16, - "seq_len_q": 1, - "seq_len_kv": 1536, - "num_heads": 128, - "head_dim_qk_nope": 448, - "head_dim_qk_rope": 64, - "head_dim_v": 512, - "kv_lora_rank": 1024, - "dtype_q": "fp8", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsv4" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 32, - "seq_len_q": 1024, - "seq_len_kv": 1024, - "num_heads": 128, - "head_dim_qk_nope": 448, - "head_dim_qk_rope": 64, - "head_dim_v": 512, - "kv_lora_rank": 1024, - "dtype_q": "fp8", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsv4" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 32, - "seq_len_q": 1, - "seq_len_kv": 1536, - "num_heads": 128, - "head_dim_qk_nope": 448, - "head_dim_qk_rope": 64, - "head_dim_v": 512, - "kv_lora_rank": 1024, - "dtype_q": "fp8", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsv4" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 1, - "seq_len_q": 8192, - "seq_len_kv": 8192, - "num_heads": 16, - "head_dim_qk_nope": 448, - "head_dim_qk_rope": 64, - "head_dim_v": 512, - "kv_lora_rank": 1024, - "dtype_q": "fp8", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsv4" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 1, - "seq_len_q": 1, - "seq_len_kv": 8704, - "num_heads": 16, - "head_dim_qk_nope": 448, - "head_dim_qk_rope": 64, - "head_dim_v": 512, - "kv_lora_rank": 1024, - "dtype_q": "fp8", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsv4" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 2, - "seq_len_q": 8192, - "seq_len_kv": 8192, - "num_heads": 16, - "head_dim_qk_nope": 448, - "head_dim_qk_rope": 64, - "head_dim_v": 512, - "kv_lora_rank": 1024, - "dtype_q": "fp8", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsv4" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 2, - "seq_len_q": 1, - "seq_len_kv": 8704, - "num_heads": 16, - "head_dim_qk_nope": 448, - "head_dim_qk_rope": 64, - "head_dim_v": 512, - "kv_lora_rank": 1024, - "dtype_q": "fp8", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsv4" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 4, - "seq_len_q": 8192, - "seq_len_kv": 8192, - "num_heads": 16, - "head_dim_qk_nope": 448, - "head_dim_qk_rope": 64, - "head_dim_v": 512, - "kv_lora_rank": 1024, - "dtype_q": "fp8", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsv4" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 4, - "seq_len_q": 1, - "seq_len_kv": 8704, - "num_heads": 16, - "head_dim_qk_nope": 448, - "head_dim_qk_rope": 64, - "head_dim_v": 512, - "kv_lora_rank": 1024, - "dtype_q": "fp8", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsv4" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 8, - "seq_len_q": 8192, - "seq_len_kv": 8192, - "num_heads": 16, - "head_dim_qk_nope": 448, - "head_dim_qk_rope": 64, - "head_dim_v": 512, - "kv_lora_rank": 1024, - "dtype_q": "fp8", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsv4" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 8, - "seq_len_q": 1, - "seq_len_kv": 8704, - "num_heads": 16, - "head_dim_qk_nope": 448, - "head_dim_qk_rope": 64, - "head_dim_v": 512, - "kv_lora_rank": 1024, - "dtype_q": "fp8", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsv4" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 16, - "seq_len_q": 8192, - "seq_len_kv": 8192, - "num_heads": 16, - "head_dim_qk_nope": 448, - "head_dim_qk_rope": 64, - "head_dim_v": 512, - "kv_lora_rank": 1024, - "dtype_q": "fp8", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsv4" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 16, - "seq_len_q": 1, - "seq_len_kv": 8704, - "num_heads": 16, - "head_dim_qk_nope": 448, - "head_dim_qk_rope": 64, - "head_dim_v": 512, - "kv_lora_rank": 1024, - "dtype_q": "fp8", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsv4" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 32, - "seq_len_q": 8192, - "seq_len_kv": 8192, - "num_heads": 16, - "head_dim_qk_nope": 448, - "head_dim_qk_rope": 64, - "head_dim_v": 512, - "kv_lora_rank": 1024, - "dtype_q": "fp8", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsv4" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 32, - "seq_len_q": 1, - "seq_len_kv": 8704, - "num_heads": 16, - "head_dim_qk_nope": 448, - "head_dim_qk_rope": 64, - "head_dim_v": 512, - "kv_lora_rank": 1024, - "dtype_q": "fp8", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsv4" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 64, - "seq_len_q": 8192, - "seq_len_kv": 8192, - "num_heads": 16, - "head_dim_qk_nope": 448, - "head_dim_qk_rope": 64, - "head_dim_v": 512, - "kv_lora_rank": 1024, - "dtype_q": "fp8", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsv4" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 64, - "seq_len_q": 1, - "seq_len_kv": 8704, - "num_heads": 16, - "head_dim_qk_nope": 448, - "head_dim_qk_rope": 64, - "head_dim_v": 512, - "kv_lora_rank": 1024, - "dtype_q": "fp8", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsv4" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 128, - "seq_len_q": 8192, - "seq_len_kv": 8192, - "num_heads": 16, - "head_dim_qk_nope": 448, - "head_dim_qk_rope": 64, - "head_dim_v": 512, - "kv_lora_rank": 1024, - "dtype_q": "fp8", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsv4" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 128, - "seq_len_q": 1, - "seq_len_kv": 8704, - "num_heads": 16, - "head_dim_qk_nope": 448, - "head_dim_qk_rope": 64, - "head_dim_v": 512, - "kv_lora_rank": 1024, - "dtype_q": "fp8", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsv4" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 256, - "seq_len_q": 8192, - "seq_len_kv": 8192, - "num_heads": 16, - "head_dim_qk_nope": 448, - "head_dim_qk_rope": 64, - "head_dim_v": 512, - "kv_lora_rank": 1024, - "dtype_q": "fp8", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsv4" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 256, - "seq_len_q": 1, - "seq_len_kv": 8704, - "num_heads": 16, - "head_dim_qk_nope": 448, - "head_dim_qk_rope": 64, - "head_dim_v": 512, - "kv_lora_rank": 1024, - "dtype_q": "fp8", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsv4" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 1, - "seq_len_q": 8192, - "seq_len_kv": 8192, - "num_heads": 128, - "head_dim_qk_nope": 448, - "head_dim_qk_rope": 64, - "head_dim_v": 512, - "kv_lora_rank": 1024, - "dtype_q": "fp8", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsv4" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 1, - "seq_len_q": 1, - "seq_len_kv": 8704, - "num_heads": 128, - "head_dim_qk_nope": 448, - "head_dim_qk_rope": 64, - "head_dim_v": 512, - "kv_lora_rank": 1024, - "dtype_q": "fp8", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsv4" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 2, - "seq_len_q": 8192, - "seq_len_kv": 8192, - "num_heads": 128, - "head_dim_qk_nope": 448, - "head_dim_qk_rope": 64, - "head_dim_v": 512, - "kv_lora_rank": 1024, - "dtype_q": "fp8", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsv4" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 2, - "seq_len_q": 1, - "seq_len_kv": 8704, - "num_heads": 128, - "head_dim_qk_nope": 448, - "head_dim_qk_rope": 64, - "head_dim_v": 512, - "kv_lora_rank": 1024, - "dtype_q": "fp8", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsv4" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 4, - "seq_len_q": 8192, - "seq_len_kv": 8192, - "num_heads": 128, - "head_dim_qk_nope": 448, - "head_dim_qk_rope": 64, - "head_dim_v": 512, - "kv_lora_rank": 1024, - "dtype_q": "fp8", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsv4" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 4, - "seq_len_q": 1, - "seq_len_kv": 8704, - "num_heads": 128, - "head_dim_qk_nope": 448, - "head_dim_qk_rope": 64, - "head_dim_v": 512, - "kv_lora_rank": 1024, - "dtype_q": "fp8", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsv4" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 8, - "seq_len_q": 8192, - "seq_len_kv": 8192, - "num_heads": 128, - "head_dim_qk_nope": 448, - "head_dim_qk_rope": 64, - "head_dim_v": 512, - "kv_lora_rank": 1024, - "dtype_q": "fp8", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsv4" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 8, - "seq_len_q": 1, - "seq_len_kv": 8704, - "num_heads": 128, - "head_dim_qk_nope": 448, - "head_dim_qk_rope": 64, - "head_dim_v": 512, - "kv_lora_rank": 1024, - "dtype_q": "fp8", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsv4" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 16, - "seq_len_q": 8192, - "seq_len_kv": 8192, - "num_heads": 128, - "head_dim_qk_nope": 448, - "head_dim_qk_rope": 64, - "head_dim_v": 512, - "kv_lora_rank": 1024, - "dtype_q": "fp8", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsv4" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 16, - "seq_len_q": 1, - "seq_len_kv": 8704, - "num_heads": 128, - "head_dim_qk_nope": 448, - "head_dim_qk_rope": 64, - "head_dim_v": 512, - "kv_lora_rank": 1024, - "dtype_q": "fp8", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsv4" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 32, - "seq_len_q": 8192, - "seq_len_kv": 8192, - "num_heads": 128, - "head_dim_qk_nope": 448, - "head_dim_qk_rope": 64, - "head_dim_v": 512, - "kv_lora_rank": 1024, - "dtype_q": "fp8", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsv4" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 32, - "seq_len_q": 1, - "seq_len_kv": 8704, - "num_heads": 128, - "head_dim_qk_nope": 448, - "head_dim_qk_rope": 64, - "head_dim_v": 512, - "kv_lora_rank": 1024, - "dtype_q": "fp8", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsv4" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 1, - "seq_len_q": 2, - "seq_len_kv": 1536, - "num_heads": 16, - "head_dim_qk_nope": 448, - "head_dim_qk_rope": 64, - "head_dim_v": 512, - "kv_lora_rank": 1024, - "dtype_q": "fp8", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsv4" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 2, - "seq_len_q": 2, - "seq_len_kv": 1536, - "num_heads": 16, - "head_dim_qk_nope": 448, - "head_dim_qk_rope": 64, - "head_dim_v": 512, - "kv_lora_rank": 1024, - "dtype_q": "fp8", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsv4" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 4, - "seq_len_q": 2, - "seq_len_kv": 1536, - "num_heads": 16, - "head_dim_qk_nope": 448, - "head_dim_qk_rope": 64, - "head_dim_v": 512, - "kv_lora_rank": 1024, - "dtype_q": "fp8", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsv4" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 8, - "seq_len_q": 2, - "seq_len_kv": 1536, - "num_heads": 16, - "head_dim_qk_nope": 448, - "head_dim_qk_rope": 64, - "head_dim_v": 512, - "kv_lora_rank": 1024, - "dtype_q": "fp8", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsv4" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 16, - "seq_len_q": 2, - "seq_len_kv": 1536, - "num_heads": 16, - "head_dim_qk_nope": 448, - "head_dim_qk_rope": 64, - "head_dim_v": 512, - "kv_lora_rank": 1024, - "dtype_q": "fp8", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsv4" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 32, - "seq_len_q": 2, - "seq_len_kv": 1536, - "num_heads": 16, - "head_dim_qk_nope": 448, - "head_dim_qk_rope": 64, - "head_dim_v": 512, - "kv_lora_rank": 1024, - "dtype_q": "fp8", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsv4" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 64, - "seq_len_q": 2, - "seq_len_kv": 1536, - "num_heads": 16, - "head_dim_qk_nope": 448, - "head_dim_qk_rope": 64, - "head_dim_v": 512, - "kv_lora_rank": 1024, - "dtype_q": "fp8", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsv4" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 128, - "seq_len_q": 2, - "seq_len_kv": 1536, - "num_heads": 16, - "head_dim_qk_nope": 448, - "head_dim_qk_rope": 64, - "head_dim_v": 512, - "kv_lora_rank": 1024, - "dtype_q": "fp8", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsv4" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 256, - "seq_len_q": 2, - "seq_len_kv": 1536, - "num_heads": 16, - "head_dim_qk_nope": 448, - "head_dim_qk_rope": 64, - "head_dim_v": 512, - "kv_lora_rank": 1024, - "dtype_q": "fp8", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsv4" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 1, - "seq_len_q": 2, - "seq_len_kv": 1536, - "num_heads": 128, - "head_dim_qk_nope": 448, - "head_dim_qk_rope": 64, - "head_dim_v": 512, - "kv_lora_rank": 1024, - "dtype_q": "fp8", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsv4" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 2, - "seq_len_q": 2, - "seq_len_kv": 1536, - "num_heads": 128, - "head_dim_qk_nope": 448, - "head_dim_qk_rope": 64, - "head_dim_v": 512, - "kv_lora_rank": 1024, - "dtype_q": "fp8", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsv4" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 4, - "seq_len_q": 2, - "seq_len_kv": 1536, - "num_heads": 128, - "head_dim_qk_nope": 448, - "head_dim_qk_rope": 64, - "head_dim_v": 512, - "kv_lora_rank": 1024, - "dtype_q": "fp8", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsv4" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 8, - "seq_len_q": 2, - "seq_len_kv": 1536, - "num_heads": 128, - "head_dim_qk_nope": 448, - "head_dim_qk_rope": 64, - "head_dim_v": 512, - "kv_lora_rank": 1024, - "dtype_q": "fp8", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsv4" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 16, - "seq_len_q": 2, - "seq_len_kv": 1536, - "num_heads": 128, - "head_dim_qk_nope": 448, - "head_dim_qk_rope": 64, - "head_dim_v": 512, - "kv_lora_rank": 1024, - "dtype_q": "fp8", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsv4" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 32, - "seq_len_q": 2, - "seq_len_kv": 1536, - "num_heads": 128, - "head_dim_qk_nope": 448, - "head_dim_qk_rope": 64, - "head_dim_v": 512, - "kv_lora_rank": 1024, - "dtype_q": "fp8", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsv4" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 1, - "seq_len_q": 2, - "seq_len_kv": 8704, - "num_heads": 16, - "head_dim_qk_nope": 448, - "head_dim_qk_rope": 64, - "head_dim_v": 512, - "kv_lora_rank": 1024, - "dtype_q": "fp8", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsv4" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 2, - "seq_len_q": 2, - "seq_len_kv": 8704, - "num_heads": 16, - "head_dim_qk_nope": 448, - "head_dim_qk_rope": 64, - "head_dim_v": 512, - "kv_lora_rank": 1024, - "dtype_q": "fp8", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsv4" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 4, - "seq_len_q": 2, - "seq_len_kv": 8704, - "num_heads": 16, - "head_dim_qk_nope": 448, - "head_dim_qk_rope": 64, - "head_dim_v": 512, - "kv_lora_rank": 1024, - "dtype_q": "fp8", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsv4" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 8, - "seq_len_q": 2, - "seq_len_kv": 8704, - "num_heads": 16, - "head_dim_qk_nope": 448, - "head_dim_qk_rope": 64, - "head_dim_v": 512, - "kv_lora_rank": 1024, - "dtype_q": "fp8", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsv4" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 16, - "seq_len_q": 2, - "seq_len_kv": 8704, - "num_heads": 16, - "head_dim_qk_nope": 448, - "head_dim_qk_rope": 64, - "head_dim_v": 512, - "kv_lora_rank": 1024, - "dtype_q": "fp8", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsv4" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 32, - "seq_len_q": 2, - "seq_len_kv": 8704, - "num_heads": 16, - "head_dim_qk_nope": 448, - "head_dim_qk_rope": 64, - "head_dim_v": 512, - "kv_lora_rank": 1024, - "dtype_q": "fp8", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsv4" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 64, - "seq_len_q": 2, - "seq_len_kv": 8704, - "num_heads": 16, - "head_dim_qk_nope": 448, - "head_dim_qk_rope": 64, - "head_dim_v": 512, - "kv_lora_rank": 1024, - "dtype_q": "fp8", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsv4" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 128, - "seq_len_q": 2, - "seq_len_kv": 8704, - "num_heads": 16, - "head_dim_qk_nope": 448, - "head_dim_qk_rope": 64, - "head_dim_v": 512, - "kv_lora_rank": 1024, - "dtype_q": "fp8", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsv4" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 256, - "seq_len_q": 2, - "seq_len_kv": 8704, - "num_heads": 16, - "head_dim_qk_nope": 448, - "head_dim_qk_rope": 64, - "head_dim_v": 512, - "kv_lora_rank": 1024, - "dtype_q": "fp8", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsv4" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 1, - "seq_len_q": 2, - "seq_len_kv": 8704, - "num_heads": 128, - "head_dim_qk_nope": 448, - "head_dim_qk_rope": 64, - "head_dim_v": 512, - "kv_lora_rank": 1024, - "dtype_q": "fp8", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsv4" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 2, - "seq_len_q": 2, - "seq_len_kv": 8704, - "num_heads": 128, - "head_dim_qk_nope": 448, - "head_dim_qk_rope": 64, - "head_dim_v": 512, - "kv_lora_rank": 1024, - "dtype_q": "fp8", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsv4" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 4, - "seq_len_q": 2, - "seq_len_kv": 8704, - "num_heads": 128, - "head_dim_qk_nope": 448, - "head_dim_qk_rope": 64, - "head_dim_v": 512, - "kv_lora_rank": 1024, - "dtype_q": "fp8", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsv4" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 8, - "seq_len_q": 2, - "seq_len_kv": 8704, - "num_heads": 128, - "head_dim_qk_nope": 448, - "head_dim_qk_rope": 64, - "head_dim_v": 512, - "kv_lora_rank": 1024, - "dtype_q": "fp8", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsv4" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 16, - "seq_len_q": 2, - "seq_len_kv": 8704, - "num_heads": 128, - "head_dim_qk_nope": 448, - "head_dim_qk_rope": 64, - "head_dim_v": 512, - "kv_lora_rank": 1024, - "dtype_q": "fp8", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsv4" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 32, - "seq_len_q": 2, - "seq_len_kv": 8704, - "num_heads": 128, - "head_dim_qk_nope": 448, - "head_dim_qk_rope": 64, - "head_dim_v": 512, - "kv_lora_rank": 1024, - "dtype_q": "fp8", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsv4" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 1, - "seq_len_q": 1024, - "seq_len_kv": 1024, - "num_heads": 32, - "head_dim_qk_nope": 448, - "head_dim_qk_rope": 64, - "head_dim_v": 512, - "kv_lora_rank": 1024, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsv4" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 1, - "seq_len_q": 1, - "seq_len_kv": 1536, - "num_heads": 32, - "head_dim_qk_nope": 448, - "head_dim_qk_rope": 64, - "head_dim_v": 512, - "kv_lora_rank": 1024, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsv4" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 2, - "seq_len_q": 1024, - "seq_len_kv": 1024, - "num_heads": 32, - "head_dim_qk_nope": 448, - "head_dim_qk_rope": 64, - "head_dim_v": 512, - "kv_lora_rank": 1024, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsv4" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 2, - "seq_len_q": 1, - "seq_len_kv": 1536, - "num_heads": 32, - "head_dim_qk_nope": 448, - "head_dim_qk_rope": 64, - "head_dim_v": 512, - "kv_lora_rank": 1024, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsv4" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 4, - "seq_len_q": 1, - "seq_len_kv": 1536, - "num_heads": 32, - "head_dim_qk_nope": 448, - "head_dim_qk_rope": 64, - "head_dim_v": 512, - "kv_lora_rank": 1024, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsv4" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 8, - "seq_len_q": 1, - "seq_len_kv": 1536, - "num_heads": 32, - "head_dim_qk_nope": 448, - "head_dim_qk_rope": 64, - "head_dim_v": 512, - "kv_lora_rank": 1024, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsv4" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 16, - "seq_len_q": 1, - "seq_len_kv": 1536, - "num_heads": 32, - "head_dim_qk_nope": 448, - "head_dim_qk_rope": 64, - "head_dim_v": 512, - "kv_lora_rank": 1024, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsv4" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 64, - "seq_len_q": 1024, - "seq_len_kv": 1024, - "num_heads": 32, - "head_dim_qk_nope": 448, - "head_dim_qk_rope": 64, - "head_dim_v": 512, - "kv_lora_rank": 1024, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsv4" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 64, - "seq_len_q": 1, - "seq_len_kv": 1536, - "num_heads": 32, - "head_dim_qk_nope": 448, - "head_dim_qk_rope": 64, - "head_dim_v": 512, - "kv_lora_rank": 1024, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsv4" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 128, - "seq_len_q": 1024, - "seq_len_kv": 1024, - "num_heads": 32, - "head_dim_qk_nope": 448, - "head_dim_qk_rope": 64, - "head_dim_v": 512, - "kv_lora_rank": 1024, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsv4" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 128, - "seq_len_q": 1, - "seq_len_kv": 1536, - "num_heads": 32, - "head_dim_qk_nope": 448, - "head_dim_qk_rope": 64, - "head_dim_v": 512, - "kv_lora_rank": 1024, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsv4" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 1, - "seq_len_q": 8192, - "seq_len_kv": 8192, - "num_heads": 32, - "head_dim_qk_nope": 448, - "head_dim_qk_rope": 64, - "head_dim_v": 512, - "kv_lora_rank": 1024, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsv4" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 1, - "seq_len_q": 1, - "seq_len_kv": 8704, - "num_heads": 32, - "head_dim_qk_nope": 448, - "head_dim_qk_rope": 64, - "head_dim_v": 512, - "kv_lora_rank": 1024, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsv4" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 2, - "seq_len_q": 8192, - "seq_len_kv": 8192, - "num_heads": 32, - "head_dim_qk_nope": 448, - "head_dim_qk_rope": 64, - "head_dim_v": 512, - "kv_lora_rank": 1024, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsv4" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 2, - "seq_len_q": 1, - "seq_len_kv": 8704, - "num_heads": 32, - "head_dim_qk_nope": 448, - "head_dim_qk_rope": 64, - "head_dim_v": 512, - "kv_lora_rank": 1024, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsv4" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 4, - "seq_len_q": 1, - "seq_len_kv": 8704, - "num_heads": 32, - "head_dim_qk_nope": 448, - "head_dim_qk_rope": 64, - "head_dim_v": 512, - "kv_lora_rank": 1024, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsv4" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 8, - "seq_len_q": 1, - "seq_len_kv": 8704, - "num_heads": 32, - "head_dim_qk_nope": 448, - "head_dim_qk_rope": 64, - "head_dim_v": 512, - "kv_lora_rank": 1024, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsv4" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 16, - "seq_len_q": 1, - "seq_len_kv": 8704, - "num_heads": 32, - "head_dim_qk_nope": 448, - "head_dim_qk_rope": 64, - "head_dim_v": 512, - "kv_lora_rank": 1024, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsv4" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 64, - "seq_len_q": 8192, - "seq_len_kv": 8192, - "num_heads": 32, - "head_dim_qk_nope": 448, - "head_dim_qk_rope": 64, - "head_dim_v": 512, - "kv_lora_rank": 1024, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsv4" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 64, - "seq_len_q": 1, - "seq_len_kv": 8704, - "num_heads": 32, - "head_dim_qk_nope": 448, - "head_dim_qk_rope": 64, - "head_dim_v": 512, - "kv_lora_rank": 1024, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsv4" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 1, - "seq_len_q": 2, - "seq_len_kv": 1536, - "num_heads": 32, - "head_dim_qk_nope": 448, - "head_dim_qk_rope": 64, - "head_dim_v": 512, - "kv_lora_rank": 1024, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsv4" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 2, - "seq_len_q": 2, - "seq_len_kv": 1536, - "num_heads": 32, - "head_dim_qk_nope": 448, - "head_dim_qk_rope": 64, - "head_dim_v": 512, - "kv_lora_rank": 1024, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsv4" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 64, - "seq_len_q": 2, - "seq_len_kv": 1536, - "num_heads": 32, - "head_dim_qk_nope": 448, - "head_dim_qk_rope": 64, - "head_dim_v": 512, - "kv_lora_rank": 1024, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsv4" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 1, - "seq_len_q": 2, - "seq_len_kv": 8704, - "num_heads": 32, - "head_dim_qk_nope": 448, - "head_dim_qk_rope": 64, - "head_dim_v": 512, - "kv_lora_rank": 1024, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsv4" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 2, - "seq_len_q": 2, - "seq_len_kv": 8704, - "num_heads": 32, - "head_dim_qk_nope": 448, - "head_dim_qk_rope": 64, - "head_dim_v": 512, - "kv_lora_rank": 1024, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsv4" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 64, - "seq_len_q": 2, - "seq_len_kv": 8704, - "num_heads": 128, - "head_dim_qk_nope": 448, - "head_dim_qk_rope": 64, - "head_dim_v": 512, - "kv_lora_rank": 1024, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsv4" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 128, - "seq_len_q": 2, - "seq_len_kv": 8704, - "num_heads": 128, - "head_dim_qk_nope": 448, - "head_dim_qk_rope": 64, - "head_dim_v": 512, - "kv_lora_rank": 1024, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsv4" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 128, - "seq_len_q": 2, - "seq_len_kv": 1536, - "num_heads": 32, - "head_dim_qk_nope": 448, - "head_dim_qk_rope": 64, - "head_dim_v": 512, - "kv_lora_rank": 1024, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsv4" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 256, - "seq_len_q": 1024, - "seq_len_kv": 1024, - "num_heads": 32, - "head_dim_qk_nope": 448, - "head_dim_qk_rope": 64, - "head_dim_v": 512, - "kv_lora_rank": 1024, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsv4" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 256, - "seq_len_q": 2, - "seq_len_kv": 1536, - "num_heads": 32, - "head_dim_qk_nope": 448, - "head_dim_qk_rope": 64, - "head_dim_v": 512, - "kv_lora_rank": 1024, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsv4" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 64, - "seq_len_q": 2, - "seq_len_kv": 8704, - "num_heads": 32, - "head_dim_qk_nope": 448, - "head_dim_qk_rope": 64, - "head_dim_v": 512, - "kv_lora_rank": 1024, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsv4" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 64, - "seq_len_q": 8192, - "seq_len_kv": 8192, - "num_heads": 16, - "head_dim_qk_nope": 448, - "head_dim_qk_rope": 64, - "head_dim_v": 512, - "kv_lora_rank": 1024, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsv4" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 64, - "seq_len_q": 2, - "seq_len_kv": 8704, - "num_heads": 16, - "head_dim_qk_nope": 448, - "head_dim_qk_rope": 64, - "head_dim_v": 512, - "kv_lora_rank": 1024, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsv4" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 128, - "seq_len_q": 8192, - "seq_len_kv": 8192, - "num_heads": 32, - "head_dim_qk_nope": 448, - "head_dim_qk_rope": 64, - "head_dim_v": 512, - "kv_lora_rank": 1024, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsv4" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 128, - "seq_len_q": 2, - "seq_len_kv": 8704, - "num_heads": 32, - "head_dim_qk_nope": 448, - "head_dim_qk_rope": 64, - "head_dim_v": 512, - "kv_lora_rank": 1024, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsv4" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 256, - "seq_len_q": 8192, - "seq_len_kv": 8192, - "num_heads": 32, - "head_dim_qk_nope": 448, - "head_dim_qk_rope": 64, - "head_dim_v": 512, - "kv_lora_rank": 1024, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsv4" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 256, - "seq_len_q": 2, - "seq_len_kv": 8704, - "num_heads": 32, - "head_dim_qk_nope": 448, - "head_dim_qk_rope": 64, - "head_dim_v": 512, - "kv_lora_rank": 1024, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsv4" - }, - { - "type": "attention_mha", - "args": { - "batch_size": 8, - "seq_len_q": 1024, - "seq_len_kv": 1024, - "num_heads": 4, - "num_heads_kv": 1, - "head_dim": 256, - "dtype_q": "fp8", - "dtype_k": "fp8", - "dtype_v": "fp8", - "dtype_o": "bf16", - "causal": true - }, - "name": "qwen3.5" - }, - { - "type": "attention_mha", - "args": { - "batch_size": 8, - "seq_len_q": 1, - "seq_len_kv": 1536, - "num_heads": 4, - "num_heads_kv": 1, - "head_dim": 256, - "dtype_q": "fp8", - "dtype_k": "fp8", - "dtype_v": "fp8", - "dtype_o": "bf16", - "causal": true - }, - "name": "qwen3.5" - }, - { - "type": "attention_mha", - "args": { - "batch_size": 16, - "seq_len_q": 1024, - "seq_len_kv": 1024, - "num_heads": 4, - "num_heads_kv": 1, - "head_dim": 256, - "dtype_q": "fp8", - "dtype_k": "fp8", - "dtype_v": "fp8", - "dtype_o": "bf16", - "causal": true - }, - "name": "qwen3.5" - }, - { - "type": "attention_mha", - "args": { - "batch_size": 16, - "seq_len_q": 1, - "seq_len_kv": 1536, - "num_heads": 4, - "num_heads_kv": 1, - "head_dim": 256, - "dtype_q": "fp8", - "dtype_k": "fp8", - "dtype_v": "fp8", - "dtype_o": "bf16", - "causal": true - }, - "name": "qwen3.5" - }, - { - "type": "attention_mha", - "args": { - "batch_size": 32, - "seq_len_q": 1024, - "seq_len_kv": 1024, - "num_heads": 4, - "num_heads_kv": 1, - "head_dim": 256, - "dtype_q": "fp8", - "dtype_k": "fp8", - "dtype_v": "fp8", - "dtype_o": "bf16", - "causal": true - }, - "name": "qwen3.5" - }, - { - "type": "attention_mha", - "args": { - "batch_size": 32, - "seq_len_q": 1, - "seq_len_kv": 1536, - "num_heads": 4, - "num_heads_kv": 1, - "head_dim": 256, - "dtype_q": "fp8", - "dtype_k": "fp8", - "dtype_v": "fp8", - "dtype_o": "bf16", - "causal": true - }, - "name": "qwen3.5" - }, - { - "type": "attention_mha", - "args": { - "batch_size": 64, - "seq_len_q": 1024, - "seq_len_kv": 1024, - "num_heads": 4, - "num_heads_kv": 1, - "head_dim": 256, - "dtype_q": "fp8", - "dtype_k": "fp8", - "dtype_v": "fp8", - "dtype_o": "bf16", - "causal": true - }, - "name": "qwen3.5" - }, - { - "type": "attention_mha", - "args": { - "batch_size": 64, - "seq_len_q": 1, - "seq_len_kv": 1536, - "num_heads": 4, - "num_heads_kv": 1, - "head_dim": 256, - "dtype_q": "fp8", - "dtype_k": "fp8", - "dtype_v": "fp8", - "dtype_o": "bf16", - "causal": true - }, - "name": "qwen3.5" - }, - { - "type": "attention_mha", - "args": { - "batch_size": 8, - "seq_len_q": 8192, - "seq_len_kv": 8192, - "num_heads": 4, - "num_heads_kv": 1, - "head_dim": 256, - "dtype_q": "fp8", - "dtype_k": "fp8", - "dtype_v": "fp8", - "dtype_o": "bf16", - "causal": true - }, - "name": "qwen3.5" - }, - { - "type": "attention_mha", - "args": { - "batch_size": 8, - "seq_len_q": 1, - "seq_len_kv": 8704, - "num_heads": 4, - "num_heads_kv": 1, - "head_dim": 256, - "dtype_q": "fp8", - "dtype_k": "fp8", - "dtype_v": "fp8", - "dtype_o": "bf16", - "causal": true - }, - "name": "qwen3.5" - }, - { - "type": "attention_mha", - "args": { - "batch_size": 16, - "seq_len_q": 8192, - "seq_len_kv": 8192, - "num_heads": 4, - "num_heads_kv": 1, - "head_dim": 256, - "dtype_q": "fp8", - "dtype_k": "fp8", - "dtype_v": "fp8", - "dtype_o": "bf16", - "causal": true - }, - "name": "qwen3.5" - }, - { - "type": "attention_mha", - "args": { - "batch_size": 16, - "seq_len_q": 1, - "seq_len_kv": 8704, - "num_heads": 4, - "num_heads_kv": 1, - "head_dim": 256, - "dtype_q": "fp8", - "dtype_k": "fp8", - "dtype_v": "fp8", - "dtype_o": "bf16", - "causal": true - }, - "name": "qwen3.5" - }, - { - "type": "attention_mha", - "args": { - "batch_size": 32, - "seq_len_q": 8192, - "seq_len_kv": 8192, - "num_heads": 4, - "num_heads_kv": 1, - "head_dim": 256, - "dtype_q": "fp8", - "dtype_k": "fp8", - "dtype_v": "fp8", - "dtype_o": "bf16", - "causal": true - }, - "name": "qwen3.5" - }, - { - "type": "attention_mha", - "args": { - "batch_size": 32, - "seq_len_q": 1, - "seq_len_kv": 8704, - "num_heads": 4, - "num_heads_kv": 1, - "head_dim": 256, - "dtype_q": "fp8", - "dtype_k": "fp8", - "dtype_v": "fp8", - "dtype_o": "bf16", - "causal": true - }, - "name": "qwen3.5" - }, - { - "type": "attention_mha", - "args": { - "batch_size": 64, - "seq_len_q": 8192, - "seq_len_kv": 8192, - "num_heads": 4, - "num_heads_kv": 1, - "head_dim": 256, - "dtype_q": "fp8", - "dtype_k": "fp8", - "dtype_v": "fp8", - "dtype_o": "bf16", - "causal": true - }, - "name": "qwen3.5" - }, - { - "type": "attention_mha", - "args": { - "batch_size": 64, - "seq_len_q": 1, - "seq_len_kv": 8704, - "num_heads": 4, - "num_heads_kv": 1, - "head_dim": 256, - "dtype_q": "fp8", - "dtype_k": "fp8", - "dtype_v": "fp8", - "dtype_o": "bf16", - "causal": true - }, - "name": "qwen3.5" - }, - { - "type": "attention_mha", - "args": { - "batch_size": 8, - "seq_len_q": 2, - "seq_len_kv": 1536, - "num_heads": 4, - "num_heads_kv": 1, - "head_dim": 256, - "dtype_q": "fp8", - "dtype_k": "fp8", - "dtype_v": "fp8", - "dtype_o": "bf16", - "causal": true - }, - "name": "qwen3.5" - }, - { - "type": "attention_mha", - "args": { - "batch_size": 16, - "seq_len_q": 2, - "seq_len_kv": 1536, - "num_heads": 4, - "num_heads_kv": 1, - "head_dim": 256, - "dtype_q": "fp8", - "dtype_k": "fp8", - "dtype_v": "fp8", - "dtype_o": "bf16", - "causal": true - }, - "name": "qwen3.5" - }, - { - "type": "attention_mha", - "args": { - "batch_size": 32, - "seq_len_q": 2, - "seq_len_kv": 1536, - "num_heads": 4, - "num_heads_kv": 1, - "head_dim": 256, - "dtype_q": "fp8", - "dtype_k": "fp8", - "dtype_v": "fp8", - "dtype_o": "bf16", - "causal": true - }, - "name": "qwen3.5" - }, - { - "type": "attention_mha", - "args": { - "batch_size": 64, - "seq_len_q": 2, - "seq_len_kv": 1536, - "num_heads": 4, - "num_heads_kv": 1, - "head_dim": 256, - "dtype_q": "fp8", - "dtype_k": "fp8", - "dtype_v": "fp8", - "dtype_o": "bf16", - "causal": true - }, - "name": "qwen3.5" - }, - { - "type": "attention_mha", - "args": { - "batch_size": 128, - "seq_len_q": 1024, - "seq_len_kv": 1024, - "num_heads": 4, - "num_heads_kv": 1, - "head_dim": 256, - "dtype_q": "fp8", - "dtype_k": "fp8", - "dtype_v": "fp8", - "dtype_o": "bf16", - "causal": true - }, - "name": "qwen3.5" - }, - { - "type": "attention_mha", - "args": { - "batch_size": 128, - "seq_len_q": 2, - "seq_len_kv": 1536, - "num_heads": 4, - "num_heads_kv": 1, - "head_dim": 256, - "dtype_q": "fp8", - "dtype_k": "fp8", - "dtype_v": "fp8", - "dtype_o": "bf16", - "causal": true - }, - "name": "qwen3.5" - }, - { - "type": "attention_mha", - "args": { - "batch_size": 8, - "seq_len_q": 2, - "seq_len_kv": 8704, - "num_heads": 4, - "num_heads_kv": 1, - "head_dim": 256, - "dtype_q": "fp8", - "dtype_k": "fp8", - "dtype_v": "fp8", - "dtype_o": "bf16", - "causal": true - }, - "name": "qwen3.5" - }, - { - "type": "attention_mha", - "args": { - "batch_size": 16, - "seq_len_q": 2, - "seq_len_kv": 8704, - "num_heads": 4, - "num_heads_kv": 1, - "head_dim": 256, - "dtype_q": "fp8", - "dtype_k": "fp8", - "dtype_v": "fp8", - "dtype_o": "bf16", - "causal": true - }, - "name": "qwen3.5" - }, - { - "type": "attention_mha", - "args": { - "batch_size": 32, - "seq_len_q": 2, - "seq_len_kv": 8704, - "num_heads": 4, - "num_heads_kv": 1, - "head_dim": 256, - "dtype_q": "fp8", - "dtype_k": "fp8", - "dtype_v": "fp8", - "dtype_o": "bf16", - "causal": true - }, - "name": "qwen3.5" - }, - { - "type": "attention_mha", - "args": { - "batch_size": 64, - "seq_len_q": 2, - "seq_len_kv": 8704, - "num_heads": 4, - "num_heads_kv": 1, - "head_dim": 256, - "dtype_q": "fp8", - "dtype_k": "fp8", - "dtype_v": "fp8", - "dtype_o": "bf16", - "causal": true - }, - "name": "qwen3.5" - }, - { - "type": "attention_mha", - "args": { - "batch_size": 128, - "seq_len_q": 8192, - "seq_len_kv": 8192, - "num_heads": 4, - "num_heads_kv": 1, - "head_dim": 256, - "dtype_q": "fp8", - "dtype_k": "fp8", - "dtype_v": "fp8", - "dtype_o": "bf16", - "causal": true - }, - "name": "qwen3.5" - }, - { - "type": "attention_mha", - "args": { - "batch_size": 128, - "seq_len_q": 2, - "seq_len_kv": 8704, - "num_heads": 4, - "num_heads_kv": 1, - "head_dim": 256, - "dtype_q": "fp8", - "dtype_k": "fp8", - "dtype_v": "fp8", - "dtype_o": "bf16", - "causal": true - }, - "name": "qwen3.5" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 8, - "seq_len_q": 1, - "seq_len_kv": 8704, - "num_heads": 128, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "fp8", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 16, - "seq_len_q": 2, - "seq_len_kv": 1536, - "num_heads": 128, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "fp8", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 16, - "seq_len_q": 2, - "seq_len_kv": 8704, - "num_heads": 128, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "fp8", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 32, - "seq_len_q": 2, - "seq_len_kv": 8704, - "num_heads": 128, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "fp8", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 1, - "seq_len_q": 1024, - "seq_len_kv": 1024, - "num_heads": 16, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "fp8", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 1, - "seq_len_q": 2, - "seq_len_kv": 1536, - "num_heads": 16, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "fp8", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 64, - "seq_len_q": 1024, - "seq_len_kv": 1024, - "num_heads": 128, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "fp8", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 64, - "seq_len_q": 2, - "seq_len_kv": 1536, - "num_heads": 128, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "fp8", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 1, - "seq_len_q": 1, - "seq_len_kv": 1536, - "num_heads": 16, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "fp8", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 2, - "seq_len_q": 1, - "seq_len_kv": 1536, - "num_heads": 128, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "fp8", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 8, - "seq_len_q": 1, - "seq_len_kv": 1536, - "num_heads": 128, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "fp8", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 64, - "seq_len_q": 1, - "seq_len_kv": 1536, - "num_heads": 128, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "fp8", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 1, - "seq_len_q": 2, - "seq_len_kv": 8704, - "num_heads": 16, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "fp8", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 512, - "seq_len_q": 1, - "seq_len_kv": 8704, - "num_heads": 16, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "fp8", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 6, - "seq_len_q": 1024, - "seq_len_kv": 1024, - "num_heads": 8, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "fp8", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 6, - "seq_len_q": 2, - "seq_len_kv": 1536, - "num_heads": 8, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "fp8", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 9, - "seq_len_q": 1024, - "seq_len_kv": 1024, - "num_heads": 8, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "fp8", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 9, - "seq_len_q": 2, - "seq_len_kv": 1536, - "num_heads": 8, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "fp8", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 30, - "seq_len_q": 1024, - "seq_len_kv": 1024, - "num_heads": 8, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "fp8", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 30, - "seq_len_q": 2, - "seq_len_kv": 1536, - "num_heads": 8, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "fp8", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 3, - "seq_len_q": 2, - "seq_len_kv": 1536, - "num_heads": 128, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "fp8", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 60, - "seq_len_q": 1024, - "seq_len_kv": 1024, - "num_heads": 8, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "fp8", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 60, - "seq_len_q": 2, - "seq_len_kv": 1536, - "num_heads": 8, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "fp8", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 7, - "seq_len_q": 1024, - "seq_len_kv": 1024, - "num_heads": 128, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "fp8", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 7, - "seq_len_q": 2, - "seq_len_kv": 1536, - "num_heads": 128, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "fp8", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 117, - "seq_len_q": 1024, - "seq_len_kv": 1024, - "num_heads": 8, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "fp8", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 117, - "seq_len_q": 2, - "seq_len_kv": 1536, - "num_heads": 8, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "fp8", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 14, - "seq_len_q": 1024, - "seq_len_kv": 1024, - "num_heads": 128, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "fp8", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 14, - "seq_len_q": 2, - "seq_len_kv": 1536, - "num_heads": 128, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "fp8", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 28, - "seq_len_q": 1024, - "seq_len_kv": 1024, - "num_heads": 128, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "fp8", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 28, - "seq_len_q": 2, - "seq_len_kv": 1536, - "num_heads": 128, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "fp8", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 462, - "seq_len_q": 1024, - "seq_len_kv": 1024, - "num_heads": 8, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "fp8", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 462, - "seq_len_q": 2, - "seq_len_kv": 1536, - "num_heads": 8, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "fp8", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 38, - "seq_len_q": 1024, - "seq_len_kv": 1024, - "num_heads": 128, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "fp8", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 38, - "seq_len_q": 2, - "seq_len_kv": 1536, - "num_heads": 128, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "fp8", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 76, - "seq_len_q": 1024, - "seq_len_kv": 1024, - "num_heads": 128, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "fp8", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 76, - "seq_len_q": 2, - "seq_len_kv": 1536, - "num_heads": 128, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "fp8", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 6, - "seq_len_q": 1, - "seq_len_kv": 1536, - "num_heads": 8, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "fp8", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 9, - "seq_len_q": 1, - "seq_len_kv": 1536, - "num_heads": 8, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "fp8", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 30, - "seq_len_q": 1, - "seq_len_kv": 1536, - "num_heads": 8, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "fp8", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 60, - "seq_len_q": 1, - "seq_len_kv": 1536, - "num_heads": 8, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "fp8", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 14, - "seq_len_q": 1, - "seq_len_kv": 1536, - "num_heads": 128, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "fp8", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 28, - "seq_len_q": 1, - "seq_len_kv": 1536, - "num_heads": 128, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "fp8", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 57, - "seq_len_q": 1024, - "seq_len_kv": 1024, - "num_heads": 128, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "fp8", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 57, - "seq_len_q": 1, - "seq_len_kv": 1536, - "num_heads": 128, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "fp8", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 115, - "seq_len_q": 1024, - "seq_len_kv": 1024, - "num_heads": 128, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "fp8", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 115, - "seq_len_q": 1, - "seq_len_kv": 1536, - "num_heads": 128, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "fp8", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 307, - "seq_len_q": 1024, - "seq_len_kv": 1024, - "num_heads": 128, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "fp8", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 307, - "seq_len_q": 1, - "seq_len_kv": 1536, - "num_heads": 128, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "fp8", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 6, - "seq_len_q": 8192, - "seq_len_kv": 8192, - "num_heads": 8, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "fp8", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 6, - "seq_len_q": 2, - "seq_len_kv": 8704, - "num_heads": 8, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "fp8", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 9, - "seq_len_q": 8192, - "seq_len_kv": 8192, - "num_heads": 8, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "fp8", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 9, - "seq_len_q": 2, - "seq_len_kv": 8704, - "num_heads": 8, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "fp8", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 30, - "seq_len_q": 8192, - "seq_len_kv": 8192, - "num_heads": 8, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "fp8", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 30, - "seq_len_q": 2, - "seq_len_kv": 8704, - "num_heads": 8, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "fp8", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 4, - "seq_len_q": 2, - "seq_len_kv": 8704, - "num_heads": 128, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "fp8", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 1, - "seq_len_q": 1, - "seq_len_kv": 8704, - "num_heads": 128, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "fp8", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 6, - "seq_len_q": 1, - "seq_len_kv": 8704, - "num_heads": 8, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "fp8", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 9, - "seq_len_q": 1, - "seq_len_kv": 8704, - "num_heads": 8, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "fp8", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 30, - "seq_len_q": 1, - "seq_len_kv": 8704, - "num_heads": 8, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "fp8", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 9, - "seq_len_q": 1, - "seq_len_kv": 8704, - "num_heads": 128, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "fp8", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 154, - "seq_len_q": 8192, - "seq_len_kv": 8192, - "num_heads": 8, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "fp8", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 154, - "seq_len_q": 1, - "seq_len_kv": 8704, - "num_heads": 8, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "fp8", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 19, - "seq_len_q": 8192, - "seq_len_kv": 8192, - "num_heads": 128, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "fp8", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 19, - "seq_len_q": 1, - "seq_len_kv": 8704, - "num_heads": 128, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "fp8", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mha", - "args": { - "batch_size": 256, - "seq_len_q": 1024, - "seq_len_kv": 1024, - "num_heads": 64, - "num_heads_kv": 8, - "head_dim": 64, - "dtype_q": "nvfp4", - "dtype_k": "fp8", - "dtype_v": "fp8", - "dtype_o": "bf16", - "causal": true, - "sliding_window": 128 - }, - "name": "gptoss" - }, - { - "type": "attention_mha", - "args": { - "batch_size": 256, - "seq_len_q": 1, - "seq_len_kv": 1536, - "num_heads": 64, - "num_heads_kv": 8, - "head_dim": 64, - "dtype_q": "nvfp4", - "dtype_k": "fp8", - "dtype_v": "fp8", - "dtype_o": "bf16", - "causal": true, - "sliding_window": 128 - }, - "name": "gptoss" - }, - { - "type": "attention_mha", - "args": { - "batch_size": 128, - "seq_len_q": 1024, - "seq_len_kv": 1024, - "num_heads": 64, - "num_heads_kv": 8, - "head_dim": 64, - "dtype_q": "nvfp4", - "dtype_k": "fp8", - "dtype_v": "fp8", - "dtype_o": "bf16", - "causal": true, - "sliding_window": 128 - }, - "name": "gptoss" - }, - { - "type": "attention_mha", - "args": { - "batch_size": 128, - "seq_len_q": 1, - "seq_len_kv": 1536, - "num_heads": 64, - "num_heads_kv": 8, - "head_dim": 64, - "dtype_q": "nvfp4", - "dtype_k": "fp8", - "dtype_v": "fp8", - "dtype_o": "bf16", - "causal": true, - "sliding_window": 128 - }, - "name": "gptoss" - }, - { - "type": "attention_mha", - "args": { - "batch_size": 4, - "seq_len_q": 1024, - "seq_len_kv": 1024, - "num_heads": 32, - "num_heads_kv": 4, - "head_dim": 64, - "dtype_q": "nvfp4", - "dtype_k": "fp8", - "dtype_v": "fp8", - "dtype_o": "bf16", - "causal": true, - "sliding_window": 128 - }, - "name": "gptoss" - }, - { - "type": "attention_mha", - "args": { - "batch_size": 4, - "seq_len_q": 1, - "seq_len_kv": 1536, - "num_heads": 32, - "num_heads_kv": 4, - "head_dim": 64, - "dtype_q": "nvfp4", - "dtype_k": "fp8", - "dtype_v": "fp8", - "dtype_o": "bf16", - "causal": true, - "sliding_window": 128 - }, - "name": "gptoss" - }, - { - "type": "attention_mha", - "args": { - "batch_size": 8, - "seq_len_q": 1024, - "seq_len_kv": 1024, - "num_heads": 32, - "num_heads_kv": 4, - "head_dim": 64, - "dtype_q": "nvfp4", - "dtype_k": "fp8", - "dtype_v": "fp8", - "dtype_o": "bf16", - "causal": true, - "sliding_window": 128 - }, - "name": "gptoss" - }, - { - "type": "attention_mha", - "args": { - "batch_size": 8, - "seq_len_q": 1, - "seq_len_kv": 1536, - "num_heads": 32, - "num_heads_kv": 4, - "head_dim": 64, - "dtype_q": "nvfp4", - "dtype_k": "fp8", - "dtype_v": "fp8", - "dtype_o": "bf16", - "causal": true, - "sliding_window": 128 - }, - "name": "gptoss" - }, - { - "type": "attention_mha", - "args": { - "batch_size": 16, - "seq_len_q": 1024, - "seq_len_kv": 1024, - "num_heads": 32, - "num_heads_kv": 4, - "head_dim": 64, - "dtype_q": "nvfp4", - "dtype_k": "fp8", - "dtype_v": "fp8", - "dtype_o": "bf16", - "causal": true, - "sliding_window": 128 - }, - "name": "gptoss" - }, - { - "type": "attention_mha", - "args": { - "batch_size": 16, - "seq_len_q": 1, - "seq_len_kv": 1536, - "num_heads": 32, - "num_heads_kv": 4, - "head_dim": 64, - "dtype_q": "nvfp4", - "dtype_k": "fp8", - "dtype_v": "fp8", - "dtype_o": "bf16", - "causal": true, - "sliding_window": 128 - }, - "name": "gptoss" - }, - { - "type": "attention_mha", - "args": { - "batch_size": 32, - "seq_len_q": 1024, - "seq_len_kv": 1024, - "num_heads": 32, - "num_heads_kv": 4, - "head_dim": 64, - "dtype_q": "nvfp4", - "dtype_k": "fp8", - "dtype_v": "fp8", - "dtype_o": "bf16", - "causal": true, - "sliding_window": 128 - }, - "name": "gptoss" - }, - { - "type": "attention_mha", - "args": { - "batch_size": 32, - "seq_len_q": 1, - "seq_len_kv": 1536, - "num_heads": 32, - "num_heads_kv": 4, - "head_dim": 64, - "dtype_q": "nvfp4", - "dtype_k": "fp8", - "dtype_v": "fp8", - "dtype_o": "bf16", - "causal": true, - "sliding_window": 128 - }, - "name": "gptoss" - }, - { - "type": "attention_mha", - "args": { - "batch_size": 64, - "seq_len_q": 1024, - "seq_len_kv": 1024, - "num_heads": 32, - "num_heads_kv": 4, - "head_dim": 64, - "dtype_q": "nvfp4", - "dtype_k": "fp8", - "dtype_v": "fp8", - "dtype_o": "bf16", - "causal": true, - "sliding_window": 128 - }, - "name": "gptoss" - }, - { - "type": "attention_mha", - "args": { - "batch_size": 64, - "seq_len_q": 1, - "seq_len_kv": 1536, - "num_heads": 32, - "num_heads_kv": 4, - "head_dim": 64, - "dtype_q": "nvfp4", - "dtype_k": "fp8", - "dtype_v": "fp8", - "dtype_o": "bf16", - "causal": true, - "sliding_window": 128 - }, - "name": "gptoss" - }, - { - "type": "attention_mha", - "args": { - "batch_size": 128, - "seq_len_q": 1024, - "seq_len_kv": 1024, - "num_heads": 32, - "num_heads_kv": 4, - "head_dim": 64, - "dtype_q": "nvfp4", - "dtype_k": "fp8", - "dtype_v": "fp8", - "dtype_o": "bf16", - "causal": true, - "sliding_window": 128 - }, - "name": "gptoss" - }, - { - "type": "attention_mha", - "args": { - "batch_size": 128, - "seq_len_q": 1, - "seq_len_kv": 1536, - "num_heads": 32, - "num_heads_kv": 4, - "head_dim": 64, - "dtype_q": "nvfp4", - "dtype_k": "fp8", - "dtype_v": "fp8", - "dtype_o": "bf16", - "causal": true, - "sliding_window": 128 - }, - "name": "gptoss" - }, - { - "type": "attention_mha", - "args": { - "batch_size": 256, - "seq_len_q": 1024, - "seq_len_kv": 1024, - "num_heads": 32, - "num_heads_kv": 4, - "head_dim": 64, - "dtype_q": "nvfp4", - "dtype_k": "fp8", - "dtype_v": "fp8", - "dtype_o": "bf16", - "causal": true, - "sliding_window": 128 - }, - "name": "gptoss" - }, - { - "type": "attention_mha", - "args": { - "batch_size": 256, - "seq_len_q": 1, - "seq_len_kv": 1536, - "num_heads": 32, - "num_heads_kv": 4, - "head_dim": 64, - "dtype_q": "nvfp4", - "dtype_k": "fp8", - "dtype_v": "fp8", - "dtype_o": "bf16", - "causal": true, - "sliding_window": 128 - }, - "name": "gptoss" - }, - { - "type": "attention_mha", - "args": { - "batch_size": 16, - "seq_len_q": 1024, - "seq_len_kv": 1024, - "num_heads": 64, - "num_heads_kv": 8, - "head_dim": 64, - "dtype_q": "nvfp4", - "dtype_k": "fp8", - "dtype_v": "fp8", - "dtype_o": "bf16", - "causal": true, - "sliding_window": 128 - }, - "name": "gptoss" - }, - { - "type": "attention_mha", - "args": { - "batch_size": 16, - "seq_len_q": 1, - "seq_len_kv": 1536, - "num_heads": 64, - "num_heads_kv": 8, - "head_dim": 64, - "dtype_q": "nvfp4", - "dtype_k": "fp8", - "dtype_v": "fp8", - "dtype_o": "bf16", - "causal": true, - "sliding_window": 128 - }, - "name": "gptoss" - }, - { - "type": "attention_mha", - "args": { - "batch_size": 4, - "seq_len_q": 1024, - "seq_len_kv": 1024, - "num_heads": 16, - "num_heads_kv": 2, - "head_dim": 64, - "dtype_q": "nvfp4", - "dtype_k": "fp8", - "dtype_v": "fp8", - "dtype_o": "bf16", - "causal": true, - "sliding_window": 128 - }, - "name": "gptoss" - }, - { - "type": "attention_mha", - "args": { - "batch_size": 4, - "seq_len_q": 1, - "seq_len_kv": 1536, - "num_heads": 16, - "num_heads_kv": 2, - "head_dim": 64, - "dtype_q": "nvfp4", - "dtype_k": "fp8", - "dtype_v": "fp8", - "dtype_o": "bf16", - "causal": true, - "sliding_window": 128 - }, - "name": "gptoss" - }, - { - "type": "attention_mha", - "args": { - "batch_size": 4, - "seq_len_q": 1024, - "seq_len_kv": 1024, - "num_heads": 8, - "num_heads_kv": 1, - "head_dim": 64, - "dtype_q": "nvfp4", - "dtype_k": "fp8", - "dtype_v": "fp8", - "dtype_o": "bf16", - "causal": true, - "sliding_window": 128 - }, - "name": "gptoss" - }, - { - "type": "attention_mha", - "args": { - "batch_size": 4, - "seq_len_q": 1, - "seq_len_kv": 1536, - "num_heads": 8, - "num_heads_kv": 1, - "head_dim": 64, - "dtype_q": "nvfp4", - "dtype_k": "fp8", - "dtype_v": "fp8", - "dtype_o": "bf16", - "causal": true, - "sliding_window": 128 - }, - "name": "gptoss" - }, - { - "type": "attention_mha", - "args": { - "batch_size": 4, - "seq_len_q": 8192, - "seq_len_kv": 8192, - "num_heads": 64, - "num_heads_kv": 8, - "head_dim": 64, - "dtype_q": "nvfp4", - "dtype_k": "fp8", - "dtype_v": "fp8", - "dtype_o": "bf16", - "causal": true, - "sliding_window": 128 - }, - "name": "gptoss" - }, - { - "type": "attention_mha", - "args": { - "batch_size": 4, - "seq_len_q": 1, - "seq_len_kv": 8704, - "num_heads": 64, - "num_heads_kv": 8, - "head_dim": 64, - "dtype_q": "nvfp4", - "dtype_k": "fp8", - "dtype_v": "fp8", - "dtype_o": "bf16", - "causal": true, - "sliding_window": 128 - }, - "name": "gptoss" - }, - { - "type": "attention_mha", - "args": { - "batch_size": 8, - "seq_len_q": 8192, - "seq_len_kv": 8192, - "num_heads": 64, - "num_heads_kv": 8, - "head_dim": 64, - "dtype_q": "nvfp4", - "dtype_k": "fp8", - "dtype_v": "fp8", - "dtype_o": "bf16", - "causal": true, - "sliding_window": 128 - }, - "name": "gptoss" - }, - { - "type": "attention_mha", - "args": { - "batch_size": 8, - "seq_len_q": 1, - "seq_len_kv": 8704, - "num_heads": 64, - "num_heads_kv": 8, - "head_dim": 64, - "dtype_q": "nvfp4", - "dtype_k": "fp8", - "dtype_v": "fp8", - "dtype_o": "bf16", - "causal": true, - "sliding_window": 128 - }, - "name": "gptoss" - }, - { - "type": "attention_mha", - "args": { - "batch_size": 16, - "seq_len_q": 8192, - "seq_len_kv": 8192, - "num_heads": 64, - "num_heads_kv": 8, - "head_dim": 64, - "dtype_q": "nvfp4", - "dtype_k": "fp8", - "dtype_v": "fp8", - "dtype_o": "bf16", - "causal": true, - "sliding_window": 128 - }, - "name": "gptoss" - }, - { - "type": "attention_mha", - "args": { - "batch_size": 16, - "seq_len_q": 1, - "seq_len_kv": 8704, - "num_heads": 64, - "num_heads_kv": 8, - "head_dim": 64, - "dtype_q": "nvfp4", - "dtype_k": "fp8", - "dtype_v": "fp8", - "dtype_o": "bf16", - "causal": true, - "sliding_window": 128 - }, - "name": "gptoss" - }, - { - "type": "attention_mha", - "args": { - "batch_size": 32, - "seq_len_q": 8192, - "seq_len_kv": 8192, - "num_heads": 64, - "num_heads_kv": 8, - "head_dim": 64, - "dtype_q": "nvfp4", - "dtype_k": "fp8", - "dtype_v": "fp8", - "dtype_o": "bf16", - "causal": true, - "sliding_window": 128 - }, - "name": "gptoss" - }, - { - "type": "attention_mha", - "args": { - "batch_size": 32, - "seq_len_q": 1, - "seq_len_kv": 8704, - "num_heads": 64, - "num_heads_kv": 8, - "head_dim": 64, - "dtype_q": "nvfp4", - "dtype_k": "fp8", - "dtype_v": "fp8", - "dtype_o": "bf16", - "causal": true, - "sliding_window": 128 - }, - "name": "gptoss" - }, - { - "type": "attention_mha", - "args": { - "batch_size": 64, - "seq_len_q": 8192, - "seq_len_kv": 8192, - "num_heads": 64, - "num_heads_kv": 8, - "head_dim": 64, - "dtype_q": "nvfp4", - "dtype_k": "fp8", - "dtype_v": "fp8", - "dtype_o": "bf16", - "causal": true, - "sliding_window": 128 - }, - "name": "gptoss" - }, - { - "type": "attention_mha", - "args": { - "batch_size": 64, - "seq_len_q": 1, - "seq_len_kv": 8704, - "num_heads": 64, - "num_heads_kv": 8, - "head_dim": 64, - "dtype_q": "nvfp4", - "dtype_k": "fp8", - "dtype_v": "fp8", - "dtype_o": "bf16", - "causal": true, - "sliding_window": 128 - }, - "name": "gptoss" - }, - { - "type": "attention_mha", - "args": { - "batch_size": 128, - "seq_len_q": 8192, - "seq_len_kv": 8192, - "num_heads": 64, - "num_heads_kv": 8, - "head_dim": 64, - "dtype_q": "nvfp4", - "dtype_k": "fp8", - "dtype_v": "fp8", - "dtype_o": "bf16", - "causal": true, - "sliding_window": 128 - }, - "name": "gptoss" - }, - { - "type": "attention_mha", - "args": { - "batch_size": 128, - "seq_len_q": 1, - "seq_len_kv": 8704, - "num_heads": 64, - "num_heads_kv": 8, - "head_dim": 64, - "dtype_q": "nvfp4", - "dtype_k": "fp8", - "dtype_v": "fp8", - "dtype_o": "bf16", - "causal": true, - "sliding_window": 128 - }, - "name": "gptoss" - }, - { - "type": "attention_mha", - "args": { - "batch_size": 256, - "seq_len_q": 8192, - "seq_len_kv": 8192, - "num_heads": 64, - "num_heads_kv": 8, - "head_dim": 64, - "dtype_q": "nvfp4", - "dtype_k": "fp8", - "dtype_v": "fp8", - "dtype_o": "bf16", - "causal": true, - "sliding_window": 128 - }, - "name": "gptoss" - }, - { - "type": "attention_mha", - "args": { - "batch_size": 256, - "seq_len_q": 1, - "seq_len_kv": 8704, - "num_heads": 64, - "num_heads_kv": 8, - "head_dim": 64, - "dtype_q": "nvfp4", - "dtype_k": "fp8", - "dtype_v": "fp8", - "dtype_o": "bf16", - "causal": true, - "sliding_window": 128 - }, - "name": "gptoss" - }, - { - "type": "attention_mha", - "args": { - "batch_size": 4, - "seq_len_q": 8192, - "seq_len_kv": 8192, - "num_heads": 32, - "num_heads_kv": 4, - "head_dim": 64, - "dtype_q": "nvfp4", - "dtype_k": "fp8", - "dtype_v": "fp8", - "dtype_o": "bf16", - "causal": true, - "sliding_window": 128 - }, - "name": "gptoss" - }, - { - "type": "attention_mha", - "args": { - "batch_size": 4, - "seq_len_q": 1, - "seq_len_kv": 8704, - "num_heads": 32, - "num_heads_kv": 4, - "head_dim": 64, - "dtype_q": "nvfp4", - "dtype_k": "fp8", - "dtype_v": "fp8", - "dtype_o": "bf16", - "causal": true, - "sliding_window": 128 - }, - "name": "gptoss" - }, - { - "type": "attention_mha", - "args": { - "batch_size": 8, - "seq_len_q": 8192, - "seq_len_kv": 8192, - "num_heads": 32, - "num_heads_kv": 4, - "head_dim": 64, - "dtype_q": "nvfp4", - "dtype_k": "fp8", - "dtype_v": "fp8", - "dtype_o": "bf16", - "causal": true, - "sliding_window": 128 - }, - "name": "gptoss" - }, - { - "type": "attention_mha", - "args": { - "batch_size": 8, - "seq_len_q": 1, - "seq_len_kv": 8704, - "num_heads": 32, - "num_heads_kv": 4, - "head_dim": 64, - "dtype_q": "nvfp4", - "dtype_k": "fp8", - "dtype_v": "fp8", - "dtype_o": "bf16", - "causal": true, - "sliding_window": 128 - }, - "name": "gptoss" - }, - { - "type": "attention_mha", - "args": { - "batch_size": 16, - "seq_len_q": 8192, - "seq_len_kv": 8192, - "num_heads": 32, - "num_heads_kv": 4, - "head_dim": 64, - "dtype_q": "nvfp4", - "dtype_k": "fp8", - "dtype_v": "fp8", - "dtype_o": "bf16", - "causal": true, - "sliding_window": 128 - }, - "name": "gptoss" - }, - { - "type": "attention_mha", - "args": { - "batch_size": 16, - "seq_len_q": 1, - "seq_len_kv": 8704, - "num_heads": 32, - "num_heads_kv": 4, - "head_dim": 64, - "dtype_q": "nvfp4", - "dtype_k": "fp8", - "dtype_v": "fp8", - "dtype_o": "bf16", - "causal": true, - "sliding_window": 128 - }, - "name": "gptoss" - }, - { - "type": "attention_mha", - "args": { - "batch_size": 32, - "seq_len_q": 8192, - "seq_len_kv": 8192, - "num_heads": 32, - "num_heads_kv": 4, - "head_dim": 64, - "dtype_q": "nvfp4", - "dtype_k": "fp8", - "dtype_v": "fp8", - "dtype_o": "bf16", - "causal": true, - "sliding_window": 128 - }, - "name": "gptoss" - }, - { - "type": "attention_mha", - "args": { - "batch_size": 32, - "seq_len_q": 1, - "seq_len_kv": 8704, - "num_heads": 32, - "num_heads_kv": 4, - "head_dim": 64, - "dtype_q": "nvfp4", - "dtype_k": "fp8", - "dtype_v": "fp8", - "dtype_o": "bf16", - "causal": true, - "sliding_window": 128 - }, - "name": "gptoss" - }, - { - "type": "attention_mha", - "args": { - "batch_size": 64, - "seq_len_q": 8192, - "seq_len_kv": 8192, - "num_heads": 32, - "num_heads_kv": 4, - "head_dim": 64, - "dtype_q": "nvfp4", - "dtype_k": "fp8", - "dtype_v": "fp8", - "dtype_o": "bf16", - "causal": true, - "sliding_window": 128 - }, - "name": "gptoss" - }, - { - "type": "attention_mha", - "args": { - "batch_size": 64, - "seq_len_q": 1, - "seq_len_kv": 8704, - "num_heads": 32, - "num_heads_kv": 4, - "head_dim": 64, - "dtype_q": "nvfp4", - "dtype_k": "fp8", - "dtype_v": "fp8", - "dtype_o": "bf16", - "causal": true, - "sliding_window": 128 - }, - "name": "gptoss" - }, - { - "type": "attention_mha", - "args": { - "batch_size": 128, - "seq_len_q": 8192, - "seq_len_kv": 8192, - "num_heads": 32, - "num_heads_kv": 4, - "head_dim": 64, - "dtype_q": "nvfp4", - "dtype_k": "fp8", - "dtype_v": "fp8", - "dtype_o": "bf16", - "causal": true, - "sliding_window": 128 - }, - "name": "gptoss" - }, - { - "type": "attention_mha", - "args": { - "batch_size": 128, - "seq_len_q": 1, - "seq_len_kv": 8704, - "num_heads": 32, - "num_heads_kv": 4, - "head_dim": 64, - "dtype_q": "nvfp4", - "dtype_k": "fp8", - "dtype_v": "fp8", - "dtype_o": "bf16", - "causal": true, - "sliding_window": 128 - }, - "name": "gptoss" - }, - { - "type": "attention_mha", - "args": { - "batch_size": 256, - "seq_len_q": 8192, - "seq_len_kv": 8192, - "num_heads": 32, - "num_heads_kv": 4, - "head_dim": 64, - "dtype_q": "nvfp4", - "dtype_k": "fp8", - "dtype_v": "fp8", - "dtype_o": "bf16", - "causal": true, - "sliding_window": 128 - }, - "name": "gptoss" - }, - { - "type": "attention_mha", - "args": { - "batch_size": 256, - "seq_len_q": 1, - "seq_len_kv": 8704, - "num_heads": 32, - "num_heads_kv": 4, - "head_dim": 64, - "dtype_q": "nvfp4", - "dtype_k": "fp8", - "dtype_v": "fp8", - "dtype_o": "bf16", - "causal": true, - "sliding_window": 128 - }, - "name": "gptoss" - }, - { - "type": "attention_mha", - "args": { - "batch_size": 4, - "seq_len_q": 8192, - "seq_len_kv": 8192, - "num_heads": 16, - "num_heads_kv": 2, - "head_dim": 64, - "dtype_q": "nvfp4", - "dtype_k": "fp8", - "dtype_v": "fp8", - "dtype_o": "bf16", - "causal": true, - "sliding_window": 128 - }, - "name": "gptoss" - }, - { - "type": "attention_mha", - "args": { - "batch_size": 4, - "seq_len_q": 1, - "seq_len_kv": 8704, - "num_heads": 16, - "num_heads_kv": 2, - "head_dim": 64, - "dtype_q": "nvfp4", - "dtype_k": "fp8", - "dtype_v": "fp8", - "dtype_o": "bf16", - "causal": true, - "sliding_window": 128 - }, - "name": "gptoss" - }, - { - "type": "attention_mha", - "args": { - "batch_size": 8, - "seq_len_q": 8192, - "seq_len_kv": 8192, - "num_heads": 16, - "num_heads_kv": 2, - "head_dim": 64, - "dtype_q": "nvfp4", - "dtype_k": "fp8", - "dtype_v": "fp8", - "dtype_o": "bf16", - "causal": true, - "sliding_window": 128 - }, - "name": "gptoss" - }, - { - "type": "attention_mha", - "args": { - "batch_size": 8, - "seq_len_q": 1, - "seq_len_kv": 8704, - "num_heads": 16, - "num_heads_kv": 2, - "head_dim": 64, - "dtype_q": "nvfp4", - "dtype_k": "fp8", - "dtype_v": "fp8", - "dtype_o": "bf16", - "causal": true, - "sliding_window": 128 - }, - "name": "gptoss" - }, - { - "type": "attention_mha", - "args": { - "batch_size": 16, - "seq_len_q": 8192, - "seq_len_kv": 8192, - "num_heads": 16, - "num_heads_kv": 2, - "head_dim": 64, - "dtype_q": "nvfp4", - "dtype_k": "fp8", - "dtype_v": "fp8", - "dtype_o": "bf16", - "causal": true, - "sliding_window": 128 - }, - "name": "gptoss" - }, - { - "type": "attention_mha", - "args": { - "batch_size": 16, - "seq_len_q": 1, - "seq_len_kv": 8704, - "num_heads": 16, - "num_heads_kv": 2, - "head_dim": 64, - "dtype_q": "nvfp4", - "dtype_k": "fp8", - "dtype_v": "fp8", - "dtype_o": "bf16", - "causal": true, - "sliding_window": 128 - }, - "name": "gptoss" - }, - { - "type": "attention_mha", - "args": { - "batch_size": 32, - "seq_len_q": 8192, - "seq_len_kv": 8192, - "num_heads": 16, - "num_heads_kv": 2, - "head_dim": 64, - "dtype_q": "nvfp4", - "dtype_k": "fp8", - "dtype_v": "fp8", - "dtype_o": "bf16", - "causal": true, - "sliding_window": 128 - }, - "name": "gptoss" - }, - { - "type": "attention_mha", - "args": { - "batch_size": 32, - "seq_len_q": 1, - "seq_len_kv": 8704, - "num_heads": 16, - "num_heads_kv": 2, - "head_dim": 64, - "dtype_q": "nvfp4", - "dtype_k": "fp8", - "dtype_v": "fp8", - "dtype_o": "bf16", - "causal": true, - "sliding_window": 128 - }, - "name": "gptoss" - }, - { - "type": "attention_mha", - "args": { - "batch_size": 4, - "seq_len_q": 8192, - "seq_len_kv": 8192, - "num_heads": 8, - "num_heads_kv": 1, - "head_dim": 64, - "dtype_q": "nvfp4", - "dtype_k": "fp8", - "dtype_v": "fp8", - "dtype_o": "bf16", - "causal": true, - "sliding_window": 128 - }, - "name": "gptoss" - }, - { - "type": "attention_mha", - "args": { - "batch_size": 4, - "seq_len_q": 1, - "seq_len_kv": 8704, - "num_heads": 8, - "num_heads_kv": 1, - "head_dim": 64, - "dtype_q": "nvfp4", - "dtype_k": "fp8", - "dtype_v": "fp8", - "dtype_o": "bf16", - "causal": true, - "sliding_window": 128 - }, - "name": "gptoss" - }, - { - "type": "attention_mha", - "args": { - "batch_size": 4, - "seq_len_q": 1024, - "seq_len_kv": 1024, - "num_heads": 64, - "num_heads_kv": 8, - "head_dim": 64, - "dtype_q": "nvfp4", - "dtype_k": "fp8", - "dtype_v": "fp8", - "dtype_o": "bf16", - "causal": true, - "sliding_window": 128 - }, - "name": "gptoss" - }, - { - "type": "attention_mha", - "args": { - "batch_size": 4, - "seq_len_q": 1, - "seq_len_kv": 1536, - "num_heads": 64, - "num_heads_kv": 8, - "head_dim": 64, - "dtype_q": "nvfp4", - "dtype_k": "fp8", - "dtype_v": "fp8", - "dtype_o": "bf16", - "causal": true, - "sliding_window": 128 - }, - "name": "gptoss" - }, - { - "type": "attention_mha", - "args": { - "batch_size": 8, - "seq_len_q": 1024, - "seq_len_kv": 1024, - "num_heads": 64, - "num_heads_kv": 8, - "head_dim": 64, - "dtype_q": "nvfp4", - "dtype_k": "fp8", - "dtype_v": "fp8", - "dtype_o": "bf16", - "causal": true, - "sliding_window": 128 - }, - "name": "gptoss" - }, - { - "type": "attention_mha", - "args": { - "batch_size": 8, - "seq_len_q": 1, - "seq_len_kv": 1536, - "num_heads": 64, - "num_heads_kv": 8, - "head_dim": 64, - "dtype_q": "nvfp4", - "dtype_k": "fp8", - "dtype_v": "fp8", - "dtype_o": "bf16", - "causal": true, - "sliding_window": 128 - }, - "name": "gptoss" - }, - { - "type": "attention_mha", - "args": { - "batch_size": 32, - "seq_len_q": 1024, - "seq_len_kv": 1024, - "num_heads": 64, - "num_heads_kv": 8, - "head_dim": 64, - "dtype_q": "nvfp4", - "dtype_k": "fp8", - "dtype_v": "fp8", - "dtype_o": "bf16", - "causal": true, - "sliding_window": 128 - }, - "name": "gptoss" - }, - { - "type": "attention_mha", - "args": { - "batch_size": 32, - "seq_len_q": 1, - "seq_len_kv": 1536, - "num_heads": 64, - "num_heads_kv": 8, - "head_dim": 64, - "dtype_q": "nvfp4", - "dtype_k": "fp8", - "dtype_v": "fp8", - "dtype_o": "bf16", - "causal": true, - "sliding_window": 128 - }, - "name": "gptoss" - }, - { - "type": "attention_mha", - "args": { - "batch_size": 64, - "seq_len_q": 1024, - "seq_len_kv": 1024, - "num_heads": 64, - "num_heads_kv": 8, - "head_dim": 64, - "dtype_q": "nvfp4", - "dtype_k": "fp8", - "dtype_v": "fp8", - "dtype_o": "bf16", - "causal": true, - "sliding_window": 128 - }, - "name": "gptoss" - }, - { - "type": "attention_mha", - "args": { - "batch_size": 64, - "seq_len_q": 1, - "seq_len_kv": 1536, - "num_heads": 64, - "num_heads_kv": 8, - "head_dim": 64, - "dtype_q": "nvfp4", - "dtype_k": "fp8", - "dtype_v": "fp8", - "dtype_o": "bf16", - "causal": true, - "sliding_window": 128 - }, - "name": "gptoss" - }, - { - "type": "attention_mha", - "args": { - "batch_size": 8, - "seq_len_q": 1024, - "seq_len_kv": 1024, - "num_heads": 16, - "num_heads_kv": 2, - "head_dim": 64, - "dtype_q": "nvfp4", - "dtype_k": "fp8", - "dtype_v": "fp8", - "dtype_o": "bf16", - "causal": true, - "sliding_window": 128 - }, - "name": "gptoss" - }, - { - "type": "attention_mha", - "args": { - "batch_size": 8, - "seq_len_q": 1, - "seq_len_kv": 1536, - "num_heads": 16, - "num_heads_kv": 2, - "head_dim": 64, - "dtype_q": "nvfp4", - "dtype_k": "fp8", - "dtype_v": "fp8", - "dtype_o": "bf16", - "causal": true, - "sliding_window": 128 - }, - "name": "gptoss" - }, - { - "type": "attention_mha", - "args": { - "batch_size": 16, - "seq_len_q": 1024, - "seq_len_kv": 1024, - "num_heads": 16, - "num_heads_kv": 2, - "head_dim": 64, - "dtype_q": "nvfp4", - "dtype_k": "fp8", - "dtype_v": "fp8", - "dtype_o": "bf16", - "causal": true, - "sliding_window": 128 - }, - "name": "gptoss" - }, - { - "type": "attention_mha", - "args": { - "batch_size": 16, - "seq_len_q": 1, - "seq_len_kv": 1536, - "num_heads": 16, - "num_heads_kv": 2, - "head_dim": 64, - "dtype_q": "nvfp4", - "dtype_k": "fp8", - "dtype_v": "fp8", - "dtype_o": "bf16", - "causal": true, - "sliding_window": 128 - }, - "name": "gptoss" - }, - { - "type": "attention_mha", - "args": { - "batch_size": 32, - "seq_len_q": 1024, - "seq_len_kv": 1024, - "num_heads": 16, - "num_heads_kv": 2, - "head_dim": 64, - "dtype_q": "nvfp4", - "dtype_k": "fp8", - "dtype_v": "fp8", - "dtype_o": "bf16", - "causal": true, - "sliding_window": 128 - }, - "name": "gptoss" - }, - { - "type": "attention_mha", - "args": { - "batch_size": 32, - "seq_len_q": 1, - "seq_len_kv": 1536, - "num_heads": 16, - "num_heads_kv": 2, - "head_dim": 64, - "dtype_q": "nvfp4", - "dtype_k": "fp8", - "dtype_v": "fp8", - "dtype_o": "bf16", - "causal": true, - "sliding_window": 128 - }, - "name": "gptoss" - }, - { - "type": "attention_mha", - "args": { - "batch_size": 64, - "seq_len_q": 1024, - "seq_len_kv": 1024, - "num_heads": 16, - "num_heads_kv": 2, - "head_dim": 64, - "dtype_q": "nvfp4", - "dtype_k": "fp8", - "dtype_v": "fp8", - "dtype_o": "bf16", - "causal": true, - "sliding_window": 128 - }, - "name": "gptoss" - }, - { - "type": "attention_mha", - "args": { - "batch_size": 64, - "seq_len_q": 1, - "seq_len_kv": 1536, - "num_heads": 16, - "num_heads_kv": 2, - "head_dim": 64, - "dtype_q": "nvfp4", - "dtype_k": "fp8", - "dtype_v": "fp8", - "dtype_o": "bf16", - "causal": true, - "sliding_window": 128 - }, - "name": "gptoss" - }, - { - "type": "attention_mha", - "args": { - "batch_size": 8, - "seq_len_q": 1024, - "seq_len_kv": 1024, - "num_heads": 8, - "num_heads_kv": 1, - "head_dim": 64, - "dtype_q": "nvfp4", - "dtype_k": "fp8", - "dtype_v": "fp8", - "dtype_o": "bf16", - "causal": true, - "sliding_window": 128 - }, - "name": "gptoss" - }, - { - "type": "attention_mha", - "args": { - "batch_size": 8, - "seq_len_q": 1, - "seq_len_kv": 1536, - "num_heads": 8, - "num_heads_kv": 1, - "head_dim": 64, - "dtype_q": "nvfp4", - "dtype_k": "fp8", - "dtype_v": "fp8", - "dtype_o": "bf16", - "causal": true, - "sliding_window": 128 - }, - "name": "gptoss" - }, - { - "type": "attention_mha", - "args": { - "batch_size": 64, - "seq_len_q": 8192, - "seq_len_kv": 8192, - "num_heads": 16, - "num_heads_kv": 2, - "head_dim": 64, - "dtype_q": "nvfp4", - "dtype_k": "fp8", - "dtype_v": "fp8", - "dtype_o": "bf16", - "causal": true, - "sliding_window": 128 - }, - "name": "gptoss" - }, - { - "type": "attention_mha", - "args": { - "batch_size": 64, - "seq_len_q": 1, - "seq_len_kv": 8704, - "num_heads": 16, - "num_heads_kv": 2, - "head_dim": 64, - "dtype_q": "nvfp4", - "dtype_k": "fp8", - "dtype_v": "fp8", - "dtype_o": "bf16", - "causal": true, - "sliding_window": 128 - }, - "name": "gptoss" - }, - { - "type": "attention_mha", - "args": { - "batch_size": 512, - "seq_len_q": 1024, - "seq_len_kv": 1024, - "num_heads": 24, - "num_heads_kv": 4, - "head_dim": 128, - "dtype_q": "fp8", - "dtype_k": "fp8", - "dtype_v": "fp8", - "dtype_o": "bf16", - "causal": true - }, - "name": "minimaxm2.5" - }, - { - "type": "attention_mha", - "args": { - "batch_size": 512, - "seq_len_q": 1, - "seq_len_kv": 1536, - "num_heads": 24, - "num_heads_kv": 4, - "head_dim": 128, - "dtype_q": "fp8", - "dtype_k": "fp8", - "dtype_v": "fp8", - "dtype_o": "bf16", - "causal": true - }, - "name": "minimaxm2.5" - }, - { - "type": "attention_mha", - "args": { - "batch_size": 4, - "seq_len_q": 1024, - "seq_len_kv": 1024, - "num_heads": 12, - "num_heads_kv": 2, - "head_dim": 128, - "dtype_q": "fp8", - "dtype_k": "fp8", - "dtype_v": "fp8", - "dtype_o": "bf16", - "causal": true - }, - "name": "minimaxm2.5" - }, - { - "type": "attention_mha", - "args": { - "batch_size": 4, - "seq_len_q": 1, - "seq_len_kv": 1536, - "num_heads": 12, - "num_heads_kv": 2, - "head_dim": 128, - "dtype_q": "fp8", - "dtype_k": "fp8", - "dtype_v": "fp8", - "dtype_o": "bf16", - "causal": true - }, - "name": "minimaxm2.5" - }, - { - "type": "attention_mha", - "args": { - "batch_size": 8, - "seq_len_q": 1024, - "seq_len_kv": 1024, - "num_heads": 12, - "num_heads_kv": 2, - "head_dim": 128, - "dtype_q": "fp8", - "dtype_k": "fp8", - "dtype_v": "fp8", - "dtype_o": "bf16", - "causal": true - }, - "name": "minimaxm2.5" - }, - { - "type": "attention_mha", - "args": { - "batch_size": 8, - "seq_len_q": 1, - "seq_len_kv": 1536, - "num_heads": 12, - "num_heads_kv": 2, - "head_dim": 128, - "dtype_q": "fp8", - "dtype_k": "fp8", - "dtype_v": "fp8", - "dtype_o": "bf16", - "causal": true - }, - "name": "minimaxm2.5" - }, - { - "type": "attention_mha", - "args": { - "batch_size": 16, - "seq_len_q": 1024, - "seq_len_kv": 1024, - "num_heads": 12, - "num_heads_kv": 2, - "head_dim": 128, - "dtype_q": "fp8", - "dtype_k": "fp8", - "dtype_v": "fp8", - "dtype_o": "bf16", - "causal": true - }, - "name": "minimaxm2.5" - }, - { - "type": "attention_mha", - "args": { - "batch_size": 16, - "seq_len_q": 1, - "seq_len_kv": 1536, - "num_heads": 12, - "num_heads_kv": 2, - "head_dim": 128, - "dtype_q": "fp8", - "dtype_k": "fp8", - "dtype_v": "fp8", - "dtype_o": "bf16", - "causal": true - }, - "name": "minimaxm2.5" - }, - { - "type": "attention_mha", - "args": { - "batch_size": 32, - "seq_len_q": 1024, - "seq_len_kv": 1024, - "num_heads": 12, - "num_heads_kv": 2, - "head_dim": 128, - "dtype_q": "fp8", - "dtype_k": "fp8", - "dtype_v": "fp8", - "dtype_o": "bf16", - "causal": true - }, - "name": "minimaxm2.5" - }, - { - "type": "attention_mha", - "args": { - "batch_size": 32, - "seq_len_q": 1, - "seq_len_kv": 1536, - "num_heads": 12, - "num_heads_kv": 2, - "head_dim": 128, - "dtype_q": "fp8", - "dtype_k": "fp8", - "dtype_v": "fp8", - "dtype_o": "bf16", - "causal": true - }, - "name": "minimaxm2.5" - }, - { - "type": "attention_mha", - "args": { - "batch_size": 64, - "seq_len_q": 1024, - "seq_len_kv": 1024, - "num_heads": 12, - "num_heads_kv": 2, - "head_dim": 128, - "dtype_q": "fp8", - "dtype_k": "fp8", - "dtype_v": "fp8", - "dtype_o": "bf16", - "causal": true - }, - "name": "minimaxm2.5" - }, - { - "type": "attention_mha", - "args": { - "batch_size": 64, - "seq_len_q": 1, - "seq_len_kv": 1536, - "num_heads": 12, - "num_heads_kv": 2, - "head_dim": 128, - "dtype_q": "fp8", - "dtype_k": "fp8", - "dtype_v": "fp8", - "dtype_o": "bf16", - "causal": true - }, - "name": "minimaxm2.5" - }, - { - "type": "attention_mha", - "args": { - "batch_size": 128, - "seq_len_q": 1024, - "seq_len_kv": 1024, - "num_heads": 12, - "num_heads_kv": 2, - "head_dim": 128, - "dtype_q": "fp8", - "dtype_k": "fp8", - "dtype_v": "fp8", - "dtype_o": "bf16", - "causal": true - }, - "name": "minimaxm2.5" - }, - { - "type": "attention_mha", - "args": { - "batch_size": 128, - "seq_len_q": 1, - "seq_len_kv": 1536, - "num_heads": 12, - "num_heads_kv": 2, - "head_dim": 128, - "dtype_q": "fp8", - "dtype_k": "fp8", - "dtype_v": "fp8", - "dtype_o": "bf16", - "causal": true - }, - "name": "minimaxm2.5" - }, - { - "type": "attention_mha", - "args": { - "batch_size": 256, - "seq_len_q": 1024, - "seq_len_kv": 1024, - "num_heads": 12, - "num_heads_kv": 2, - "head_dim": 128, - "dtype_q": "fp8", - "dtype_k": "fp8", - "dtype_v": "fp8", - "dtype_o": "bf16", - "causal": true - }, - "name": "minimaxm2.5" - }, - { - "type": "attention_mha", - "args": { - "batch_size": 256, - "seq_len_q": 1, - "seq_len_kv": 1536, - "num_heads": 12, - "num_heads_kv": 2, - "head_dim": 128, - "dtype_q": "fp8", - "dtype_k": "fp8", - "dtype_v": "fp8", - "dtype_o": "bf16", - "causal": true - }, - "name": "minimaxm2.5" - }, - { - "type": "attention_mha", - "args": { - "batch_size": 512, - "seq_len_q": 1024, - "seq_len_kv": 1024, - "num_heads": 12, - "num_heads_kv": 2, - "head_dim": 128, - "dtype_q": "fp8", - "dtype_k": "fp8", - "dtype_v": "fp8", - "dtype_o": "bf16", - "causal": true - }, - "name": "minimaxm2.5" - }, - { - "type": "attention_mha", - "args": { - "batch_size": 512, - "seq_len_q": 1, - "seq_len_kv": 1536, - "num_heads": 12, - "num_heads_kv": 2, - "head_dim": 128, - "dtype_q": "fp8", - "dtype_k": "fp8", - "dtype_v": "fp8", - "dtype_o": "bf16", - "causal": true - }, - "name": "minimaxm2.5" - }, - { - "type": "attention_mha", - "args": { - "batch_size": 4, - "seq_len_q": 8192, - "seq_len_kv": 8192, - "num_heads": 24, - "num_heads_kv": 4, - "head_dim": 128, - "dtype_q": "fp8", - "dtype_k": "fp8", - "dtype_v": "fp8", - "dtype_o": "bf16", - "causal": true - }, - "name": "minimaxm2.5" - }, - { - "type": "attention_mha", - "args": { - "batch_size": 4, - "seq_len_q": 1, - "seq_len_kv": 8704, - "num_heads": 24, - "num_heads_kv": 4, - "head_dim": 128, - "dtype_q": "fp8", - "dtype_k": "fp8", - "dtype_v": "fp8", - "dtype_o": "bf16", - "causal": true - }, - "name": "minimaxm2.5" - }, - { - "type": "attention_mha", - "args": { - "batch_size": 8, - "seq_len_q": 8192, - "seq_len_kv": 8192, - "num_heads": 24, - "num_heads_kv": 4, - "head_dim": 128, - "dtype_q": "fp8", - "dtype_k": "fp8", - "dtype_v": "fp8", - "dtype_o": "bf16", - "causal": true - }, - "name": "minimaxm2.5" - }, - { - "type": "attention_mha", - "args": { - "batch_size": 8, - "seq_len_q": 1, - "seq_len_kv": 8704, - "num_heads": 24, - "num_heads_kv": 4, - "head_dim": 128, - "dtype_q": "fp8", - "dtype_k": "fp8", - "dtype_v": "fp8", - "dtype_o": "bf16", - "causal": true - }, - "name": "minimaxm2.5" - }, - { - "type": "attention_mha", - "args": { - "batch_size": 16, - "seq_len_q": 8192, - "seq_len_kv": 8192, - "num_heads": 24, - "num_heads_kv": 4, - "head_dim": 128, - "dtype_q": "fp8", - "dtype_k": "fp8", - "dtype_v": "fp8", - "dtype_o": "bf16", - "causal": true - }, - "name": "minimaxm2.5" - }, - { - "type": "attention_mha", - "args": { - "batch_size": 16, - "seq_len_q": 1, - "seq_len_kv": 8704, - "num_heads": 24, - "num_heads_kv": 4, - "head_dim": 128, - "dtype_q": "fp8", - "dtype_k": "fp8", - "dtype_v": "fp8", - "dtype_o": "bf16", - "causal": true - }, - "name": "minimaxm2.5" - }, - { - "type": "attention_mha", - "args": { - "batch_size": 32, - "seq_len_q": 8192, - "seq_len_kv": 8192, - "num_heads": 24, - "num_heads_kv": 4, - "head_dim": 128, - "dtype_q": "fp8", - "dtype_k": "fp8", - "dtype_v": "fp8", - "dtype_o": "bf16", - "causal": true - }, - "name": "minimaxm2.5" - }, - { - "type": "attention_mha", - "args": { - "batch_size": 32, - "seq_len_q": 1, - "seq_len_kv": 8704, - "num_heads": 24, - "num_heads_kv": 4, - "head_dim": 128, - "dtype_q": "fp8", - "dtype_k": "fp8", - "dtype_v": "fp8", - "dtype_o": "bf16", - "causal": true - }, - "name": "minimaxm2.5" - }, - { - "type": "attention_mha", - "args": { - "batch_size": 64, - "seq_len_q": 8192, - "seq_len_kv": 8192, - "num_heads": 24, - "num_heads_kv": 4, - "head_dim": 128, - "dtype_q": "fp8", - "dtype_k": "fp8", - "dtype_v": "fp8", - "dtype_o": "bf16", - "causal": true - }, - "name": "minimaxm2.5" - }, - { - "type": "attention_mha", - "args": { - "batch_size": 64, - "seq_len_q": 1, - "seq_len_kv": 8704, - "num_heads": 24, - "num_heads_kv": 4, - "head_dim": 128, - "dtype_q": "fp8", - "dtype_k": "fp8", - "dtype_v": "fp8", - "dtype_o": "bf16", - "causal": true - }, - "name": "minimaxm2.5" - }, - { - "type": "attention_mha", - "args": { - "batch_size": 128, - "seq_len_q": 8192, - "seq_len_kv": 8192, - "num_heads": 24, - "num_heads_kv": 4, - "head_dim": 128, - "dtype_q": "fp8", - "dtype_k": "fp8", - "dtype_v": "fp8", - "dtype_o": "bf16", - "causal": true - }, - "name": "minimaxm2.5" - }, - { - "type": "attention_mha", - "args": { - "batch_size": 128, - "seq_len_q": 1, - "seq_len_kv": 8704, - "num_heads": 24, - "num_heads_kv": 4, - "head_dim": 128, - "dtype_q": "fp8", - "dtype_k": "fp8", - "dtype_v": "fp8", - "dtype_o": "bf16", - "causal": true - }, - "name": "minimaxm2.5" - }, - { - "type": "attention_mha", - "args": { - "batch_size": 256, - "seq_len_q": 8192, - "seq_len_kv": 8192, - "num_heads": 24, - "num_heads_kv": 4, - "head_dim": 128, - "dtype_q": "fp8", - "dtype_k": "fp8", - "dtype_v": "fp8", - "dtype_o": "bf16", - "causal": true - }, - "name": "minimaxm2.5" - }, - { - "type": "attention_mha", - "args": { - "batch_size": 256, - "seq_len_q": 1, - "seq_len_kv": 8704, - "num_heads": 24, - "num_heads_kv": 4, - "head_dim": 128, - "dtype_q": "fp8", - "dtype_k": "fp8", - "dtype_v": "fp8", - "dtype_o": "bf16", - "causal": true - }, - "name": "minimaxm2.5" - }, - { - "type": "attention_mha", - "args": { - "batch_size": 512, - "seq_len_q": 8192, - "seq_len_kv": 8192, - "num_heads": 24, - "num_heads_kv": 4, - "head_dim": 128, - "dtype_q": "fp8", - "dtype_k": "fp8", - "dtype_v": "fp8", - "dtype_o": "bf16", - "causal": true - }, - "name": "minimaxm2.5" - }, - { - "type": "attention_mha", - "args": { - "batch_size": 512, - "seq_len_q": 1, - "seq_len_kv": 8704, - "num_heads": 24, - "num_heads_kv": 4, - "head_dim": 128, - "dtype_q": "fp8", - "dtype_k": "fp8", - "dtype_v": "fp8", - "dtype_o": "bf16", - "causal": true - }, - "name": "minimaxm2.5" - }, - { - "type": "attention_mha", - "args": { - "batch_size": 4, - "seq_len_q": 8192, - "seq_len_kv": 8192, - "num_heads": 12, - "num_heads_kv": 2, - "head_dim": 128, - "dtype_q": "fp8", - "dtype_k": "fp8", - "dtype_v": "fp8", - "dtype_o": "bf16", - "causal": true - }, - "name": "minimaxm2.5" - }, - { - "type": "attention_mha", - "args": { - "batch_size": 4, - "seq_len_q": 1, - "seq_len_kv": 8704, - "num_heads": 12, - "num_heads_kv": 2, - "head_dim": 128, - "dtype_q": "fp8", - "dtype_k": "fp8", - "dtype_v": "fp8", - "dtype_o": "bf16", - "causal": true - }, - "name": "minimaxm2.5" - }, - { - "type": "attention_mha", - "args": { - "batch_size": 8, - "seq_len_q": 8192, - "seq_len_kv": 8192, - "num_heads": 12, - "num_heads_kv": 2, - "head_dim": 128, - "dtype_q": "fp8", - "dtype_k": "fp8", - "dtype_v": "fp8", - "dtype_o": "bf16", - "causal": true - }, - "name": "minimaxm2.5" - }, - { - "type": "attention_mha", - "args": { - "batch_size": 8, - "seq_len_q": 1, - "seq_len_kv": 8704, - "num_heads": 12, - "num_heads_kv": 2, - "head_dim": 128, - "dtype_q": "fp8", - "dtype_k": "fp8", - "dtype_v": "fp8", - "dtype_o": "bf16", - "causal": true - }, - "name": "minimaxm2.5" - }, - { - "type": "attention_mha", - "args": { - "batch_size": 16, - "seq_len_q": 8192, - "seq_len_kv": 8192, - "num_heads": 12, - "num_heads_kv": 2, - "head_dim": 128, - "dtype_q": "fp8", - "dtype_k": "fp8", - "dtype_v": "fp8", - "dtype_o": "bf16", - "causal": true - }, - "name": "minimaxm2.5" - }, - { - "type": "attention_mha", - "args": { - "batch_size": 16, - "seq_len_q": 1, - "seq_len_kv": 8704, - "num_heads": 12, - "num_heads_kv": 2, - "head_dim": 128, - "dtype_q": "fp8", - "dtype_k": "fp8", - "dtype_v": "fp8", - "dtype_o": "bf16", - "causal": true - }, - "name": "minimaxm2.5" - }, - { - "type": "attention_mha", - "args": { - "batch_size": 32, - "seq_len_q": 8192, - "seq_len_kv": 8192, - "num_heads": 12, - "num_heads_kv": 2, - "head_dim": 128, - "dtype_q": "fp8", - "dtype_k": "fp8", - "dtype_v": "fp8", - "dtype_o": "bf16", - "causal": true - }, - "name": "minimaxm2.5" - }, - { - "type": "attention_mha", - "args": { - "batch_size": 32, - "seq_len_q": 1, - "seq_len_kv": 8704, - "num_heads": 12, - "num_heads_kv": 2, - "head_dim": 128, - "dtype_q": "fp8", - "dtype_k": "fp8", - "dtype_v": "fp8", - "dtype_o": "bf16", - "causal": true - }, - "name": "minimaxm2.5" - }, - { - "type": "attention_mha", - "args": { - "batch_size": 64, - "seq_len_q": 8192, - "seq_len_kv": 8192, - "num_heads": 12, - "num_heads_kv": 2, - "head_dim": 128, - "dtype_q": "fp8", - "dtype_k": "fp8", - "dtype_v": "fp8", - "dtype_o": "bf16", - "causal": true - }, - "name": "minimaxm2.5" - }, - { - "type": "attention_mha", - "args": { - "batch_size": 64, - "seq_len_q": 1, - "seq_len_kv": 8704, - "num_heads": 12, - "num_heads_kv": 2, - "head_dim": 128, - "dtype_q": "fp8", - "dtype_k": "fp8", - "dtype_v": "fp8", - "dtype_o": "bf16", - "causal": true - }, - "name": "minimaxm2.5" - }, - { - "type": "attention_mha", - "args": { - "batch_size": 128, - "seq_len_q": 8192, - "seq_len_kv": 8192, - "num_heads": 12, - "num_heads_kv": 2, - "head_dim": 128, - "dtype_q": "fp8", - "dtype_k": "fp8", - "dtype_v": "fp8", - "dtype_o": "bf16", - "causal": true - }, - "name": "minimaxm2.5" - }, - { - "type": "attention_mha", - "args": { - "batch_size": 128, - "seq_len_q": 1, - "seq_len_kv": 8704, - "num_heads": 12, - "num_heads_kv": 2, - "head_dim": 128, - "dtype_q": "fp8", - "dtype_k": "fp8", - "dtype_v": "fp8", - "dtype_o": "bf16", - "causal": true - }, - "name": "minimaxm2.5" - }, - { - "type": "attention_mha", - "args": { - "batch_size": 256, - "seq_len_q": 8192, - "seq_len_kv": 8192, - "num_heads": 12, - "num_heads_kv": 2, - "head_dim": 128, - "dtype_q": "fp8", - "dtype_k": "fp8", - "dtype_v": "fp8", - "dtype_o": "bf16", - "causal": true - }, - "name": "minimaxm2.5" - }, - { - "type": "attention_mha", - "args": { - "batch_size": 256, - "seq_len_q": 1, - "seq_len_kv": 8704, - "num_heads": 12, - "num_heads_kv": 2, - "head_dim": 128, - "dtype_q": "fp8", - "dtype_k": "fp8", - "dtype_v": "fp8", - "dtype_o": "bf16", - "causal": true - }, - "name": "minimaxm2.5" - }, - { - "type": "attention_mha", - "args": { - "batch_size": 512, - "seq_len_q": 8192, - "seq_len_kv": 8192, - "num_heads": 12, - "num_heads_kv": 2, - "head_dim": 128, - "dtype_q": "fp8", - "dtype_k": "fp8", - "dtype_v": "fp8", - "dtype_o": "bf16", - "causal": true - }, - "name": "minimaxm2.5" - }, - { - "type": "attention_mha", - "args": { - "batch_size": 512, - "seq_len_q": 1, - "seq_len_kv": 8704, - "num_heads": 12, - "num_heads_kv": 2, - "head_dim": 128, - "dtype_q": "fp8", - "dtype_k": "fp8", - "dtype_v": "fp8", - "dtype_o": "bf16", - "causal": true - }, - "name": "minimaxm2.5" - }, - { - "type": "attention_mha", - "args": { - "batch_size": 1024, - "seq_len_q": 1024, - "seq_len_kv": 1024, - "num_heads": 24, - "num_heads_kv": 4, - "head_dim": 128, - "dtype_q": "fp8", - "dtype_k": "fp8", - "dtype_v": "fp8", - "dtype_o": "bf16", - "causal": true - }, - "name": "minimaxm2.5" - }, - { - "type": "attention_mha", - "args": { - "batch_size": 1024, - "seq_len_q": 1, - "seq_len_kv": 1536, - "num_heads": 24, - "num_heads_kv": 4, - "head_dim": 128, - "dtype_q": "fp8", - "dtype_k": "fp8", - "dtype_v": "fp8", - "dtype_o": "bf16", - "causal": true - }, - "name": "minimaxm2.5" - }, - { - "type": "attention_mha", - "args": { - "batch_size": 512, - "seq_len_q": 1024, - "seq_len_kv": 1024, - "num_heads": 48, - "num_heads_kv": 8, - "head_dim": 128, - "dtype_q": "fp8", - "dtype_k": "fp8", - "dtype_v": "fp8", - "dtype_o": "bf16", - "causal": true - }, - "name": "minimaxm2.5" - }, - { - "type": "attention_mha", - "args": { - "batch_size": 512, - "seq_len_q": 1, - "seq_len_kv": 1536, - "num_heads": 48, - "num_heads_kv": 8, - "head_dim": 128, - "dtype_q": "fp8", - "dtype_k": "fp8", - "dtype_v": "fp8", - "dtype_o": "bf16", - "causal": true - }, - "name": "minimaxm2.5" - }, - { - "type": "attention_mha", - "args": { - "batch_size": 4, - "seq_len_q": 8192, - "seq_len_kv": 8192, - "num_heads": 48, - "num_heads_kv": 8, - "head_dim": 128, - "dtype_q": "fp8", - "dtype_k": "fp8", - "dtype_v": "fp8", - "dtype_o": "bf16", - "causal": true - }, - "name": "minimaxm2.5" - }, - { - "type": "attention_mha", - "args": { - "batch_size": 4, - "seq_len_q": 1, - "seq_len_kv": 8704, - "num_heads": 48, - "num_heads_kv": 8, - "head_dim": 128, - "dtype_q": "fp8", - "dtype_k": "fp8", - "dtype_v": "fp8", - "dtype_o": "bf16", - "causal": true - }, - "name": "minimaxm2.5" - }, - { - "type": "attention_mha", - "args": { - "batch_size": 8, - "seq_len_q": 8192, - "seq_len_kv": 8192, - "num_heads": 48, - "num_heads_kv": 8, - "head_dim": 128, - "dtype_q": "fp8", - "dtype_k": "fp8", - "dtype_v": "fp8", - "dtype_o": "bf16", - "causal": true - }, - "name": "minimaxm2.5" - }, - { - "type": "attention_mha", - "args": { - "batch_size": 8, - "seq_len_q": 1, - "seq_len_kv": 8704, - "num_heads": 48, - "num_heads_kv": 8, - "head_dim": 128, - "dtype_q": "fp8", - "dtype_k": "fp8", - "dtype_v": "fp8", - "dtype_o": "bf16", - "causal": true - }, - "name": "minimaxm2.5" - }, - { - "type": "attention_mha", - "args": { - "batch_size": 16, - "seq_len_q": 8192, - "seq_len_kv": 8192, - "num_heads": 48, - "num_heads_kv": 8, - "head_dim": 128, - "dtype_q": "fp8", - "dtype_k": "fp8", - "dtype_v": "fp8", - "dtype_o": "bf16", - "causal": true - }, - "name": "minimaxm2.5" - }, - { - "type": "attention_mha", - "args": { - "batch_size": 16, - "seq_len_q": 1, - "seq_len_kv": 8704, - "num_heads": 48, - "num_heads_kv": 8, - "head_dim": 128, - "dtype_q": "fp8", - "dtype_k": "fp8", - "dtype_v": "fp8", - "dtype_o": "bf16", - "causal": true - }, - "name": "minimaxm2.5" - }, - { - "type": "attention_mha", - "args": { - "batch_size": 4, - "seq_len_q": 1024, - "seq_len_kv": 1024, - "num_heads": 48, - "num_heads_kv": 8, - "head_dim": 128, - "dtype_q": "nvfp4", - "dtype_k": "fp8", - "dtype_v": "fp8", - "dtype_o": "bf16", - "causal": true - }, - "name": "minimaxm2.5" - }, - { - "type": "attention_mha", - "args": { - "batch_size": 4, - "seq_len_q": 1, - "seq_len_kv": 1536, - "num_heads": 48, - "num_heads_kv": 8, - "head_dim": 128, - "dtype_q": "nvfp4", - "dtype_k": "fp8", - "dtype_v": "fp8", - "dtype_o": "bf16", - "causal": true - }, - "name": "minimaxm2.5" - }, - { - "type": "attention_mha", - "args": { - "batch_size": 8, - "seq_len_q": 1024, - "seq_len_kv": 1024, - "num_heads": 48, - "num_heads_kv": 8, - "head_dim": 128, - "dtype_q": "nvfp4", - "dtype_k": "fp8", - "dtype_v": "fp8", - "dtype_o": "bf16", - "causal": true - }, - "name": "minimaxm2.5" - }, - { - "type": "attention_mha", - "args": { - "batch_size": 8, - "seq_len_q": 1, - "seq_len_kv": 1536, - "num_heads": 48, - "num_heads_kv": 8, - "head_dim": 128, - "dtype_q": "nvfp4", - "dtype_k": "fp8", - "dtype_v": "fp8", - "dtype_o": "bf16", - "causal": true - }, - "name": "minimaxm2.5" - }, - { - "type": "attention_mha", - "args": { - "batch_size": 16, - "seq_len_q": 1024, - "seq_len_kv": 1024, - "num_heads": 48, - "num_heads_kv": 8, - "head_dim": 128, - "dtype_q": "nvfp4", - "dtype_k": "fp8", - "dtype_v": "fp8", - "dtype_o": "bf16", - "causal": true - }, - "name": "minimaxm2.5" - }, - { - "type": "attention_mha", - "args": { - "batch_size": 16, - "seq_len_q": 1, - "seq_len_kv": 1536, - "num_heads": 48, - "num_heads_kv": 8, - "head_dim": 128, - "dtype_q": "nvfp4", - "dtype_k": "fp8", - "dtype_v": "fp8", - "dtype_o": "bf16", - "causal": true - }, - "name": "minimaxm2.5" - }, - { - "type": "attention_mha", - "args": { - "batch_size": 16, - "seq_len_q": 1024, - "seq_len_kv": 1024, - "num_heads": 24, - "num_heads_kv": 4, - "head_dim": 128, - "dtype_q": "nvfp4", - "dtype_k": "fp8", - "dtype_v": "fp8", - "dtype_o": "bf16", - "causal": true - }, - "name": "minimaxm2.5" - }, - { - "type": "attention_mha", - "args": { - "batch_size": 16, - "seq_len_q": 1, - "seq_len_kv": 1536, - "num_heads": 24, - "num_heads_kv": 4, - "head_dim": 128, - "dtype_q": "nvfp4", - "dtype_k": "fp8", - "dtype_v": "fp8", - "dtype_o": "bf16", - "causal": true - }, - "name": "minimaxm2.5" - }, - { - "type": "attention_mha", - "args": { - "batch_size": 128, - "seq_len_q": 1024, - "seq_len_kv": 1024, - "num_heads": 24, - "num_heads_kv": 4, - "head_dim": 128, - "dtype_q": "nvfp4", - "dtype_k": "fp8", - "dtype_v": "fp8", - "dtype_o": "bf16", - "causal": true - }, - "name": "minimaxm2.5" - }, - { - "type": "attention_mha", - "args": { - "batch_size": 128, - "seq_len_q": 1, - "seq_len_kv": 1536, - "num_heads": 24, - "num_heads_kv": 4, - "head_dim": 128, - "dtype_q": "nvfp4", - "dtype_k": "fp8", - "dtype_v": "fp8", - "dtype_o": "bf16", - "causal": true - }, - "name": "minimaxm2.5" - }, - { - "type": "attention_mha", - "args": { - "batch_size": 128, - "seq_len_q": 1024, - "seq_len_kv": 1024, - "num_heads": 48, - "num_heads_kv": 8, - "head_dim": 128, - "dtype_q": "nvfp4", - "dtype_k": "fp8", - "dtype_v": "fp8", - "dtype_o": "bf16", - "causal": true - }, - "name": "minimaxm2.5" - }, - { - "type": "attention_mha", - "args": { - "batch_size": 128, - "seq_len_q": 1, - "seq_len_kv": 1536, - "num_heads": 48, - "num_heads_kv": 8, - "head_dim": 128, - "dtype_q": "nvfp4", - "dtype_k": "fp8", - "dtype_v": "fp8", - "dtype_o": "bf16", - "causal": true - }, - "name": "minimaxm2.5" - }, - { - "type": "attention_mha", - "args": { - "batch_size": 256, - "seq_len_q": 1024, - "seq_len_kv": 1024, - "num_heads": 48, - "num_heads_kv": 8, - "head_dim": 128, - "dtype_q": "nvfp4", - "dtype_k": "fp8", - "dtype_v": "fp8", - "dtype_o": "bf16", - "causal": true - }, - "name": "minimaxm2.5" - }, - { - "type": "attention_mha", - "args": { - "batch_size": 256, - "seq_len_q": 1, - "seq_len_kv": 1536, - "num_heads": 48, - "num_heads_kv": 8, - "head_dim": 128, - "dtype_q": "nvfp4", - "dtype_k": "fp8", - "dtype_v": "fp8", - "dtype_o": "bf16", - "causal": true - }, - "name": "minimaxm2.5" - }, - { - "type": "attention_mha", - "args": { - "batch_size": 512, - "seq_len_q": 1024, - "seq_len_kv": 1024, - "num_heads": 48, - "num_heads_kv": 8, - "head_dim": 128, - "dtype_q": "nvfp4", - "dtype_k": "fp8", - "dtype_v": "fp8", - "dtype_o": "bf16", - "causal": true - }, - "name": "minimaxm2.5" - }, - { - "type": "attention_mha", - "args": { - "batch_size": 512, - "seq_len_q": 1, - "seq_len_kv": 1536, - "num_heads": 48, - "num_heads_kv": 8, - "head_dim": 128, - "dtype_q": "nvfp4", - "dtype_k": "fp8", - "dtype_v": "fp8", - "dtype_o": "bf16", - "causal": true - }, - "name": "minimaxm2.5" - }, - { - "type": "attention_mha", - "args": { - "batch_size": 4, - "seq_len_q": 1024, - "seq_len_kv": 1024, - "num_heads": 12, - "num_heads_kv": 2, - "head_dim": 128, - "dtype_q": "nvfp4", - "dtype_k": "fp8", - "dtype_v": "fp8", - "dtype_o": "bf16", - "causal": true - }, - "name": "minimaxm2.5" - }, - { - "type": "attention_mha", - "args": { - "batch_size": 4, - "seq_len_q": 1, - "seq_len_kv": 1536, - "num_heads": 12, - "num_heads_kv": 2, - "head_dim": 128, - "dtype_q": "nvfp4", - "dtype_k": "fp8", - "dtype_v": "fp8", - "dtype_o": "bf16", - "causal": true - }, - "name": "minimaxm2.5" - }, - { - "type": "attention_mha", - "args": { - "batch_size": 8, - "seq_len_q": 1024, - "seq_len_kv": 1024, - "num_heads": 12, - "num_heads_kv": 2, - "head_dim": 128, - "dtype_q": "nvfp4", - "dtype_k": "fp8", - "dtype_v": "fp8", - "dtype_o": "bf16", - "causal": true - }, - "name": "minimaxm2.5" - }, - { - "type": "attention_mha", - "args": { - "batch_size": 8, - "seq_len_q": 1, - "seq_len_kv": 1536, - "num_heads": 12, - "num_heads_kv": 2, - "head_dim": 128, - "dtype_q": "nvfp4", - "dtype_k": "fp8", - "dtype_v": "fp8", - "dtype_o": "bf16", - "causal": true - }, - "name": "minimaxm2.5" - }, - { - "type": "attention_mha", - "args": { - "batch_size": 16, - "seq_len_q": 1024, - "seq_len_kv": 1024, - "num_heads": 12, - "num_heads_kv": 2, - "head_dim": 128, - "dtype_q": "nvfp4", - "dtype_k": "fp8", - "dtype_v": "fp8", - "dtype_o": "bf16", - "causal": true - }, - "name": "minimaxm2.5" - }, - { - "type": "attention_mha", - "args": { - "batch_size": 16, - "seq_len_q": 1, - "seq_len_kv": 1536, - "num_heads": 12, - "num_heads_kv": 2, - "head_dim": 128, - "dtype_q": "nvfp4", - "dtype_k": "fp8", - "dtype_v": "fp8", - "dtype_o": "bf16", - "causal": true - }, - "name": "minimaxm2.5" - }, - { - "type": "attention_mha", - "args": { - "batch_size": 64, - "seq_len_q": 1024, - "seq_len_kv": 1024, - "num_heads": 12, - "num_heads_kv": 2, - "head_dim": 128, - "dtype_q": "nvfp4", - "dtype_k": "fp8", - "dtype_v": "fp8", - "dtype_o": "bf16", - "causal": true - }, - "name": "minimaxm2.5" - }, - { - "type": "attention_mha", - "args": { - "batch_size": 64, - "seq_len_q": 1, - "seq_len_kv": 1536, - "num_heads": 12, - "num_heads_kv": 2, - "head_dim": 128, - "dtype_q": "nvfp4", - "dtype_k": "fp8", - "dtype_v": "fp8", - "dtype_o": "bf16", - "causal": true - }, - "name": "minimaxm2.5" - }, - { - "type": "attention_mha", - "args": { - "batch_size": 128, - "seq_len_q": 1024, - "seq_len_kv": 1024, - "num_heads": 12, - "num_heads_kv": 2, - "head_dim": 128, - "dtype_q": "nvfp4", - "dtype_k": "fp8", - "dtype_v": "fp8", - "dtype_o": "bf16", - "causal": true - }, - "name": "minimaxm2.5" - }, - { - "type": "attention_mha", - "args": { - "batch_size": 128, - "seq_len_q": 1, - "seq_len_kv": 1536, - "num_heads": 12, - "num_heads_kv": 2, - "head_dim": 128, - "dtype_q": "nvfp4", - "dtype_k": "fp8", - "dtype_v": "fp8", - "dtype_o": "bf16", - "causal": true - }, - "name": "minimaxm2.5" - }, - { - "type": "attention_mha", - "args": { - "batch_size": 4, - "seq_len_q": 1024, - "seq_len_kv": 1024, - "num_heads": 6, - "num_heads_kv": 1, - "head_dim": 128, - "dtype_q": "nvfp4", - "dtype_k": "fp8", - "dtype_v": "fp8", - "dtype_o": "bf16", - "causal": true - }, - "name": "minimaxm2.5" - }, - { - "type": "attention_mha", - "args": { - "batch_size": 4, - "seq_len_q": 1, - "seq_len_kv": 1536, - "num_heads": 6, - "num_heads_kv": 1, - "head_dim": 128, - "dtype_q": "nvfp4", - "dtype_k": "fp8", - "dtype_v": "fp8", - "dtype_o": "bf16", - "causal": true - }, - "name": "minimaxm2.5" - }, - { - "type": "attention_mha", - "args": { - "batch_size": 8, - "seq_len_q": 1024, - "seq_len_kv": 1024, - "num_heads": 6, - "num_heads_kv": 1, - "head_dim": 128, - "dtype_q": "nvfp4", - "dtype_k": "fp8", - "dtype_v": "fp8", - "dtype_o": "bf16", - "causal": true - }, - "name": "minimaxm2.5" - }, - { - "type": "attention_mha", - "args": { - "batch_size": 8, - "seq_len_q": 1, - "seq_len_kv": 1536, - "num_heads": 6, - "num_heads_kv": 1, - "head_dim": 128, - "dtype_q": "nvfp4", - "dtype_k": "fp8", - "dtype_v": "fp8", - "dtype_o": "bf16", - "causal": true - }, - "name": "minimaxm2.5" - }, - { - "type": "attention_mha", - "args": { - "batch_size": 4, - "seq_len_q": 8192, - "seq_len_kv": 8192, - "num_heads": 48, - "num_heads_kv": 8, - "head_dim": 128, - "dtype_q": "nvfp4", - "dtype_k": "fp8", - "dtype_v": "fp8", - "dtype_o": "bf16", - "causal": true - }, - "name": "minimaxm2.5" - }, - { - "type": "attention_mha", - "args": { - "batch_size": 4, - "seq_len_q": 1, - "seq_len_kv": 8704, - "num_heads": 48, - "num_heads_kv": 8, - "head_dim": 128, - "dtype_q": "nvfp4", - "dtype_k": "fp8", - "dtype_v": "fp8", - "dtype_o": "bf16", - "causal": true - }, - "name": "minimaxm2.5" - }, - { - "type": "attention_mha", - "args": { - "batch_size": 8, - "seq_len_q": 8192, - "seq_len_kv": 8192, - "num_heads": 48, - "num_heads_kv": 8, - "head_dim": 128, - "dtype_q": "nvfp4", - "dtype_k": "fp8", - "dtype_v": "fp8", - "dtype_o": "bf16", - "causal": true - }, - "name": "minimaxm2.5" - }, - { - "type": "attention_mha", - "args": { - "batch_size": 8, - "seq_len_q": 1, - "seq_len_kv": 8704, - "num_heads": 48, - "num_heads_kv": 8, - "head_dim": 128, - "dtype_q": "nvfp4", - "dtype_k": "fp8", - "dtype_v": "fp8", - "dtype_o": "bf16", - "causal": true - }, - "name": "minimaxm2.5" - }, - { - "type": "attention_mha", - "args": { - "batch_size": 16, - "seq_len_q": 8192, - "seq_len_kv": 8192, - "num_heads": 48, - "num_heads_kv": 8, - "head_dim": 128, - "dtype_q": "nvfp4", - "dtype_k": "fp8", - "dtype_v": "fp8", - "dtype_o": "bf16", - "causal": true - }, - "name": "minimaxm2.5" - }, - { - "type": "attention_mha", - "args": { - "batch_size": 16, - "seq_len_q": 1, - "seq_len_kv": 8704, - "num_heads": 48, - "num_heads_kv": 8, - "head_dim": 128, - "dtype_q": "nvfp4", - "dtype_k": "fp8", - "dtype_v": "fp8", - "dtype_o": "bf16", - "causal": true - }, - "name": "minimaxm2.5" - }, - { - "type": "attention_mha", - "args": { - "batch_size": 32, - "seq_len_q": 8192, - "seq_len_kv": 8192, - "num_heads": 48, - "num_heads_kv": 8, - "head_dim": 128, - "dtype_q": "nvfp4", - "dtype_k": "fp8", - "dtype_v": "fp8", - "dtype_o": "bf16", - "causal": true - }, - "name": "minimaxm2.5" - }, - { - "type": "attention_mha", - "args": { - "batch_size": 32, - "seq_len_q": 1, - "seq_len_kv": 8704, - "num_heads": 48, - "num_heads_kv": 8, - "head_dim": 128, - "dtype_q": "nvfp4", - "dtype_k": "fp8", - "dtype_v": "fp8", - "dtype_o": "bf16", - "causal": true - }, - "name": "minimaxm2.5" - }, - { - "type": "attention_mha", - "args": { - "batch_size": 256, - "seq_len_q": 8192, - "seq_len_kv": 8192, - "num_heads": 48, - "num_heads_kv": 8, - "head_dim": 128, - "dtype_q": "nvfp4", - "dtype_k": "fp8", - "dtype_v": "fp8", - "dtype_o": "bf16", - "causal": true - }, - "name": "minimaxm2.5" - }, - { - "type": "attention_mha", - "args": { - "batch_size": 256, - "seq_len_q": 1, - "seq_len_kv": 8704, - "num_heads": 48, - "num_heads_kv": 8, - "head_dim": 128, - "dtype_q": "nvfp4", - "dtype_k": "fp8", - "dtype_v": "fp8", - "dtype_o": "bf16", - "causal": true - }, - "name": "minimaxm2.5" - }, - { - "type": "attention_mha", - "args": { - "batch_size": 128, - "seq_len_q": 8192, - "seq_len_kv": 8192, - "num_heads": 24, - "num_heads_kv": 4, - "head_dim": 128, - "dtype_q": "nvfp4", - "dtype_k": "fp8", - "dtype_v": "fp8", - "dtype_o": "bf16", - "causal": true - }, - "name": "minimaxm2.5" - }, - { - "type": "attention_mha", - "args": { - "batch_size": 128, - "seq_len_q": 1, - "seq_len_kv": 8704, - "num_heads": 24, - "num_heads_kv": 4, - "head_dim": 128, - "dtype_q": "nvfp4", - "dtype_k": "fp8", - "dtype_v": "fp8", - "dtype_o": "bf16", - "causal": true - }, - "name": "minimaxm2.5" - }, - { - "type": "attention_mha", - "args": { - "batch_size": 256, - "seq_len_q": 8192, - "seq_len_kv": 8192, - "num_heads": 24, - "num_heads_kv": 4, - "head_dim": 128, - "dtype_q": "nvfp4", - "dtype_k": "fp8", - "dtype_v": "fp8", - "dtype_o": "bf16", - "causal": true - }, - "name": "minimaxm2.5" - }, - { - "type": "attention_mha", - "args": { - "batch_size": 256, - "seq_len_q": 1, - "seq_len_kv": 8704, - "num_heads": 24, - "num_heads_kv": 4, - "head_dim": 128, - "dtype_q": "nvfp4", - "dtype_k": "fp8", - "dtype_v": "fp8", - "dtype_o": "bf16", - "causal": true - }, - "name": "minimaxm2.5" - }, - { - "type": "attention_mha", - "args": { - "batch_size": 512, - "seq_len_q": 8192, - "seq_len_kv": 8192, - "num_heads": 24, - "num_heads_kv": 4, - "head_dim": 128, - "dtype_q": "nvfp4", - "dtype_k": "fp8", - "dtype_v": "fp8", - "dtype_o": "bf16", - "causal": true - }, - "name": "minimaxm2.5" - }, - { - "type": "attention_mha", - "args": { - "batch_size": 512, - "seq_len_q": 1, - "seq_len_kv": 8704, - "num_heads": 24, - "num_heads_kv": 4, - "head_dim": 128, - "dtype_q": "nvfp4", - "dtype_k": "fp8", - "dtype_v": "fp8", - "dtype_o": "bf16", - "causal": true - }, - "name": "minimaxm2.5" - }, - { - "type": "attention_mha", - "args": { - "batch_size": 4, - "seq_len_q": 8192, - "seq_len_kv": 8192, - "num_heads": 12, - "num_heads_kv": 2, - "head_dim": 128, - "dtype_q": "nvfp4", - "dtype_k": "fp8", - "dtype_v": "fp8", - "dtype_o": "bf16", - "causal": true - }, - "name": "minimaxm2.5" - }, - { - "type": "attention_mha", - "args": { - "batch_size": 4, - "seq_len_q": 1, - "seq_len_kv": 8704, - "num_heads": 12, - "num_heads_kv": 2, - "head_dim": 128, - "dtype_q": "nvfp4", - "dtype_k": "fp8", - "dtype_v": "fp8", - "dtype_o": "bf16", - "causal": true - }, - "name": "minimaxm2.5" - }, - { - "type": "attention_mha", - "args": { - "batch_size": 8, - "seq_len_q": 8192, - "seq_len_kv": 8192, - "num_heads": 12, - "num_heads_kv": 2, - "head_dim": 128, - "dtype_q": "nvfp4", - "dtype_k": "fp8", - "dtype_v": "fp8", - "dtype_o": "bf16", - "causal": true - }, - "name": "minimaxm2.5" - }, - { - "type": "attention_mha", - "args": { - "batch_size": 8, - "seq_len_q": 1, - "seq_len_kv": 8704, - "num_heads": 12, - "num_heads_kv": 2, - "head_dim": 128, - "dtype_q": "nvfp4", - "dtype_k": "fp8", - "dtype_v": "fp8", - "dtype_o": "bf16", - "causal": true - }, - "name": "minimaxm2.5" - }, - { - "type": "attention_mha", - "args": { - "batch_size": 4, - "seq_len_q": 8192, - "seq_len_kv": 8192, - "num_heads": 6, - "num_heads_kv": 1, - "head_dim": 128, - "dtype_q": "nvfp4", - "dtype_k": "fp8", - "dtype_v": "fp8", - "dtype_o": "bf16", - "causal": true - }, - "name": "minimaxm2.5" - }, - { - "type": "attention_mha", - "args": { - "batch_size": 4, - "seq_len_q": 1, - "seq_len_kv": 8704, - "num_heads": 6, - "num_heads_kv": 1, - "head_dim": 128, - "dtype_q": "nvfp4", - "dtype_k": "fp8", - "dtype_v": "fp8", - "dtype_o": "bf16", - "causal": true - }, - "name": "minimaxm2.5" - }, - { - "type": "attention_mha", - "args": { - "batch_size": 1024, - "seq_len_q": 1024, - "seq_len_kv": 1024, - "num_heads": 48, - "num_heads_kv": 8, - "head_dim": 128, - "dtype_q": "nvfp4", - "dtype_k": "fp8", - "dtype_v": "fp8", - "dtype_o": "bf16", - "causal": true - }, - "name": "minimaxm2.5" - }, - { - "type": "attention_mha", - "args": { - "batch_size": 1024, - "seq_len_q": 1, - "seq_len_kv": 1536, - "num_heads": 48, - "num_heads_kv": 8, - "head_dim": 128, - "dtype_q": "nvfp4", - "dtype_k": "fp8", - "dtype_v": "fp8", - "dtype_o": "bf16", - "causal": true - }, - "name": "minimaxm2.5" - }, - { - "type": "attention_mha", - "args": { - "batch_size": 64, - "seq_len_q": 8192, - "seq_len_kv": 8192, - "num_heads": 48, - "num_heads_kv": 8, - "head_dim": 128, - "dtype_q": "nvfp4", - "dtype_k": "fp8", - "dtype_v": "fp8", - "dtype_o": "bf16", - "causal": true - }, - "name": "minimaxm2.5" - }, - { - "type": "attention_mha", - "args": { - "batch_size": 64, - "seq_len_q": 1, - "seq_len_kv": 8704, - "num_heads": 48, - "num_heads_kv": 8, - "head_dim": 128, - "dtype_q": "nvfp4", - "dtype_k": "fp8", - "dtype_v": "fp8", - "dtype_o": "bf16", - "causal": true - }, - "name": "minimaxm2.5" - }, - { - "type": "attention_mha", - "args": { - "batch_size": 128, - "seq_len_q": 8192, - "seq_len_kv": 8192, - "num_heads": 48, - "num_heads_kv": 8, - "head_dim": 128, - "dtype_q": "nvfp4", - "dtype_k": "fp8", - "dtype_v": "fp8", - "dtype_o": "bf16", - "causal": true - }, - "name": "minimaxm2.5" - }, - { - "type": "attention_mha", - "args": { - "batch_size": 128, - "seq_len_q": 1, - "seq_len_kv": 8704, - "num_heads": 48, - "num_heads_kv": 8, - "head_dim": 128, - "dtype_q": "nvfp4", - "dtype_k": "fp8", - "dtype_v": "fp8", - "dtype_o": "bf16", - "causal": true - }, - "name": "minimaxm2.5" - }, - { - "type": "attention_mha", - "args": { - "batch_size": 16, - "seq_len_q": 1024, - "seq_len_kv": 1024, - "num_heads": 8, - "num_heads_kv": 1, - "head_dim": 64, - "dtype_q": "nvfp4", - "dtype_k": "fp8", - "dtype_v": "fp8", - "dtype_o": "bf16", - "causal": true, - "sliding_window": 128 - }, - "name": "gptoss" - }, - { - "type": "attention_mha", - "args": { - "batch_size": 16, - "seq_len_q": 1, - "seq_len_kv": 1536, - "num_heads": 8, - "num_heads_kv": 1, - "head_dim": 64, - "dtype_q": "nvfp4", - "dtype_k": "fp8", - "dtype_v": "fp8", - "dtype_o": "bf16", - "causal": true, - "sliding_window": 128 - }, - "name": "gptoss" - }, - { - "type": "attention_mha", - "args": { - "batch_size": 32, - "seq_len_q": 1024, - "seq_len_kv": 1024, - "num_heads": 8, - "num_heads_kv": 1, - "head_dim": 64, - "dtype_q": "nvfp4", - "dtype_k": "fp8", - "dtype_v": "fp8", - "dtype_o": "bf16", - "causal": true, - "sliding_window": 128 - }, - "name": "gptoss" - }, - { - "type": "attention_mha", - "args": { - "batch_size": 32, - "seq_len_q": 1, - "seq_len_kv": 1536, - "num_heads": 8, - "num_heads_kv": 1, - "head_dim": 64, - "dtype_q": "nvfp4", - "dtype_k": "fp8", - "dtype_v": "fp8", - "dtype_o": "bf16", - "causal": true, - "sliding_window": 128 - }, - "name": "gptoss" - }, - { - "type": "attention_mha", - "args": { - "batch_size": 64, - "seq_len_q": 1024, - "seq_len_kv": 1024, - "num_heads": 8, - "num_heads_kv": 1, - "head_dim": 64, - "dtype_q": "nvfp4", - "dtype_k": "fp8", - "dtype_v": "fp8", - "dtype_o": "bf16", - "causal": true, - "sliding_window": 128 - }, - "name": "gptoss" - }, - { - "type": "attention_mha", - "args": { - "batch_size": 64, - "seq_len_q": 1, - "seq_len_kv": 1536, - "num_heads": 8, - "num_heads_kv": 1, - "head_dim": 64, - "dtype_q": "nvfp4", - "dtype_k": "fp8", - "dtype_v": "fp8", - "dtype_o": "bf16", - "causal": true, - "sliding_window": 128 - }, - "name": "gptoss" - }, - { - "type": "attention_mha", - "args": { - "batch_size": 8, - "seq_len_q": 8192, - "seq_len_kv": 8192, - "num_heads": 8, - "num_heads_kv": 1, - "head_dim": 64, - "dtype_q": "nvfp4", - "dtype_k": "fp8", - "dtype_v": "fp8", - "dtype_o": "bf16", - "causal": true, - "sliding_window": 128 - }, - "name": "gptoss" - }, - { - "type": "attention_mha", - "args": { - "batch_size": 8, - "seq_len_q": 1, - "seq_len_kv": 8704, - "num_heads": 8, - "num_heads_kv": 1, - "head_dim": 64, - "dtype_q": "nvfp4", - "dtype_k": "fp8", - "dtype_v": "fp8", - "dtype_o": "bf16", - "causal": true, - "sliding_window": 128 - }, - "name": "gptoss" - }, - { - "type": "attention_mha", - "args": { - "batch_size": 16, - "seq_len_q": 8192, - "seq_len_kv": 8192, - "num_heads": 8, - "num_heads_kv": 1, - "head_dim": 64, - "dtype_q": "nvfp4", - "dtype_k": "fp8", - "dtype_v": "fp8", - "dtype_o": "bf16", - "causal": true, - "sliding_window": 128 - }, - "name": "gptoss" - }, - { - "type": "attention_mha", - "args": { - "batch_size": 16, - "seq_len_q": 1, - "seq_len_kv": 8704, - "num_heads": 8, - "num_heads_kv": 1, - "head_dim": 64, - "dtype_q": "nvfp4", - "dtype_k": "fp8", - "dtype_v": "fp8", - "dtype_o": "bf16", - "causal": true, - "sliding_window": 128 - }, - "name": "gptoss" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 1, - "seq_len_q": 1024, - "seq_len_kv": 1024, - "num_heads": 8, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "fp8", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 1, - "seq_len_q": 2, - "seq_len_kv": 1536, - "num_heads": 8, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "fp8", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 2, - "seq_len_q": 1024, - "seq_len_kv": 1024, - "num_heads": 8, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "fp8", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 2, - "seq_len_q": 2, - "seq_len_kv": 1536, - "num_heads": 8, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "fp8", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 4, - "seq_len_q": 1024, - "seq_len_kv": 1024, - "num_heads": 8, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "fp8", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 4, - "seq_len_q": 2, - "seq_len_kv": 1536, - "num_heads": 8, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "fp8", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 8, - "seq_len_q": 1024, - "seq_len_kv": 1024, - "num_heads": 8, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "fp8", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 8, - "seq_len_q": 2, - "seq_len_kv": 1536, - "num_heads": 8, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "fp8", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 16, - "seq_len_q": 1024, - "seq_len_kv": 1024, - "num_heads": 8, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "fp8", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 16, - "seq_len_q": 2, - "seq_len_kv": 1536, - "num_heads": 8, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "fp8", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 32, - "seq_len_q": 1024, - "seq_len_kv": 1024, - "num_heads": 8, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "fp8", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 32, - "seq_len_q": 2, - "seq_len_kv": 1536, - "num_heads": 8, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "fp8", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 64, - "seq_len_q": 1024, - "seq_len_kv": 1024, - "num_heads": 8, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "fp8", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 64, - "seq_len_q": 2, - "seq_len_kv": 1536, - "num_heads": 8, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "fp8", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 128, - "seq_len_q": 1024, - "seq_len_kv": 1024, - "num_heads": 8, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "fp8", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 128, - "seq_len_q": 2, - "seq_len_kv": 1536, - "num_heads": 8, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "fp8", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 1, - "seq_len_q": 8192, - "seq_len_kv": 8192, - "num_heads": 8, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "fp8", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 1, - "seq_len_q": 2, - "seq_len_kv": 8704, - "num_heads": 8, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "fp8", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 2, - "seq_len_q": 8192, - "seq_len_kv": 8192, - "num_heads": 8, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "fp8", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 2, - "seq_len_q": 2, - "seq_len_kv": 8704, - "num_heads": 8, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "fp8", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 4, - "seq_len_q": 8192, - "seq_len_kv": 8192, - "num_heads": 8, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "fp8", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 4, - "seq_len_q": 2, - "seq_len_kv": 8704, - "num_heads": 8, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "fp8", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 8, - "seq_len_q": 8192, - "seq_len_kv": 8192, - "num_heads": 8, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "fp8", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 8, - "seq_len_q": 2, - "seq_len_kv": 8704, - "num_heads": 8, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "fp8", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 16, - "seq_len_q": 8192, - "seq_len_kv": 8192, - "num_heads": 8, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "fp8", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 16, - "seq_len_q": 2, - "seq_len_kv": 8704, - "num_heads": 8, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "fp8", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 32, - "seq_len_q": 8192, - "seq_len_kv": 8192, - "num_heads": 8, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "fp8", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 32, - "seq_len_q": 2, - "seq_len_kv": 8704, - "num_heads": 8, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "fp8", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 64, - "seq_len_q": 8192, - "seq_len_kv": 8192, - "num_heads": 8, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "fp8", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 64, - "seq_len_q": 2, - "seq_len_kv": 8704, - "num_heads": 8, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "fp8", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 128, - "seq_len_q": 8192, - "seq_len_kv": 8192, - "num_heads": 8, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "fp8", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 128, - "seq_len_q": 2, - "seq_len_kv": 8704, - "num_heads": 8, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "fp8", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mha", - "args": { - "batch_size": 32, - "seq_len_q": 8192, - "seq_len_kv": 8192, - "num_heads": 8, - "num_heads_kv": 1, - "head_dim": 64, - "dtype_q": "nvfp4", - "dtype_k": "fp8", - "dtype_v": "fp8", - "dtype_o": "bf16", - "causal": true, - "sliding_window": 128 - }, - "name": "gptoss" - }, - { - "type": "attention_mha", - "args": { - "batch_size": 32, - "seq_len_q": 1, - "seq_len_kv": 8704, - "num_heads": 8, - "num_heads_kv": 1, - "head_dim": 64, - "dtype_q": "nvfp4", - "dtype_k": "fp8", - "dtype_v": "fp8", - "dtype_o": "bf16", - "causal": true, - "sliding_window": 128 - }, - "name": "gptoss" - }, - { - "type": "attention_mha", - "args": { - "batch_size": 4, - "seq_len_q": 1024, - "seq_len_kv": 1024, - "num_heads": 6, - "num_heads_kv": 1, - "head_dim": 128, - "dtype_q": "fp8", - "dtype_k": "fp8", - "dtype_v": "fp8", - "dtype_o": "bf16", - "causal": true - }, - "name": "minimaxm2.5" - }, - { - "type": "attention_mha", - "args": { - "batch_size": 4, - "seq_len_q": 1, - "seq_len_kv": 1536, - "num_heads": 6, - "num_heads_kv": 1, - "head_dim": 128, - "dtype_q": "fp8", - "dtype_k": "fp8", - "dtype_v": "fp8", - "dtype_o": "bf16", - "causal": true - }, - "name": "minimaxm2.5" - }, - { - "type": "attention_mha", - "args": { - "batch_size": 8, - "seq_len_q": 1024, - "seq_len_kv": 1024, - "num_heads": 6, - "num_heads_kv": 1, - "head_dim": 128, - "dtype_q": "fp8", - "dtype_k": "fp8", - "dtype_v": "fp8", - "dtype_o": "bf16", - "causal": true - }, - "name": "minimaxm2.5" - }, - { - "type": "attention_mha", - "args": { - "batch_size": 8, - "seq_len_q": 1, - "seq_len_kv": 1536, - "num_heads": 6, - "num_heads_kv": 1, - "head_dim": 128, - "dtype_q": "fp8", - "dtype_k": "fp8", - "dtype_v": "fp8", - "dtype_o": "bf16", - "causal": true - }, - "name": "minimaxm2.5" - }, - { - "type": "attention_mha", - "args": { - "batch_size": 16, - "seq_len_q": 1024, - "seq_len_kv": 1024, - "num_heads": 6, - "num_heads_kv": 1, - "head_dim": 128, - "dtype_q": "fp8", - "dtype_k": "fp8", - "dtype_v": "fp8", - "dtype_o": "bf16", - "causal": true - }, - "name": "minimaxm2.5" - }, - { - "type": "attention_mha", - "args": { - "batch_size": 16, - "seq_len_q": 1, - "seq_len_kv": 1536, - "num_heads": 6, - "num_heads_kv": 1, - "head_dim": 128, - "dtype_q": "fp8", - "dtype_k": "fp8", - "dtype_v": "fp8", - "dtype_o": "bf16", - "causal": true - }, - "name": "minimaxm2.5" - }, - { - "type": "attention_mha", - "args": { - "batch_size": 32, - "seq_len_q": 1024, - "seq_len_kv": 1024, - "num_heads": 6, - "num_heads_kv": 1, - "head_dim": 128, - "dtype_q": "fp8", - "dtype_k": "fp8", - "dtype_v": "fp8", - "dtype_o": "bf16", - "causal": true - }, - "name": "minimaxm2.5" - }, - { - "type": "attention_mha", - "args": { - "batch_size": 32, - "seq_len_q": 1, - "seq_len_kv": 1536, - "num_heads": 6, - "num_heads_kv": 1, - "head_dim": 128, - "dtype_q": "fp8", - "dtype_k": "fp8", - "dtype_v": "fp8", - "dtype_o": "bf16", - "causal": true - }, - "name": "minimaxm2.5" - }, - { - "type": "attention_mha", - "args": { - "batch_size": 64, - "seq_len_q": 1024, - "seq_len_kv": 1024, - "num_heads": 6, - "num_heads_kv": 1, - "head_dim": 128, - "dtype_q": "fp8", - "dtype_k": "fp8", - "dtype_v": "fp8", - "dtype_o": "bf16", - "causal": true - }, - "name": "minimaxm2.5" - }, - { - "type": "attention_mha", - "args": { - "batch_size": 64, - "seq_len_q": 1, - "seq_len_kv": 1536, - "num_heads": 6, - "num_heads_kv": 1, - "head_dim": 128, - "dtype_q": "fp8", - "dtype_k": "fp8", - "dtype_v": "fp8", - "dtype_o": "bf16", - "causal": true - }, - "name": "minimaxm2.5" - }, - { - "type": "attention_mha", - "args": { - "batch_size": 128, - "seq_len_q": 1024, - "seq_len_kv": 1024, - "num_heads": 6, - "num_heads_kv": 1, - "head_dim": 128, - "dtype_q": "fp8", - "dtype_k": "fp8", - "dtype_v": "fp8", - "dtype_o": "bf16", - "causal": true - }, - "name": "minimaxm2.5" - }, - { - "type": "attention_mha", - "args": { - "batch_size": 128, - "seq_len_q": 1, - "seq_len_kv": 1536, - "num_heads": 6, - "num_heads_kv": 1, - "head_dim": 128, - "dtype_q": "fp8", - "dtype_k": "fp8", - "dtype_v": "fp8", - "dtype_o": "bf16", - "causal": true - }, - "name": "minimaxm2.5" - }, - { - "type": "attention_mha", - "args": { - "batch_size": 4, - "seq_len_q": 8192, - "seq_len_kv": 8192, - "num_heads": 6, - "num_heads_kv": 1, - "head_dim": 128, - "dtype_q": "fp8", - "dtype_k": "fp8", - "dtype_v": "fp8", - "dtype_o": "bf16", - "causal": true - }, - "name": "minimaxm2.5" - }, - { - "type": "attention_mha", - "args": { - "batch_size": 4, - "seq_len_q": 1, - "seq_len_kv": 8704, - "num_heads": 6, - "num_heads_kv": 1, - "head_dim": 128, - "dtype_q": "fp8", - "dtype_k": "fp8", - "dtype_v": "fp8", - "dtype_o": "bf16", - "causal": true - }, - "name": "minimaxm2.5" - }, - { - "type": "attention_mha", - "args": { - "batch_size": 8, - "seq_len_q": 8192, - "seq_len_kv": 8192, - "num_heads": 6, - "num_heads_kv": 1, - "head_dim": 128, - "dtype_q": "fp8", - "dtype_k": "fp8", - "dtype_v": "fp8", - "dtype_o": "bf16", - "causal": true - }, - "name": "minimaxm2.5" - }, - { - "type": "attention_mha", - "args": { - "batch_size": 8, - "seq_len_q": 1, - "seq_len_kv": 8704, - "num_heads": 6, - "num_heads_kv": 1, - "head_dim": 128, - "dtype_q": "fp8", - "dtype_k": "fp8", - "dtype_v": "fp8", - "dtype_o": "bf16", - "causal": true - }, - "name": "minimaxm2.5" - }, - { - "type": "attention_mha", - "args": { - "batch_size": 16, - "seq_len_q": 8192, - "seq_len_kv": 8192, - "num_heads": 6, - "num_heads_kv": 1, - "head_dim": 128, - "dtype_q": "fp8", - "dtype_k": "fp8", - "dtype_v": "fp8", - "dtype_o": "bf16", - "causal": true - }, - "name": "minimaxm2.5" - }, - { - "type": "attention_mha", - "args": { - "batch_size": 16, - "seq_len_q": 1, - "seq_len_kv": 8704, - "num_heads": 6, - "num_heads_kv": 1, - "head_dim": 128, - "dtype_q": "fp8", - "dtype_k": "fp8", - "dtype_v": "fp8", - "dtype_o": "bf16", - "causal": true - }, - "name": "minimaxm2.5" - }, - { - "type": "attention_mha", - "args": { - "batch_size": 32, - "seq_len_q": 8192, - "seq_len_kv": 8192, - "num_heads": 6, - "num_heads_kv": 1, - "head_dim": 128, - "dtype_q": "fp8", - "dtype_k": "fp8", - "dtype_v": "fp8", - "dtype_o": "bf16", - "causal": true - }, - "name": "minimaxm2.5" - }, - { - "type": "attention_mha", - "args": { - "batch_size": 32, - "seq_len_q": 1, - "seq_len_kv": 8704, - "num_heads": 6, - "num_heads_kv": 1, - "head_dim": 128, - "dtype_q": "fp8", - "dtype_k": "fp8", - "dtype_v": "fp8", - "dtype_o": "bf16", - "causal": true - }, - "name": "minimaxm2.5" - }, - { - "type": "attention_mha", - "args": { - "batch_size": 64, - "seq_len_q": 8192, - "seq_len_kv": 8192, - "num_heads": 6, - "num_heads_kv": 1, - "head_dim": 128, - "dtype_q": "fp8", - "dtype_k": "fp8", - "dtype_v": "fp8", - "dtype_o": "bf16", - "causal": true - }, - "name": "minimaxm2.5" - }, - { - "type": "attention_mha", - "args": { - "batch_size": 64, - "seq_len_q": 1, - "seq_len_kv": 8704, - "num_heads": 6, - "num_heads_kv": 1, - "head_dim": 128, - "dtype_q": "fp8", - "dtype_k": "fp8", - "dtype_v": "fp8", - "dtype_o": "bf16", - "causal": true - }, - "name": "minimaxm2.5" - }, - { - "type": "attention_mha", - "args": { - "batch_size": 128, - "seq_len_q": 8192, - "seq_len_kv": 8192, - "num_heads": 6, - "num_heads_kv": 1, - "head_dim": 128, - "dtype_q": "fp8", - "dtype_k": "fp8", - "dtype_v": "fp8", - "dtype_o": "bf16", - "causal": true - }, - "name": "minimaxm2.5" - }, - { - "type": "attention_mha", - "args": { - "batch_size": 128, - "seq_len_q": 1, - "seq_len_kv": 8704, - "num_heads": 6, - "num_heads_kv": 1, - "head_dim": 128, - "dtype_q": "fp8", - "dtype_k": "fp8", - "dtype_v": "fp8", - "dtype_o": "bf16", - "causal": true - }, - "name": "minimaxm2.5" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 8, - "seq_len_q": 2, - "seq_len_kv": 1536, - "num_heads": 16, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 12, - "seq_len_q": 1024, - "seq_len_kv": 1024, - "num_heads": 16, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 12, - "seq_len_q": 2, - "seq_len_kv": 1536, - "num_heads": 16, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 24, - "seq_len_q": 1024, - "seq_len_kv": 1024, - "num_heads": 16, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 24, - "seq_len_q": 2, - "seq_len_kv": 1536, - "num_heads": 16, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 48, - "seq_len_q": 1024, - "seq_len_kv": 1024, - "num_heads": 16, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 48, - "seq_len_q": 2, - "seq_len_kv": 1536, - "num_heads": 16, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 1075, - "seq_len_q": 1024, - "seq_len_kv": 1024, - "num_heads": 128, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 1075, - "seq_len_q": 2, - "seq_len_kv": 1536, - "num_heads": 128, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 268, - "seq_len_q": 1024, - "seq_len_kv": 1024, - "num_heads": 128, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 268, - "seq_len_q": 2, - "seq_len_kv": 1536, - "num_heads": 128, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 563, - "seq_len_q": 1024, - "seq_len_kv": 1024, - "num_heads": 128, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 563, - "seq_len_q": 2, - "seq_len_kv": 1536, - "num_heads": 128, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 70, - "seq_len_q": 1024, - "seq_len_kv": 1024, - "num_heads": 128, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 70, - "seq_len_q": 2, - "seq_len_kv": 1536, - "num_heads": 128, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 4032, - "seq_len_q": 1024, - "seq_len_kv": 1024, - "num_heads": 128, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 4032, - "seq_len_q": 2, - "seq_len_kv": 1536, - "num_heads": 128, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 1075, - "seq_len_q": 1, - "seq_len_kv": 1536, - "num_heads": 128, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 537, - "seq_len_q": 1024, - "seq_len_kv": 1024, - "num_heads": 128, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 537, - "seq_len_q": 1, - "seq_len_kv": 1536, - "num_heads": 128, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 166, - "seq_len_q": 1024, - "seq_len_kv": 1024, - "num_heads": 128, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 166, - "seq_len_q": 1, - "seq_len_kv": 1536, - "num_heads": 128, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 20, - "seq_len_q": 1024, - "seq_len_kv": 1024, - "num_heads": 128, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 20, - "seq_len_q": 1, - "seq_len_kv": 1536, - "num_heads": 128, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 1536, - "seq_len_q": 1024, - "seq_len_kv": 1024, - "num_heads": 128, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 1536, - "seq_len_q": 1, - "seq_len_kv": 1536, - "num_heads": 128, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 24, - "seq_len_q": 1024, - "seq_len_kv": 1024, - "num_heads": 128, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 24, - "seq_len_q": 1, - "seq_len_kv": 1536, - "num_heads": 128, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 48, - "seq_len_q": 1024, - "seq_len_kv": 1024, - "num_heads": 128, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 48, - "seq_len_q": 1, - "seq_len_kv": 1536, - "num_heads": 128, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 12, - "seq_len_q": 1, - "seq_len_kv": 1536, - "num_heads": 16, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 24, - "seq_len_q": 1, - "seq_len_kv": 1536, - "num_heads": 16, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 48, - "seq_len_q": 1, - "seq_len_kv": 1536, - "num_heads": 16, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 96, - "seq_len_q": 1024, - "seq_len_kv": 1024, - "num_heads": 16, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 96, - "seq_len_q": 1, - "seq_len_kv": 1536, - "num_heads": 16, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 192, - "seq_len_q": 1024, - "seq_len_kv": 1024, - "num_heads": 16, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 192, - "seq_len_q": 1, - "seq_len_kv": 1536, - "num_heads": 16, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 5, - "seq_len_q": 1024, - "seq_len_kv": 1024, - "num_heads": 16, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 5, - "seq_len_q": 1, - "seq_len_kv": 1536, - "num_heads": 16, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 268, - "seq_len_q": 1, - "seq_len_kv": 1536, - "num_heads": 128, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 563, - "seq_len_q": 1, - "seq_len_kv": 1536, - "num_heads": 128, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 70, - "seq_len_q": 1, - "seq_len_kv": 1536, - "num_heads": 128, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 6, - "seq_len_q": 2, - "seq_len_kv": 8704, - "num_heads": 128, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 12, - "seq_len_q": 2, - "seq_len_kv": 8704, - "num_heads": 128, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 12, - "seq_len_q": 2, - "seq_len_kv": 8704, - "num_heads": 16, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 24, - "seq_len_q": 8192, - "seq_len_kv": 8192, - "num_heads": 16, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 24, - "seq_len_q": 2, - "seq_len_kv": 8704, - "num_heads": 16, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 48, - "seq_len_q": 8192, - "seq_len_kv": 8192, - "num_heads": 16, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 48, - "seq_len_q": 2, - "seq_len_kv": 8704, - "num_heads": 16, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 45, - "seq_len_q": 8192, - "seq_len_kv": 8192, - "num_heads": 128, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 45, - "seq_len_q": 2, - "seq_len_kv": 8704, - "num_heads": 128, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 5, - "seq_len_q": 8192, - "seq_len_kv": 8192, - "num_heads": 128, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 5, - "seq_len_q": 2, - "seq_len_kv": 8704, - "num_heads": 128, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 307, - "seq_len_q": 8192, - "seq_len_kv": 8192, - "num_heads": 128, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 307, - "seq_len_q": 2, - "seq_len_kv": 8704, - "num_heads": 128, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 76, - "seq_len_q": 8192, - "seq_len_kv": 8192, - "num_heads": 128, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 76, - "seq_len_q": 2, - "seq_len_kv": 8704, - "num_heads": 128, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 166, - "seq_len_q": 8192, - "seq_len_kv": 8192, - "num_heads": 128, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 166, - "seq_len_q": 2, - "seq_len_kv": 8704, - "num_heads": 128, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 20, - "seq_len_q": 8192, - "seq_len_kv": 8192, - "num_heads": 128, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 20, - "seq_len_q": 2, - "seq_len_kv": 8704, - "num_heads": 128, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 1075, - "seq_len_q": 8192, - "seq_len_kv": 8192, - "num_heads": 128, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 1075, - "seq_len_q": 2, - "seq_len_kv": 8704, - "num_heads": 128, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 268, - "seq_len_q": 8192, - "seq_len_kv": 8192, - "num_heads": 128, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 268, - "seq_len_q": 2, - "seq_len_kv": 8704, - "num_heads": 128, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 11, - "seq_len_q": 1, - "seq_len_kv": 8704, - "num_heads": 128, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 19, - "seq_len_q": 8192, - "seq_len_kv": 8192, - "num_heads": 128, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 19, - "seq_len_q": 1, - "seq_len_kv": 8704, - "num_heads": 128, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 44, - "seq_len_q": 8192, - "seq_len_kv": 8192, - "num_heads": 16, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 44, - "seq_len_q": 1, - "seq_len_kv": 8704, - "num_heads": 16, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 76, - "seq_len_q": 8192, - "seq_len_kv": 8192, - "num_heads": 16, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 76, - "seq_len_q": 1, - "seq_len_kv": 8704, - "num_heads": 16, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 5, - "seq_len_q": 8192, - "seq_len_kv": 8192, - "num_heads": 16, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 5, - "seq_len_q": 1, - "seq_len_kv": 8704, - "num_heads": 16, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 83, - "seq_len_q": 8192, - "seq_len_kv": 8192, - "num_heads": 128, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 83, - "seq_len_q": 1, - "seq_len_kv": 8704, - "num_heads": 128, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 10, - "seq_len_q": 8192, - "seq_len_kv": 8192, - "num_heads": 128, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 10, - "seq_len_q": 1, - "seq_len_kv": 8704, - "num_heads": 128, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 307, - "seq_len_q": 1, - "seq_len_kv": 8704, - "num_heads": 128, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 38, - "seq_len_q": 8192, - "seq_len_kv": 8192, - "num_heads": 128, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 38, - "seq_len_q": 1, - "seq_len_kv": 8704, - "num_heads": 128, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 563, - "seq_len_q": 8192, - "seq_len_kv": 8192, - "num_heads": 128, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 563, - "seq_len_q": 1, - "seq_len_kv": 8704, - "num_heads": 128, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 140, - "seq_len_q": 8192, - "seq_len_kv": 8192, - "num_heads": 128, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 140, - "seq_len_q": 1, - "seq_len_kv": 8704, - "num_heads": 128, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 537, - "seq_len_q": 1024, - "seq_len_kv": 1024, - "num_heads": 128, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "fp8", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 537, - "seq_len_q": 2, - "seq_len_kv": 1536, - "num_heads": 128, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "fp8", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 268, - "seq_len_q": 1024, - "seq_len_kv": 1024, - "num_heads": 128, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "fp8", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 268, - "seq_len_q": 2, - "seq_len_kv": 1536, - "num_heads": 128, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "fp8", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 153, - "seq_len_q": 1024, - "seq_len_kv": 1024, - "num_heads": 128, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "fp8", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 153, - "seq_len_q": 2, - "seq_len_kv": 1536, - "num_heads": 128, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "fp8", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 19, - "seq_len_q": 1024, - "seq_len_kv": 1024, - "num_heads": 128, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "fp8", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 19, - "seq_len_q": 2, - "seq_len_kv": 1536, - "num_heads": 128, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "fp8", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 36, - "seq_len_q": 1024, - "seq_len_kv": 1024, - "num_heads": 16, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "fp8", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 36, - "seq_len_q": 2, - "seq_len_kv": 1536, - "num_heads": 16, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "fp8", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 18, - "seq_len_q": 1024, - "seq_len_kv": 1024, - "num_heads": 16, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "fp8", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 18, - "seq_len_q": 2, - "seq_len_kv": 1536, - "num_heads": 16, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "fp8", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 9, - "seq_len_q": 1024, - "seq_len_kv": 1024, - "num_heads": 16, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "fp8", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 9, - "seq_len_q": 2, - "seq_len_kv": 1536, - "num_heads": 16, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "fp8", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 537, - "seq_len_q": 1, - "seq_len_kv": 1536, - "num_heads": 128, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "fp8", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 268, - "seq_len_q": 1, - "seq_len_kv": 1536, - "num_heads": 128, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "fp8", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 134, - "seq_len_q": 1024, - "seq_len_kv": 1024, - "num_heads": 128, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "fp8", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 134, - "seq_len_q": 1, - "seq_len_kv": 1536, - "num_heads": 128, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "fp8", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 140, - "seq_len_q": 1024, - "seq_len_kv": 1024, - "num_heads": 128, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "fp8", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 140, - "seq_len_q": 1, - "seq_len_kv": 1536, - "num_heads": 128, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "fp8", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 35, - "seq_len_q": 1024, - "seq_len_kv": 1024, - "num_heads": 128, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "fp8", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 35, - "seq_len_q": 1, - "seq_len_kv": 1536, - "num_heads": 128, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "fp8", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 27, - "seq_len_q": 1024, - "seq_len_kv": 1024, - "num_heads": 16, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "fp8", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 27, - "seq_len_q": 1, - "seq_len_kv": 1536, - "num_heads": 16, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "fp8", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 3, - "seq_len_q": 1024, - "seq_len_kv": 1024, - "num_heads": 16, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "fp8", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 3, - "seq_len_q": 1, - "seq_len_kv": 1536, - "num_heads": 16, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "fp8", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 83, - "seq_len_q": 8192, - "seq_len_kv": 8192, - "num_heads": 128, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "fp8", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 83, - "seq_len_q": 2, - "seq_len_kv": 8704, - "num_heads": 128, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "fp8", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 41, - "seq_len_q": 8192, - "seq_len_kv": 8192, - "num_heads": 128, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "fp8", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 41, - "seq_len_q": 2, - "seq_len_kv": 8704, - "num_heads": 128, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "fp8", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 20, - "seq_len_q": 8192, - "seq_len_kv": 8192, - "num_heads": 128, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "fp8", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 20, - "seq_len_q": 2, - "seq_len_kv": 8704, - "num_heads": 128, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "fp8", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 11, - "seq_len_q": 8192, - "seq_len_kv": 8192, - "num_heads": 128, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "fp8", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 11, - "seq_len_q": 2, - "seq_len_kv": 8704, - "num_heads": 128, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "fp8", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 15, - "seq_len_q": 8192, - "seq_len_kv": 8192, - "num_heads": 16, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "fp8", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 15, - "seq_len_q": 2, - "seq_len_kv": 8704, - "num_heads": 16, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "fp8", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 6, - "seq_len_q": 8192, - "seq_len_kv": 8192, - "num_heads": 16, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "fp8", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 6, - "seq_len_q": 2, - "seq_len_kv": 8704, - "num_heads": 16, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "fp8", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 153, - "seq_len_q": 8192, - "seq_len_kv": 8192, - "num_heads": 128, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "fp8", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 153, - "seq_len_q": 1, - "seq_len_kv": 8704, - "num_heads": 128, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "fp8", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 76, - "seq_len_q": 8192, - "seq_len_kv": 8192, - "num_heads": 128, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "fp8", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 76, - "seq_len_q": 1, - "seq_len_kv": 8704, - "num_heads": 128, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "fp8", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 83, - "seq_len_q": 1, - "seq_len_kv": 8704, - "num_heads": 128, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "fp8", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 20, - "seq_len_q": 1, - "seq_len_kv": 8704, - "num_heads": 128, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "fp8", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 38, - "seq_len_q": 8192, - "seq_len_kv": 8192, - "num_heads": 128, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "fp8", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 38, - "seq_len_q": 1, - "seq_len_kv": 8704, - "num_heads": 128, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "fp8", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 41, - "seq_len_q": 1, - "seq_len_kv": 8704, - "num_heads": 128, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "fp8", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 10, - "seq_len_q": 1, - "seq_len_kv": 8704, - "num_heads": 128, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "fp8", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 7, - "seq_len_q": 8192, - "seq_len_kv": 8192, - "num_heads": 128, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "fp8", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 7, - "seq_len_q": 1, - "seq_len_kv": 8704, - "num_heads": 128, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "fp8", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 63, - "seq_len_q": 8192, - "seq_len_kv": 8192, - "num_heads": 16, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "fp8", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 63, - "seq_len_q": 1, - "seq_len_kv": 8704, - "num_heads": 16, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "fp8", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 2, - "seq_len_q": 1, - "seq_len_kv": 8704, - "num_heads": 128, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "fp8", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 18, - "seq_len_q": 8192, - "seq_len_kv": 8192, - "num_heads": 16, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "fp8", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 18, - "seq_len_q": 1, - "seq_len_kv": 8704, - "num_heads": 16, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "fp8", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 6, - "seq_len_q": 1, - "seq_len_kv": 8704, - "num_heads": 16, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "fp8", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 4, - "seq_len_q": 1024, - "seq_len_kv": 1024, - "num_heads": 32, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "fp8", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 4, - "seq_len_q": 1, - "seq_len_kv": 1536, - "num_heads": 32, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "fp8", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 128, - "seq_len_q": 1, - "seq_len_kv": 1536, - "num_heads": 128, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "fp8", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 256, - "seq_len_q": 1024, - "seq_len_kv": 1024, - "num_heads": 128, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "fp8", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 256, - "seq_len_q": 1, - "seq_len_kv": 1536, - "num_heads": 128, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "fp8", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 21, - "seq_len_q": 1024, - "seq_len_kv": 1024, - "num_heads": 128, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "fp8", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 21, - "seq_len_q": 1, - "seq_len_kv": 1536, - "num_heads": 128, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "fp8", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 42, - "seq_len_q": 1024, - "seq_len_kv": 1024, - "num_heads": 128, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "fp8", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 42, - "seq_len_q": 1, - "seq_len_kv": 1536, - "num_heads": 128, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "fp8", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 85, - "seq_len_q": 1024, - "seq_len_kv": 1024, - "num_heads": 128, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "fp8", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 85, - "seq_len_q": 1, - "seq_len_kv": 1536, - "num_heads": 128, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "fp8", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 192, - "seq_len_q": 1024, - "seq_len_kv": 1024, - "num_heads": 128, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "fp8", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 192, - "seq_len_q": 1, - "seq_len_kv": 1536, - "num_heads": 128, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "fp8", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 256, - "seq_len_q": 8192, - "seq_len_kv": 8192, - "num_heads": 128, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "fp8", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 256, - "seq_len_q": 1, - "seq_len_kv": 8704, - "num_heads": 128, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "fp8", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 192, - "seq_len_q": 8192, - "seq_len_kv": 8192, - "num_heads": 128, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "fp8", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 192, - "seq_len_q": 1, - "seq_len_kv": 8704, - "num_heads": 128, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "fp8", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 512, - "seq_len_q": 8192, - "seq_len_kv": 8192, - "num_heads": 128, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "fp8", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 512, - "seq_len_q": 1, - "seq_len_kv": 8704, - "num_heads": 128, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "fp8", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 85, - "seq_len_q": 8192, - "seq_len_kv": 8192, - "num_heads": 128, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "fp8", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 85, - "seq_len_q": 1, - "seq_len_kv": 8704, - "num_heads": 128, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "fp8", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 170, - "seq_len_q": 8192, - "seq_len_kv": 8192, - "num_heads": 128, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "fp8", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 170, - "seq_len_q": 1, - "seq_len_kv": 8704, - "num_heads": 128, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "fp8", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 32, - "seq_len_q": 1024, - "seq_len_kv": 1024, - "num_heads": 32, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "fp8", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 32, - "seq_len_q": 1, - "seq_len_kv": 1536, - "num_heads": 32, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "fp8", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 896, - "seq_len_q": 1024, - "seq_len_kv": 1024, - "num_heads": 128, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "fp8", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 896, - "seq_len_q": 1, - "seq_len_kv": 1536, - "num_heads": 128, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "fp8", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 960, - "seq_len_q": 1024, - "seq_len_kv": 1024, - "num_heads": 128, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "fp8", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 960, - "seq_len_q": 1, - "seq_len_kv": 1536, - "num_heads": 128, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "fp8", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 128, - "seq_len_q": 1024, - "seq_len_kv": 1024, - "num_heads": 128, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 128, - "seq_len_q": 1, - "seq_len_kv": 1536, - "num_heads": 128, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 2048, - "seq_len_q": 1024, - "seq_len_kv": 1024, - "num_heads": 128, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 2048, - "seq_len_q": 1, - "seq_len_kv": 1536, - "num_heads": 128, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 256, - "seq_len_q": 1024, - "seq_len_kv": 1024, - "num_heads": 128, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 256, - "seq_len_q": 1, - "seq_len_kv": 1536, - "num_heads": 128, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 42, - "seq_len_q": 1024, - "seq_len_kv": 1024, - "num_heads": 128, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 42, - "seq_len_q": 1, - "seq_len_kv": 1536, - "num_heads": 128, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 85, - "seq_len_q": 1024, - "seq_len_kv": 1024, - "num_heads": 128, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 85, - "seq_len_q": 1, - "seq_len_kv": 1536, - "num_heads": 128, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 512, - "seq_len_q": 8192, - "seq_len_kv": 8192, - "num_heads": 32, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 512, - "seq_len_q": 1, - "seq_len_kv": 8704, - "num_heads": 32, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 2048, - "seq_len_q": 8192, - "seq_len_kv": 8192, - "num_heads": 32, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 2048, - "seq_len_q": 1, - "seq_len_kv": 8704, - "num_heads": 32, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 4096, - "seq_len_q": 8192, - "seq_len_kv": 8192, - "num_heads": 32, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 4096, - "seq_len_q": 1, - "seq_len_kv": 8704, - "num_heads": 32, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 42, - "seq_len_q": 8192, - "seq_len_kv": 8192, - "num_heads": 128, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 42, - "seq_len_q": 1, - "seq_len_kv": 8704, - "num_heads": 128, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 85, - "seq_len_q": 8192, - "seq_len_kv": 8192, - "num_heads": 128, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 85, - "seq_len_q": 1, - "seq_len_kv": 8704, - "num_heads": 128, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 1613, - "seq_len_q": 1024, - "seq_len_kv": 1024, - "num_heads": 128, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 1613, - "seq_len_q": 2, - "seq_len_kv": 1536, - "num_heads": 128, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 806, - "seq_len_q": 1024, - "seq_len_kv": 1024, - "num_heads": 128, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 806, - "seq_len_q": 2, - "seq_len_kv": 1536, - "num_heads": 128, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 166, - "seq_len_q": 2, - "seq_len_kv": 1536, - "num_heads": 128, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 5, - "seq_len_q": 2, - "seq_len_kv": 1536, - "num_heads": 16, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 24, - "seq_len_q": 2, - "seq_len_kv": 1536, - "num_heads": 128, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 1126, - "seq_len_q": 1024, - "seq_len_kv": 1024, - "num_heads": 128, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 1126, - "seq_len_q": 2, - "seq_len_kv": 1536, - "num_heads": 128, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 140, - "seq_len_q": 1024, - "seq_len_kv": 1024, - "num_heads": 128, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 140, - "seq_len_q": 2, - "seq_len_kv": 1536, - "num_heads": 128, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 614, - "seq_len_q": 1024, - "seq_len_kv": 1024, - "num_heads": 128, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 614, - "seq_len_q": 2, - "seq_len_kv": 1536, - "num_heads": 128, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 38, - "seq_len_q": 1024, - "seq_len_kv": 1024, - "num_heads": 128, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 38, - "seq_len_q": 2, - "seq_len_kv": 1536, - "num_heads": 128, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 96, - "seq_len_q": 1024, - "seq_len_kv": 1024, - "num_heads": 128, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 96, - "seq_len_q": 1, - "seq_len_kv": 1536, - "num_heads": 128, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 614, - "seq_len_q": 1, - "seq_len_kv": 1536, - "num_heads": 128, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 38, - "seq_len_q": 1, - "seq_len_kv": 1536, - "num_heads": 128, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 2150, - "seq_len_q": 1024, - "seq_len_kv": 1024, - "num_heads": 128, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 2150, - "seq_len_q": 1, - "seq_len_kv": 1536, - "num_heads": 128, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 1126, - "seq_len_q": 1, - "seq_len_kv": 1536, - "num_heads": 128, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 33, - "seq_len_q": 8192, - "seq_len_kv": 8192, - "num_heads": 16, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 33, - "seq_len_q": 2, - "seq_len_kv": 8704, - "num_heads": 16, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 5, - "seq_len_q": 2, - "seq_len_kv": 8704, - "num_heads": 16, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 154, - "seq_len_q": 8192, - "seq_len_kv": 8192, - "num_heads": 128, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 154, - "seq_len_q": 2, - "seq_len_kv": 8704, - "num_heads": 128, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 9, - "seq_len_q": 8192, - "seq_len_kv": 8192, - "num_heads": 128, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 9, - "seq_len_q": 2, - "seq_len_kv": 8704, - "num_heads": 128, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 1126, - "seq_len_q": 8192, - "seq_len_kv": 8192, - "num_heads": 128, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 1126, - "seq_len_q": 2, - "seq_len_kv": 8704, - "num_heads": 128, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 281, - "seq_len_q": 8192, - "seq_len_kv": 8192, - "num_heads": 128, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 281, - "seq_len_q": 2, - "seq_len_kv": 8704, - "num_heads": 128, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 333, - "seq_len_q": 8192, - "seq_len_kv": 8192, - "num_heads": 128, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 333, - "seq_len_q": 2, - "seq_len_kv": 8704, - "num_heads": 128, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 41, - "seq_len_q": 8192, - "seq_len_kv": 8192, - "num_heads": 128, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 41, - "seq_len_q": 2, - "seq_len_kv": 8704, - "num_heads": 128, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 563, - "seq_len_q": 2, - "seq_len_kv": 8704, - "num_heads": 128, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 70, - "seq_len_q": 8192, - "seq_len_kv": 8192, - "num_heads": 128, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 70, - "seq_len_q": 2, - "seq_len_kv": 8704, - "num_heads": 128, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 36, - "seq_len_q": 8192, - "seq_len_kv": 8192, - "num_heads": 128, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 36, - "seq_len_q": 1, - "seq_len_kv": 8704, - "num_heads": 128, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 72, - "seq_len_q": 8192, - "seq_len_kv": 8192, - "num_heads": 16, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 72, - "seq_len_q": 1, - "seq_len_kv": 8704, - "num_heads": 16, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 7, - "seq_len_q": 1, - "seq_len_kv": 8704, - "num_heads": 128, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 15, - "seq_len_q": 1, - "seq_len_kv": 8704, - "num_heads": 128, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 5, - "seq_len_q": 8192, - "seq_len_kv": 8192, - "num_heads": 32, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 5, - "seq_len_q": 1, - "seq_len_kv": 8704, - "num_heads": 32, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 15, - "seq_len_q": 8192, - "seq_len_kv": 8192, - "num_heads": 32, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 15, - "seq_len_q": 1, - "seq_len_kv": 8704, - "num_heads": 32, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 30, - "seq_len_q": 8192, - "seq_len_kv": 8192, - "num_heads": 32, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 30, - "seq_len_q": 1, - "seq_len_kv": 8704, - "num_heads": 32, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 333, - "seq_len_q": 1, - "seq_len_kv": 8704, - "num_heads": 128, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 20, - "seq_len_q": 1, - "seq_len_kv": 8704, - "num_heads": 128, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 614, - "seq_len_q": 8192, - "seq_len_kv": 8192, - "num_heads": 128, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 614, - "seq_len_q": 1, - "seq_len_kv": 8704, - "num_heads": 128, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 76, - "seq_len_q": 1, - "seq_len_kv": 8704, - "num_heads": 128, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 1614, - "seq_len_q": 8192, - "seq_len_kv": 8192, - "num_heads": 128, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 1614, - "seq_len_q": 1, - "seq_len_kv": 8704, - "num_heads": 128, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 807, - "seq_len_q": 8192, - "seq_len_kv": 8192, - "num_heads": 128, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 807, - "seq_len_q": 1, - "seq_len_kv": 8704, - "num_heads": 128, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 1126, - "seq_len_q": 1, - "seq_len_kv": 8704, - "num_heads": 128, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 10, - "seq_len_q": 1, - "seq_len_kv": 1536, - "num_heads": 128, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 6, - "seq_len_q": 1024, - "seq_len_kv": 1024, - "num_heads": 128, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "fp8", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 6, - "seq_len_q": 2, - "seq_len_kv": 1536, - "num_heads": 128, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "fp8", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 24, - "seq_len_q": 1024, - "seq_len_kv": 1024, - "num_heads": 16, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "fp8", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 24, - "seq_len_q": 2, - "seq_len_kv": 1536, - "num_heads": 16, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "fp8", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 45, - "seq_len_q": 1024, - "seq_len_kv": 1024, - "num_heads": 128, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "fp8", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 45, - "seq_len_q": 2, - "seq_len_kv": 1536, - "num_heads": 128, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "fp8", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 5, - "seq_len_q": 1024, - "seq_len_kv": 1024, - "num_heads": 128, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "fp8", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 5, - "seq_len_q": 2, - "seq_len_kv": 1536, - "num_heads": 128, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "fp8", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 141, - "seq_len_q": 1024, - "seq_len_kv": 1024, - "num_heads": 128, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "fp8", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 141, - "seq_len_q": 2, - "seq_len_kv": 1536, - "num_heads": 128, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "fp8", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 17, - "seq_len_q": 1024, - "seq_len_kv": 1024, - "num_heads": 128, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "fp8", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 17, - "seq_len_q": 2, - "seq_len_kv": 1536, - "num_heads": 128, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "fp8", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 166, - "seq_len_q": 1024, - "seq_len_kv": 1024, - "num_heads": 128, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "fp8", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 166, - "seq_len_q": 2, - "seq_len_kv": 1536, - "num_heads": 128, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "fp8", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 41, - "seq_len_q": 1024, - "seq_len_kv": 1024, - "num_heads": 128, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "fp8", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 41, - "seq_len_q": 2, - "seq_len_kv": 1536, - "num_heads": 128, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "fp8", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 563, - "seq_len_q": 1024, - "seq_len_kv": 1024, - "num_heads": 128, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "fp8", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 563, - "seq_len_q": 2, - "seq_len_kv": 1536, - "num_heads": 128, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "fp8", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 140, - "seq_len_q": 2, - "seq_len_kv": 1536, - "num_heads": 128, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "fp8", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 2048, - "seq_len_q": 2, - "seq_len_kv": 1536, - "num_heads": 128, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "fp8", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 1024, - "seq_len_q": 1024, - "seq_len_kv": 1024, - "num_heads": 128, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "fp8", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 1024, - "seq_len_q": 2, - "seq_len_kv": 1536, - "num_heads": 128, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "fp8", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 6, - "seq_len_q": 1, - "seq_len_kv": 1536, - "num_heads": 128, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "fp8", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 24, - "seq_len_q": 1, - "seq_len_kv": 1536, - "num_heads": 16, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "fp8", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 84, - "seq_len_q": 1024, - "seq_len_kv": 1024, - "num_heads": 16, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "fp8", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 84, - "seq_len_q": 1, - "seq_len_kv": 1536, - "num_heads": 16, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "fp8", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 38, - "seq_len_q": 1, - "seq_len_kv": 1536, - "num_heads": 128, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "fp8", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 563, - "seq_len_q": 1, - "seq_len_kv": 1536, - "num_heads": 128, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "fp8", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 2150, - "seq_len_q": 1024, - "seq_len_kv": 1024, - "num_heads": 128, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "fp8", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 2150, - "seq_len_q": 1, - "seq_len_kv": 1536, - "num_heads": 128, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "fp8", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 1075, - "seq_len_q": 1024, - "seq_len_kv": 1024, - "num_heads": 128, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "fp8", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 1075, - "seq_len_q": 1, - "seq_len_kv": 1536, - "num_heads": 128, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "fp8", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 3072, - "seq_len_q": 1024, - "seq_len_kv": 1024, - "num_heads": 128, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "fp8", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 3072, - "seq_len_q": 1, - "seq_len_kv": 1536, - "num_heads": 128, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "fp8", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 1536, - "seq_len_q": 1024, - "seq_len_kv": 1024, - "num_heads": 128, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "fp8", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 1536, - "seq_len_q": 1, - "seq_len_kv": 1536, - "num_heads": 128, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "fp8", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 24, - "seq_len_q": 8192, - "seq_len_kv": 8192, - "num_heads": 16, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "fp8", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 24, - "seq_len_q": 2, - "seq_len_kv": 8704, - "num_heads": 16, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "fp8", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 166, - "seq_len_q": 8192, - "seq_len_kv": 8192, - "num_heads": 128, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "fp8", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 166, - "seq_len_q": 2, - "seq_len_kv": 8704, - "num_heads": 128, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "fp8", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 307, - "seq_len_q": 8192, - "seq_len_kv": 8192, - "num_heads": 128, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "fp8", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 307, - "seq_len_q": 2, - "seq_len_kv": 8704, - "num_heads": 128, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "fp8", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 76, - "seq_len_q": 2, - "seq_len_kv": 8704, - "num_heads": 128, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "fp8", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 153, - "seq_len_q": 2, - "seq_len_kv": 8704, - "num_heads": 128, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "fp8", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 6, - "seq_len_q": 1, - "seq_len_kv": 8704, - "num_heads": 128, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "fp8", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 24, - "seq_len_q": 1, - "seq_len_kv": 8704, - "num_heads": 16, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "fp8", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 36, - "seq_len_q": 8192, - "seq_len_kv": 8192, - "num_heads": 16, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "fp8", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 36, - "seq_len_q": 1, - "seq_len_kv": 8704, - "num_heads": 16, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "fp8", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 166, - "seq_len_q": 1, - "seq_len_kv": 8704, - "num_heads": 128, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "fp8", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 307, - "seq_len_q": 1, - "seq_len_kv": 8704, - "num_heads": 128, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "fp8", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 537, - "seq_len_q": 8192, - "seq_len_kv": 8192, - "num_heads": 128, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "fp8", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 537, - "seq_len_q": 1, - "seq_len_kv": 8704, - "num_heads": 128, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "fp8", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mha", - "args": { - "batch_size": 1, - "seq_len_q": 1024, - "seq_len_kv": 1024, - "num_heads": 64, - "num_heads_kv": 8, - "head_dim": 64, - "dtype_q": "nvfp4", - "dtype_k": "fp8", - "dtype_v": "fp8", - "dtype_o": "bf16", - "causal": true, - "sliding_window": 128 - }, - "name": "gptoss" - }, - { - "type": "attention_mha", - "args": { - "batch_size": 1, - "seq_len_q": 1, - "seq_len_kv": 1536, - "num_heads": 64, - "num_heads_kv": 8, - "head_dim": 64, - "dtype_q": "nvfp4", - "dtype_k": "fp8", - "dtype_v": "fp8", - "dtype_o": "bf16", - "causal": true, - "sliding_window": 128 - }, - "name": "gptoss" - }, - { - "type": "attention_mha", - "args": { - "batch_size": 2, - "seq_len_q": 1024, - "seq_len_kv": 1024, - "num_heads": 64, - "num_heads_kv": 8, - "head_dim": 64, - "dtype_q": "nvfp4", - "dtype_k": "fp8", - "dtype_v": "fp8", - "dtype_o": "bf16", - "causal": true, - "sliding_window": 128 - }, - "name": "gptoss" - }, - { - "type": "attention_mha", - "args": { - "batch_size": 2, - "seq_len_q": 1, - "seq_len_kv": 1536, - "num_heads": 64, - "num_heads_kv": 8, - "head_dim": 64, - "dtype_q": "nvfp4", - "dtype_k": "fp8", - "dtype_v": "fp8", - "dtype_o": "bf16", - "causal": true, - "sliding_window": 128 - }, - "name": "gptoss" - }, - { - "type": "attention_mha", - "args": { - "batch_size": 1, - "seq_len_q": 1024, - "seq_len_kv": 1024, - "num_heads": 16, - "num_heads_kv": 2, - "head_dim": 64, - "dtype_q": "nvfp4", - "dtype_k": "fp8", - "dtype_v": "fp8", - "dtype_o": "bf16", - "causal": true, - "sliding_window": 128 - }, - "name": "gptoss" - }, - { - "type": "attention_mha", - "args": { - "batch_size": 1, - "seq_len_q": 1, - "seq_len_kv": 1536, - "num_heads": 16, - "num_heads_kv": 2, - "head_dim": 64, - "dtype_q": "nvfp4", - "dtype_k": "fp8", - "dtype_v": "fp8", - "dtype_o": "bf16", - "causal": true, - "sliding_window": 128 - }, - "name": "gptoss" - }, - { - "type": "attention_mha", - "args": { - "batch_size": 2, - "seq_len_q": 1024, - "seq_len_kv": 1024, - "num_heads": 16, - "num_heads_kv": 2, - "head_dim": 64, - "dtype_q": "nvfp4", - "dtype_k": "fp8", - "dtype_v": "fp8", - "dtype_o": "bf16", - "causal": true, - "sliding_window": 128 - }, - "name": "gptoss" - }, - { - "type": "attention_mha", - "args": { - "batch_size": 2, - "seq_len_q": 1, - "seq_len_kv": 1536, - "num_heads": 16, - "num_heads_kv": 2, - "head_dim": 64, - "dtype_q": "nvfp4", - "dtype_k": "fp8", - "dtype_v": "fp8", - "dtype_o": "bf16", - "causal": true, - "sliding_window": 128 - }, - "name": "gptoss" - }, - { - "type": "attention_mha", - "args": { - "batch_size": 128, - "seq_len_q": 1024, - "seq_len_kv": 1024, - "num_heads": 16, - "num_heads_kv": 2, - "head_dim": 64, - "dtype_q": "nvfp4", - "dtype_k": "fp8", - "dtype_v": "fp8", - "dtype_o": "bf16", - "causal": true, - "sliding_window": 128 - }, - "name": "gptoss" - }, - { - "type": "attention_mha", - "args": { - "batch_size": 128, - "seq_len_q": 1, - "seq_len_kv": 1536, - "num_heads": 16, - "num_heads_kv": 2, - "head_dim": 64, - "dtype_q": "nvfp4", - "dtype_k": "fp8", - "dtype_v": "fp8", - "dtype_o": "bf16", - "causal": true, - "sliding_window": 128 - }, - "name": "gptoss" - }, - { - "type": "attention_mha", - "args": { - "batch_size": 512, - "seq_len_q": 1024, - "seq_len_kv": 1024, - "num_heads": 64, - "num_heads_kv": 8, - "head_dim": 64, - "dtype_q": "nvfp4", - "dtype_k": "fp8", - "dtype_v": "fp8", - "dtype_o": "bf16", - "causal": true, - "sliding_window": 128 - }, - "name": "gptoss" - }, - { - "type": "attention_mha", - "args": { - "batch_size": 512, - "seq_len_q": 1, - "seq_len_kv": 1536, - "num_heads": 64, - "num_heads_kv": 8, - "head_dim": 64, - "dtype_q": "nvfp4", - "dtype_k": "fp8", - "dtype_v": "fp8", - "dtype_o": "bf16", - "causal": true, - "sliding_window": 128 - }, - "name": "gptoss" - }, - { - "type": "attention_mha", - "args": { - "batch_size": 1024, - "seq_len_q": 1024, - "seq_len_kv": 1024, - "num_heads": 64, - "num_heads_kv": 8, - "head_dim": 64, - "dtype_q": "nvfp4", - "dtype_k": "fp8", - "dtype_v": "fp8", - "dtype_o": "bf16", - "causal": true, - "sliding_window": 128 - }, - "name": "gptoss" - }, - { - "type": "attention_mha", - "args": { - "batch_size": 1024, - "seq_len_q": 1, - "seq_len_kv": 1536, - "num_heads": 64, - "num_heads_kv": 8, - "head_dim": 64, - "dtype_q": "nvfp4", - "dtype_k": "fp8", - "dtype_v": "fp8", - "dtype_o": "bf16", - "causal": true, - "sliding_window": 128 - }, - "name": "gptoss" - }, - { - "type": "attention_mha", - "args": { - "batch_size": 2048, - "seq_len_q": 1024, - "seq_len_kv": 1024, - "num_heads": 64, - "num_heads_kv": 8, - "head_dim": 64, - "dtype_q": "nvfp4", - "dtype_k": "fp8", - "dtype_v": "fp8", - "dtype_o": "bf16", - "causal": true, - "sliding_window": 128 - }, - "name": "gptoss" - }, - { - "type": "attention_mha", - "args": { - "batch_size": 2048, - "seq_len_q": 1, - "seq_len_kv": 1536, - "num_heads": 64, - "num_heads_kv": 8, - "head_dim": 64, - "dtype_q": "nvfp4", - "dtype_k": "fp8", - "dtype_v": "fp8", - "dtype_o": "bf16", - "causal": true, - "sliding_window": 128 - }, - "name": "gptoss" - }, - { - "type": "attention_mha", - "args": { - "batch_size": 2560, - "seq_len_q": 1024, - "seq_len_kv": 1024, - "num_heads": 64, - "num_heads_kv": 8, - "head_dim": 64, - "dtype_q": "nvfp4", - "dtype_k": "fp8", - "dtype_v": "fp8", - "dtype_o": "bf16", - "causal": true, - "sliding_window": 128 - }, - "name": "gptoss" - }, - { - "type": "attention_mha", - "args": { - "batch_size": 2560, - "seq_len_q": 1, - "seq_len_kv": 1536, - "num_heads": 64, - "num_heads_kv": 8, - "head_dim": 64, - "dtype_q": "nvfp4", - "dtype_k": "fp8", - "dtype_v": "fp8", - "dtype_o": "bf16", - "causal": true, - "sliding_window": 128 - }, - "name": "gptoss" - }, - { - "type": "attention_mha", - "args": { - "batch_size": 1280, - "seq_len_q": 1024, - "seq_len_kv": 1024, - "num_heads": 64, - "num_heads_kv": 8, - "head_dim": 64, - "dtype_q": "nvfp4", - "dtype_k": "fp8", - "dtype_v": "fp8", - "dtype_o": "bf16", - "causal": true, - "sliding_window": 128 - }, - "name": "gptoss" - }, - { - "type": "attention_mha", - "args": { - "batch_size": 1280, - "seq_len_q": 1, - "seq_len_kv": 1536, - "num_heads": 64, - "num_heads_kv": 8, - "head_dim": 64, - "dtype_q": "nvfp4", - "dtype_k": "fp8", - "dtype_v": "fp8", - "dtype_o": "bf16", - "causal": true, - "sliding_window": 128 - }, - "name": "gptoss" - }, - { - "type": "attention_mha", - "args": { - "batch_size": 1536, - "seq_len_q": 1024, - "seq_len_kv": 1024, - "num_heads": 64, - "num_heads_kv": 8, - "head_dim": 64, - "dtype_q": "nvfp4", - "dtype_k": "fp8", - "dtype_v": "fp8", - "dtype_o": "bf16", - "causal": true, - "sliding_window": 128 - }, - "name": "gptoss" - }, - { - "type": "attention_mha", - "args": { - "batch_size": 1536, - "seq_len_q": 1, - "seq_len_kv": 1536, - "num_heads": 64, - "num_heads_kv": 8, - "head_dim": 64, - "dtype_q": "nvfp4", - "dtype_k": "fp8", - "dtype_v": "fp8", - "dtype_o": "bf16", - "causal": true, - "sliding_window": 128 - }, - "name": "gptoss" - }, - { - "type": "attention_mha", - "args": { - "batch_size": 384, - "seq_len_q": 1024, - "seq_len_kv": 1024, - "num_heads": 64, - "num_heads_kv": 8, - "head_dim": 64, - "dtype_q": "nvfp4", - "dtype_k": "fp8", - "dtype_v": "fp8", - "dtype_o": "bf16", - "causal": true, - "sliding_window": 128 - }, - "name": "gptoss" - }, - { - "type": "attention_mha", - "args": { - "batch_size": 384, - "seq_len_q": 1, - "seq_len_kv": 1536, - "num_heads": 64, - "num_heads_kv": 8, - "head_dim": 64, - "dtype_q": "nvfp4", - "dtype_k": "fp8", - "dtype_v": "fp8", - "dtype_o": "bf16", - "causal": true, - "sliding_window": 128 - }, - "name": "gptoss" - }, - { - "type": "attention_mha", - "args": { - "batch_size": 3072, - "seq_len_q": 1024, - "seq_len_kv": 1024, - "num_heads": 64, - "num_heads_kv": 8, - "head_dim": 64, - "dtype_q": "nvfp4", - "dtype_k": "fp8", - "dtype_v": "fp8", - "dtype_o": "bf16", - "causal": true, - "sliding_window": 128 - }, - "name": "gptoss" - }, - { - "type": "attention_mha", - "args": { - "batch_size": 3072, - "seq_len_q": 1, - "seq_len_kv": 1536, - "num_heads": 64, - "num_heads_kv": 8, - "head_dim": 64, - "dtype_q": "nvfp4", - "dtype_k": "fp8", - "dtype_v": "fp8", - "dtype_o": "bf16", - "causal": true, - "sliding_window": 128 - }, - "name": "gptoss" - }, - { - "type": "attention_mha", - "args": { - "batch_size": 768, - "seq_len_q": 1024, - "seq_len_kv": 1024, - "num_heads": 64, - "num_heads_kv": 8, - "head_dim": 64, - "dtype_q": "nvfp4", - "dtype_k": "fp8", - "dtype_v": "fp8", - "dtype_o": "bf16", - "causal": true, - "sliding_window": 128 - }, - "name": "gptoss" - }, - { - "type": "attention_mha", - "args": { - "batch_size": 768, - "seq_len_q": 1, - "seq_len_kv": 1536, - "num_heads": 64, - "num_heads_kv": 8, - "head_dim": 64, - "dtype_q": "nvfp4", - "dtype_k": "fp8", - "dtype_v": "fp8", - "dtype_o": "bf16", - "causal": true, - "sliding_window": 128 - }, - "name": "gptoss" - }, - { - "type": "attention_mha", - "args": { - "batch_size": 1, - "seq_len_q": 8192, - "seq_len_kv": 8192, - "num_heads": 64, - "num_heads_kv": 8, - "head_dim": 64, - "dtype_q": "nvfp4", - "dtype_k": "fp8", - "dtype_v": "fp8", - "dtype_o": "bf16", - "causal": true, - "sliding_window": 128 - }, - "name": "gptoss" - }, - { - "type": "attention_mha", - "args": { - "batch_size": 1, - "seq_len_q": 1, - "seq_len_kv": 8704, - "num_heads": 64, - "num_heads_kv": 8, - "head_dim": 64, - "dtype_q": "nvfp4", - "dtype_k": "fp8", - "dtype_v": "fp8", - "dtype_o": "bf16", - "causal": true, - "sliding_window": 128 - }, - "name": "gptoss" - }, - { - "type": "attention_mha", - "args": { - "batch_size": 1, - "seq_len_q": 8192, - "seq_len_kv": 8192, - "num_heads": 8, - "num_heads_kv": 1, - "head_dim": 64, - "dtype_q": "nvfp4", - "dtype_k": "fp8", - "dtype_v": "fp8", - "dtype_o": "bf16", - "causal": true, - "sliding_window": 128 - }, - "name": "gptoss" - }, - { - "type": "attention_mha", - "args": { - "batch_size": 1, - "seq_len_q": 1, - "seq_len_kv": 8704, - "num_heads": 8, - "num_heads_kv": 1, - "head_dim": 64, - "dtype_q": "nvfp4", - "dtype_k": "fp8", - "dtype_v": "fp8", - "dtype_o": "bf16", - "causal": true, - "sliding_window": 128 - }, - "name": "gptoss" - }, - { - "type": "attention_mha", - "args": { - "batch_size": 2, - "seq_len_q": 8192, - "seq_len_kv": 8192, - "num_heads": 64, - "num_heads_kv": 8, - "head_dim": 64, - "dtype_q": "nvfp4", - "dtype_k": "fp8", - "dtype_v": "fp8", - "dtype_o": "bf16", - "causal": true, - "sliding_window": 128 - }, - "name": "gptoss" - }, - { - "type": "attention_mha", - "args": { - "batch_size": 2, - "seq_len_q": 1, - "seq_len_kv": 8704, - "num_heads": 64, - "num_heads_kv": 8, - "head_dim": 64, - "dtype_q": "nvfp4", - "dtype_k": "fp8", - "dtype_v": "fp8", - "dtype_o": "bf16", - "causal": true, - "sliding_window": 128 - }, - "name": "gptoss" - }, - { - "type": "attention_mha", - "args": { - "batch_size": 2, - "seq_len_q": 8192, - "seq_len_kv": 8192, - "num_heads": 16, - "num_heads_kv": 2, - "head_dim": 64, - "dtype_q": "nvfp4", - "dtype_k": "fp8", - "dtype_v": "fp8", - "dtype_o": "bf16", - "causal": true, - "sliding_window": 128 - }, - "name": "gptoss" - }, - { - "type": "attention_mha", - "args": { - "batch_size": 2, - "seq_len_q": 1, - "seq_len_kv": 8704, - "num_heads": 16, - "num_heads_kv": 2, - "head_dim": 64, - "dtype_q": "nvfp4", - "dtype_k": "fp8", - "dtype_v": "fp8", - "dtype_o": "bf16", - "causal": true, - "sliding_window": 128 - }, - "name": "gptoss" - }, - { - "type": "attention_mha", - "args": { - "batch_size": 512, - "seq_len_q": 8192, - "seq_len_kv": 8192, - "num_heads": 64, - "num_heads_kv": 8, - "head_dim": 64, - "dtype_q": "nvfp4", - "dtype_k": "fp8", - "dtype_v": "fp8", - "dtype_o": "bf16", - "causal": true, - "sliding_window": 128 - }, - "name": "gptoss" - }, - { - "type": "attention_mha", - "args": { - "batch_size": 512, - "seq_len_q": 1, - "seq_len_kv": 8704, - "num_heads": 64, - "num_heads_kv": 8, - "head_dim": 64, - "dtype_q": "nvfp4", - "dtype_k": "fp8", - "dtype_v": "fp8", - "dtype_o": "bf16", - "causal": true, - "sliding_window": 128 - }, - "name": "gptoss" - }, - { - "type": "attention_mha", - "args": { - "batch_size": 128, - "seq_len_q": 8192, - "seq_len_kv": 8192, - "num_heads": 16, - "num_heads_kv": 2, - "head_dim": 64, - "dtype_q": "nvfp4", - "dtype_k": "fp8", - "dtype_v": "fp8", - "dtype_o": "bf16", - "causal": true, - "sliding_window": 128 - }, - "name": "gptoss" - }, - { - "type": "attention_mha", - "args": { - "batch_size": 128, - "seq_len_q": 1, - "seq_len_kv": 8704, - "num_heads": 16, - "num_heads_kv": 2, - "head_dim": 64, - "dtype_q": "nvfp4", - "dtype_k": "fp8", - "dtype_v": "fp8", - "dtype_o": "bf16", - "causal": true, - "sliding_window": 128 - }, - "name": "gptoss" - }, - { - "type": "attention_mha", - "args": { - "batch_size": 512, - "seq_len_q": 8192, - "seq_len_kv": 8192, - "num_heads": 16, - "num_heads_kv": 2, - "head_dim": 64, - "dtype_q": "nvfp4", - "dtype_k": "fp8", - "dtype_v": "fp8", - "dtype_o": "bf16", - "causal": true, - "sliding_window": 128 - }, - "name": "gptoss" - }, - { - "type": "attention_mha", - "args": { - "batch_size": 512, - "seq_len_q": 1, - "seq_len_kv": 8704, - "num_heads": 16, - "num_heads_kv": 2, - "head_dim": 64, - "dtype_q": "nvfp4", - "dtype_k": "fp8", - "dtype_v": "fp8", - "dtype_o": "bf16", - "causal": true, - "sliding_window": 128 - }, - "name": "gptoss" - }, - { - "type": "attention_mha", - "args": { - "batch_size": 384, - "seq_len_q": 8192, - "seq_len_kv": 8192, - "num_heads": 64, - "num_heads_kv": 8, - "head_dim": 64, - "dtype_q": "nvfp4", - "dtype_k": "fp8", - "dtype_v": "fp8", - "dtype_o": "bf16", - "causal": true, - "sliding_window": 128 - }, - "name": "gptoss" - }, - { - "type": "attention_mha", - "args": { - "batch_size": 384, - "seq_len_q": 1, - "seq_len_kv": 8704, - "num_heads": 64, - "num_heads_kv": 8, - "head_dim": 64, - "dtype_q": "nvfp4", - "dtype_k": "fp8", - "dtype_v": "fp8", - "dtype_o": "bf16", - "causal": true, - "sliding_window": 128 - }, - "name": "gptoss" - }, - { - "type": "attention_mha", - "args": { - "batch_size": 384, - "seq_len_q": 8192, - "seq_len_kv": 8192, - "num_heads": 32, - "num_heads_kv": 4, - "head_dim": 64, - "dtype_q": "nvfp4", - "dtype_k": "fp8", - "dtype_v": "fp8", - "dtype_o": "bf16", - "causal": true, - "sliding_window": 128 - }, - "name": "gptoss" - }, - { - "type": "attention_mha", - "args": { - "batch_size": 384, - "seq_len_q": 1, - "seq_len_kv": 8704, - "num_heads": 32, - "num_heads_kv": 4, - "head_dim": 64, - "dtype_q": "nvfp4", - "dtype_k": "fp8", - "dtype_v": "fp8", - "dtype_o": "bf16", - "causal": true, - "sliding_window": 128 - }, - "name": "gptoss" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 256, - "seq_len_q": 1, - "seq_len_kv": 1536, - "num_heads": 16, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "fp8", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 512, - "seq_len_q": 1, - "seq_len_kv": 1536, - "num_heads": 16, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "fp8", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 1024, - "seq_len_q": 1024, - "seq_len_kv": 1024, - "num_heads": 16, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "fp8", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 1024, - "seq_len_q": 1, - "seq_len_kv": 1536, - "num_heads": 16, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "fp8", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 2048, - "seq_len_q": 1024, - "seq_len_kv": 1024, - "num_heads": 16, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "fp8", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 2048, - "seq_len_q": 1, - "seq_len_kv": 1536, - "num_heads": 16, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "fp8", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 1024, - "seq_len_q": 2, - "seq_len_kv": 1536, - "num_heads": 16, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "fp8", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 2048, - "seq_len_q": 2, - "seq_len_kv": 1536, - "num_heads": 16, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "fp8", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 256, - "seq_len_q": 2, - "seq_len_kv": 1536, - "num_heads": 128, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "fp8", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 2, - "seq_len_q": 8192, - "seq_len_kv": 8192, - "num_heads": 16, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "fp8", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 2, - "seq_len_q": 2, - "seq_len_kv": 8704, - "num_heads": 16, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "fp8", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 4, - "seq_len_q": 1, - "seq_len_kv": 1536, - "num_heads": 128, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 8, - "seq_len_q": 1, - "seq_len_kv": 1536, - "num_heads": 128, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 256, - "seq_len_q": 1024, - "seq_len_kv": 1024, - "num_heads": 16, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 256, - "seq_len_q": 1, - "seq_len_kv": 1536, - "num_heads": 16, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 512, - "seq_len_q": 8192, - "seq_len_kv": 8192, - "num_heads": 128, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 512, - "seq_len_q": 1, - "seq_len_kv": 8704, - "num_heads": 128, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 128, - "seq_len_q": 8192, - "seq_len_kv": 8192, - "num_heads": 128, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 128, - "seq_len_q": 1, - "seq_len_kv": 8704, - "num_heads": 128, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 36, - "seq_len_q": 1, - "seq_len_kv": 8704, - "num_heads": 128, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "fp8", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 512, - "seq_len_q": 2, - "seq_len_kv": 1536, - "num_heads": 128, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "fp8", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 128, - "seq_len_q": 2, - "seq_len_kv": 1536, - "num_heads": 128, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 16, - "seq_len_q": 2, - "seq_len_kv": 1536, - "num_heads": 16, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 512, - "seq_len_q": 1024, - "seq_len_kv": 1024, - "num_heads": 16, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 512, - "seq_len_q": 2, - "seq_len_kv": 1536, - "num_heads": 16, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 16, - "seq_len_q": 2, - "seq_len_kv": 1536, - "num_heads": 128, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 256, - "seq_len_q": 2, - "seq_len_kv": 1536, - "num_heads": 16, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 256, - "seq_len_q": 2, - "seq_len_kv": 1536, - "num_heads": 128, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 64, - "seq_len_q": 2, - "seq_len_kv": 8704, - "num_heads": 16, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 128, - "seq_len_q": 2, - "seq_len_kv": 8704, - "num_heads": 16, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 64, - "seq_len_q": 2, - "seq_len_kv": 8704, - "num_heads": 32, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 16, - "seq_len_q": 2, - "seq_len_kv": 8704, - "num_heads": 16, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 1, - "seq_len_q": 1024, - "seq_len_kv": 1024, - "num_heads": 64, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "kimik2.5" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 1, - "seq_len_q": 1, - "seq_len_kv": 1536, - "num_heads": 64, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "kimik2.5" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 48, - "seq_len_q": 1024, - "seq_len_kv": 1024, - "num_heads": 64, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "kimik2.5" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 48, - "seq_len_q": 1, - "seq_len_kv": 1536, - "num_heads": 64, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "kimik2.5" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 90, - "seq_len_q": 1024, - "seq_len_kv": 1024, - "num_heads": 64, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "kimik2.5" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 90, - "seq_len_q": 1, - "seq_len_kv": 1536, - "num_heads": 64, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "kimik2.5" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 167, - "seq_len_q": 1024, - "seq_len_kv": 1024, - "num_heads": 64, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "kimik2.5" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 167, - "seq_len_q": 1, - "seq_len_kv": 1536, - "num_heads": 64, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "kimik2.5" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 192, - "seq_len_q": 1024, - "seq_len_kv": 1024, - "num_heads": 8, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "kimik2.5" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 192, - "seq_len_q": 1, - "seq_len_kv": 1536, - "num_heads": 8, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "kimik2.5" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 360, - "seq_len_q": 1024, - "seq_len_kv": 1024, - "num_heads": 8, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "kimik2.5" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 360, - "seq_len_q": 1, - "seq_len_kv": 1536, - "num_heads": 8, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "kimik2.5" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 668, - "seq_len_q": 1024, - "seq_len_kv": 1024, - "num_heads": 8, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "kimik2.5" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 668, - "seq_len_q": 1, - "seq_len_kv": 1536, - "num_heads": 8, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "kimik2.5" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 3, - "seq_len_q": 1024, - "seq_len_kv": 1024, - "num_heads": 64, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "kimik2.5" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 3, - "seq_len_q": 1, - "seq_len_kv": 1536, - "num_heads": 64, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "kimik2.5" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 7, - "seq_len_q": 1024, - "seq_len_kv": 1024, - "num_heads": 64, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "kimik2.5" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 7, - "seq_len_q": 1, - "seq_len_kv": 1536, - "num_heads": 64, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "kimik2.5" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 13, - "seq_len_q": 1024, - "seq_len_kv": 1024, - "num_heads": 64, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "kimik2.5" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 13, - "seq_len_q": 1, - "seq_len_kv": 1536, - "num_heads": 64, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "kimik2.5" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 30, - "seq_len_q": 1024, - "seq_len_kv": 1024, - "num_heads": 16, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "kimik2.5" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 30, - "seq_len_q": 1, - "seq_len_kv": 1536, - "num_heads": 16, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "kimik2.5" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 55, - "seq_len_q": 1024, - "seq_len_kv": 1024, - "num_heads": 16, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "kimik2.5" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 55, - "seq_len_q": 1, - "seq_len_kv": 1536, - "num_heads": 16, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "kimik2.5" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 166, - "seq_len_q": 1024, - "seq_len_kv": 1024, - "num_heads": 64, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "kimik2.5" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 166, - "seq_len_q": 1, - "seq_len_kv": 1536, - "num_heads": 64, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "kimik2.5" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 41, - "seq_len_q": 1024, - "seq_len_kv": 1024, - "num_heads": 64, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "kimik2.5" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 41, - "seq_len_q": 1, - "seq_len_kv": 1536, - "num_heads": 64, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "kimik2.5" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 563, - "seq_len_q": 1024, - "seq_len_kv": 1024, - "num_heads": 64, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "kimik2.5" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 563, - "seq_len_q": 1, - "seq_len_kv": 1536, - "num_heads": 64, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "kimik2.5" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 70, - "seq_len_q": 1024, - "seq_len_kv": 1024, - "num_heads": 64, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "kimik2.5" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 70, - "seq_len_q": 1, - "seq_len_kv": 1536, - "num_heads": 64, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "kimik2.5" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 1075, - "seq_len_q": 1024, - "seq_len_kv": 1024, - "num_heads": 64, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "kimik2.5" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 1075, - "seq_len_q": 1, - "seq_len_kv": 1536, - "num_heads": 64, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "kimik2.5" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 1613, - "seq_len_q": 1024, - "seq_len_kv": 1024, - "num_heads": 64, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "kimik2.5" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 1613, - "seq_len_q": 1, - "seq_len_kv": 1536, - "num_heads": 64, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "kimik2.5" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 537, - "seq_len_q": 1024, - "seq_len_kv": 1024, - "num_heads": 64, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "kimik2.5" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 537, - "seq_len_q": 1, - "seq_len_kv": 1536, - "num_heads": 64, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "kimik2.5" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 806, - "seq_len_q": 1024, - "seq_len_kv": 1024, - "num_heads": 64, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "kimik2.5" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 806, - "seq_len_q": 1, - "seq_len_kv": 1536, - "num_heads": 64, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "kimik2.5" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 268, - "seq_len_q": 1024, - "seq_len_kv": 1024, - "num_heads": 64, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "kimik2.5" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 268, - "seq_len_q": 1, - "seq_len_kv": 1536, - "num_heads": 64, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "kimik2.5" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 134, - "seq_len_q": 1024, - "seq_len_kv": 1024, - "num_heads": 64, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "kimik2.5" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 134, - "seq_len_q": 1, - "seq_len_kv": 1536, - "num_heads": 64, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "kimik2.5" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 1, - "seq_len_q": 8192, - "seq_len_kv": 8192, - "num_heads": 64, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "kimik2.5" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 1, - "seq_len_q": 1, - "seq_len_kv": 8704, - "num_heads": 64, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "kimik2.5" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 39, - "seq_len_q": 8192, - "seq_len_kv": 8192, - "num_heads": 64, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "kimik2.5" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 39, - "seq_len_q": 1, - "seq_len_kv": 8704, - "num_heads": 64, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "kimik2.5" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 156, - "seq_len_q": 8192, - "seq_len_kv": 8192, - "num_heads": 16, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "kimik2.5" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 156, - "seq_len_q": 1, - "seq_len_kv": 8704, - "num_heads": 16, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "kimik2.5" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 3, - "seq_len_q": 8192, - "seq_len_kv": 8192, - "num_heads": 64, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "kimik2.5" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 3, - "seq_len_q": 1, - "seq_len_kv": 8704, - "num_heads": 64, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "kimik2.5" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 7, - "seq_len_q": 8192, - "seq_len_kv": 8192, - "num_heads": 64, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "kimik2.5" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 7, - "seq_len_q": 1, - "seq_len_kv": 8704, - "num_heads": 64, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "kimik2.5" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 15, - "seq_len_q": 8192, - "seq_len_kv": 8192, - "num_heads": 64, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "kimik2.5" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 15, - "seq_len_q": 1, - "seq_len_kv": 8704, - "num_heads": 64, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "kimik2.5" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 26, - "seq_len_q": 8192, - "seq_len_kv": 8192, - "num_heads": 64, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "kimik2.5" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 26, - "seq_len_q": 1, - "seq_len_kv": 8704, - "num_heads": 64, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "kimik2.5" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 30, - "seq_len_q": 1, - "seq_len_kv": 8704, - "num_heads": 16, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "kimik2.5" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 60, - "seq_len_q": 1, - "seq_len_kv": 8704, - "num_heads": 16, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "kimik2.5" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 105, - "seq_len_q": 8192, - "seq_len_kv": 8192, - "num_heads": 16, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "kimik2.5" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 105, - "seq_len_q": 1, - "seq_len_kv": 8704, - "num_heads": 16, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "kimik2.5" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 83, - "seq_len_q": 8192, - "seq_len_kv": 8192, - "num_heads": 64, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "kimik2.5" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 83, - "seq_len_q": 1, - "seq_len_kv": 8704, - "num_heads": 64, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "kimik2.5" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 20, - "seq_len_q": 8192, - "seq_len_kv": 8192, - "num_heads": 64, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "kimik2.5" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 20, - "seq_len_q": 1, - "seq_len_kv": 8704, - "num_heads": 64, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "kimik2.5" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 153, - "seq_len_q": 8192, - "seq_len_kv": 8192, - "num_heads": 64, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "kimik2.5" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 153, - "seq_len_q": 1, - "seq_len_kv": 8704, - "num_heads": 64, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "kimik2.5" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 38, - "seq_len_q": 8192, - "seq_len_kv": 8192, - "num_heads": 64, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "kimik2.5" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 38, - "seq_len_q": 1, - "seq_len_kv": 8704, - "num_heads": 64, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "kimik2.5" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 537, - "seq_len_q": 8192, - "seq_len_kv": 8192, - "num_heads": 64, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "kimik2.5" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 537, - "seq_len_q": 1, - "seq_len_kv": 8704, - "num_heads": 64, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "kimik2.5" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 268, - "seq_len_q": 8192, - "seq_len_kv": 8192, - "num_heads": 64, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "kimik2.5" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 268, - "seq_len_q": 1, - "seq_len_kv": 8704, - "num_heads": 64, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "kimik2.5" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 563, - "seq_len_q": 8192, - "seq_len_kv": 8192, - "num_heads": 64, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "kimik2.5" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 563, - "seq_len_q": 1, - "seq_len_kv": 8704, - "num_heads": 64, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "kimik2.5" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 140, - "seq_len_q": 8192, - "seq_len_kv": 8192, - "num_heads": 64, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "kimik2.5" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 140, - "seq_len_q": 1, - "seq_len_kv": 8704, - "num_heads": 64, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "kimik2.5" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 64, - "seq_len_q": 1024, - "seq_len_kv": 1024, - "num_heads": 64, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "kimik2.5" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 64, - "seq_len_q": 1, - "seq_len_kv": 1536, - "num_heads": 64, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "kimik2.5" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 128, - "seq_len_q": 1024, - "seq_len_kv": 1024, - "num_heads": 64, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "kimik2.5" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 128, - "seq_len_q": 1, - "seq_len_kv": 1536, - "num_heads": 64, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "kimik2.5" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 256, - "seq_len_q": 1024, - "seq_len_kv": 1024, - "num_heads": 64, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "kimik2.5" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 256, - "seq_len_q": 1, - "seq_len_kv": 1536, - "num_heads": 64, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "kimik2.5" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 512, - "seq_len_q": 1024, - "seq_len_kv": 1024, - "num_heads": 64, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "kimik2.5" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 512, - "seq_len_q": 1, - "seq_len_kv": 1536, - "num_heads": 64, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "kimik2.5" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 768, - "seq_len_q": 1024, - "seq_len_kv": 1024, - "num_heads": 64, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "kimik2.5" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 768, - "seq_len_q": 1, - "seq_len_kv": 1536, - "num_heads": 64, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "kimik2.5" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 1024, - "seq_len_q": 1024, - "seq_len_kv": 1024, - "num_heads": 64, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "kimik2.5" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 1024, - "seq_len_q": 1, - "seq_len_kv": 1536, - "num_heads": 64, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "kimik2.5" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 16, - "seq_len_q": 1024, - "seq_len_kv": 1024, - "num_heads": 64, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "kimik2.5" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 16, - "seq_len_q": 1, - "seq_len_kv": 1536, - "num_heads": 64, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "kimik2.5" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 32, - "seq_len_q": 1024, - "seq_len_kv": 1024, - "num_heads": 64, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "kimik2.5" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 32, - "seq_len_q": 1, - "seq_len_kv": 1536, - "num_heads": 64, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "kimik2.5" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 192, - "seq_len_q": 1024, - "seq_len_kv": 1024, - "num_heads": 64, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "kimik2.5" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 192, - "seq_len_q": 1, - "seq_len_kv": 1536, - "num_heads": 64, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "kimik2.5" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 2, - "seq_len_q": 1024, - "seq_len_kv": 1024, - "num_heads": 64, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "kimik2.5" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 2, - "seq_len_q": 1, - "seq_len_kv": 1536, - "num_heads": 64, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "kimik2.5" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 4, - "seq_len_q": 1024, - "seq_len_kv": 1024, - "num_heads": 64, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "kimik2.5" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 4, - "seq_len_q": 1, - "seq_len_kv": 1536, - "num_heads": 64, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "kimik2.5" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 8, - "seq_len_q": 1024, - "seq_len_kv": 1024, - "num_heads": 64, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "kimik2.5" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 8, - "seq_len_q": 1, - "seq_len_kv": 1536, - "num_heads": 64, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "kimik2.5" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 2, - "seq_len_q": 8192, - "seq_len_kv": 8192, - "num_heads": 64, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "kimik2.5" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 2, - "seq_len_q": 1, - "seq_len_kv": 8704, - "num_heads": 64, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "kimik2.5" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 4, - "seq_len_q": 8192, - "seq_len_kv": 8192, - "num_heads": 64, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "kimik2.5" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 4, - "seq_len_q": 1, - "seq_len_kv": 8704, - "num_heads": 64, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "kimik2.5" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 8, - "seq_len_q": 8192, - "seq_len_kv": 8192, - "num_heads": 64, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "kimik2.5" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 8, - "seq_len_q": 1, - "seq_len_kv": 8704, - "num_heads": 64, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "kimik2.5" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 32, - "seq_len_q": 8192, - "seq_len_kv": 8192, - "num_heads": 64, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "kimik2.5" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 32, - "seq_len_q": 1, - "seq_len_kv": 8704, - "num_heads": 64, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "kimik2.5" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 128, - "seq_len_q": 8192, - "seq_len_kv": 8192, - "num_heads": 64, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "kimik2.5" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 128, - "seq_len_q": 1, - "seq_len_kv": 8704, - "num_heads": 64, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "kimik2.5" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 256, - "seq_len_q": 8192, - "seq_len_kv": 8192, - "num_heads": 64, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "kimik2.5" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 256, - "seq_len_q": 1, - "seq_len_kv": 8704, - "num_heads": 64, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "kimik2.5" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 64, - "seq_len_q": 8192, - "seq_len_kv": 8192, - "num_heads": 64, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "kimik2.5" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 64, - "seq_len_q": 1, - "seq_len_kv": 8704, - "num_heads": 64, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "kimik2.5" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 512, - "seq_len_q": 8192, - "seq_len_kv": 8192, - "num_heads": 64, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "kimik2.5" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 512, - "seq_len_q": 1, - "seq_len_kv": 8704, - "num_heads": 64, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "kimik2.5" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 768, - "seq_len_q": 8192, - "seq_len_kv": 8192, - "num_heads": 64, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "kimik2.5" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 768, - "seq_len_q": 1, - "seq_len_kv": 8704, - "num_heads": 64, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "kimik2.5" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 1024, - "seq_len_q": 8192, - "seq_len_kv": 8192, - "num_heads": 64, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "kimik2.5" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 1024, - "seq_len_q": 1, - "seq_len_kv": 8704, - "num_heads": 64, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "kimik2.5" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 192, - "seq_len_q": 8192, - "seq_len_kv": 8192, - "num_heads": 64, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "kimik2.5" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 192, - "seq_len_q": 1, - "seq_len_kv": 8704, - "num_heads": 64, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "kimik2.5" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 1, - "seq_len_q": 8192, - "seq_len_kv": 8192, - "num_heads": 128, - "head_dim_qk_nope": 448, - "head_dim_qk_rope": 64, - "head_dim_v": 512, - "kv_lora_rank": 1024, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsv4" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 1, - "seq_len_q": 1, - "seq_len_kv": 8704, - "num_heads": 128, - "head_dim_qk_nope": 448, - "head_dim_qk_rope": 64, - "head_dim_v": 512, - "kv_lora_rank": 1024, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsv4" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 2, - "seq_len_q": 8192, - "seq_len_kv": 8192, - "num_heads": 128, - "head_dim_qk_nope": 448, - "head_dim_qk_rope": 64, - "head_dim_v": 512, - "kv_lora_rank": 1024, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsv4" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 2, - "seq_len_q": 1, - "seq_len_kv": 8704, - "num_heads": 128, - "head_dim_qk_nope": 448, - "head_dim_qk_rope": 64, - "head_dim_v": 512, - "kv_lora_rank": 1024, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsv4" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 64, - "seq_len_q": 1, - "seq_len_kv": 8704, - "num_heads": 16, - "head_dim_qk_nope": 448, - "head_dim_qk_rope": 64, - "head_dim_v": 512, - "kv_lora_rank": 1024, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsv4" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 128, - "seq_len_q": 8192, - "seq_len_kv": 8192, - "num_heads": 16, - "head_dim_qk_nope": 448, - "head_dim_qk_rope": 64, - "head_dim_v": 512, - "kv_lora_rank": 1024, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsv4" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 128, - "seq_len_q": 1, - "seq_len_kv": 8704, - "num_heads": 16, - "head_dim_qk_nope": 448, - "head_dim_qk_rope": 64, - "head_dim_v": 512, - "kv_lora_rank": 1024, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsv4" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 1024, - "seq_len_q": 8192, - "seq_len_kv": 8192, - "num_heads": 128, - "head_dim_qk_nope": 448, - "head_dim_qk_rope": 64, - "head_dim_v": 512, - "kv_lora_rank": 1024, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsv4" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 1024, - "seq_len_q": 1, - "seq_len_kv": 8704, - "num_heads": 128, - "head_dim_qk_nope": 448, - "head_dim_qk_rope": 64, - "head_dim_v": 512, - "kv_lora_rank": 1024, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsv4" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 1543, - "seq_len_q": 8192, - "seq_len_kv": 8192, - "num_heads": 128, - "head_dim_qk_nope": 448, - "head_dim_qk_rope": 64, - "head_dim_v": 512, - "kv_lora_rank": 1024, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsv4" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 1543, - "seq_len_q": 1, - "seq_len_kv": 8704, - "num_heads": 128, - "head_dim_qk_nope": 448, - "head_dim_qk_rope": 64, - "head_dim_v": 512, - "kv_lora_rank": 1024, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsv4" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 256, - "seq_len_q": 8192, - "seq_len_kv": 8192, - "num_heads": 16, - "head_dim_qk_nope": 448, - "head_dim_qk_rope": 64, - "head_dim_v": 512, - "kv_lora_rank": 1024, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsv4" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 256, - "seq_len_q": 1, - "seq_len_kv": 8704, - "num_heads": 16, - "head_dim_qk_nope": 448, - "head_dim_qk_rope": 64, - "head_dim_v": 512, - "kv_lora_rank": 1024, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsv4" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 512, - "seq_len_q": 8192, - "seq_len_kv": 8192, - "num_heads": 16, - "head_dim_qk_nope": 448, - "head_dim_qk_rope": 64, - "head_dim_v": 512, - "kv_lora_rank": 1024, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsv4" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 512, - "seq_len_q": 1, - "seq_len_kv": 8704, - "num_heads": 16, - "head_dim_qk_nope": 448, - "head_dim_qk_rope": 64, - "head_dim_v": 512, - "kv_lora_rank": 1024, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsv4" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 2, - "seq_len_q": 2, - "seq_len_kv": 8704, - "num_heads": 128, - "head_dim_qk_nope": 448, - "head_dim_qk_rope": 64, - "head_dim_v": 512, - "kv_lora_rank": 1024, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsv4" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 48, - "seq_len_q": 8192, - "seq_len_kv": 8192, - "num_heads": 128, - "head_dim_qk_nope": 448, - "head_dim_qk_rope": 64, - "head_dim_v": 512, - "kv_lora_rank": 1024, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsv4" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 48, - "seq_len_q": 1, - "seq_len_kv": 8704, - "num_heads": 128, - "head_dim_qk_nope": 448, - "head_dim_qk_rope": 64, - "head_dim_v": 512, - "kv_lora_rank": 1024, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsv4" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 192, - "seq_len_q": 8192, - "seq_len_kv": 8192, - "num_heads": 32, - "head_dim_qk_nope": 448, - "head_dim_qk_rope": 64, - "head_dim_v": 512, - "kv_lora_rank": 1024, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsv4" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 192, - "seq_len_q": 1, - "seq_len_kv": 8704, - "num_heads": 32, - "head_dim_qk_nope": 448, - "head_dim_qk_rope": 64, - "head_dim_v": 512, - "kv_lora_rank": 1024, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsv4" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 18, - "seq_len_q": 8192, - "seq_len_kv": 8192, - "num_heads": 32, - "head_dim_qk_nope": 448, - "head_dim_qk_rope": 64, - "head_dim_v": 512, - "kv_lora_rank": 1024, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsv4" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 18, - "seq_len_q": 1, - "seq_len_kv": 8704, - "num_heads": 32, - "head_dim_qk_nope": 448, - "head_dim_qk_rope": 64, - "head_dim_v": 512, - "kv_lora_rank": 1024, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsv4" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 768, - "seq_len_q": 8192, - "seq_len_kv": 8192, - "num_heads": 128, - "head_dim_qk_nope": 448, - "head_dim_qk_rope": 64, - "head_dim_v": 512, - "kv_lora_rank": 1024, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsv4" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 768, - "seq_len_q": 1, - "seq_len_kv": 8704, - "num_heads": 128, - "head_dim_qk_nope": 448, - "head_dim_qk_rope": 64, - "head_dim_v": 512, - "kv_lora_rank": 1024, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsv4" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 192, - "seq_len_q": 8192, - "seq_len_kv": 8192, - "num_heads": 128, - "head_dim_qk_nope": 448, - "head_dim_qk_rope": 64, - "head_dim_v": 512, - "kv_lora_rank": 1024, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsv4" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 192, - "seq_len_q": 1, - "seq_len_kv": 8704, - "num_heads": 128, - "head_dim_qk_nope": 448, - "head_dim_qk_rope": 64, - "head_dim_v": 512, - "kv_lora_rank": 1024, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsv4" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 2048, - "seq_len_q": 8192, - "seq_len_kv": 8192, - "num_heads": 128, - "head_dim_qk_nope": 448, - "head_dim_qk_rope": 64, - "head_dim_v": 512, - "kv_lora_rank": 1024, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsv4" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 2048, - "seq_len_q": 1, - "seq_len_kv": 8704, - "num_heads": 128, - "head_dim_qk_nope": 448, - "head_dim_qk_rope": 64, - "head_dim_v": 512, - "kv_lora_rank": 1024, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsv4" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 5376, - "seq_len_q": 8192, - "seq_len_kv": 8192, - "num_heads": 128, - "head_dim_qk_nope": 448, - "head_dim_qk_rope": 64, - "head_dim_v": 512, - "kv_lora_rank": 1024, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsv4" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 5376, - "seq_len_q": 1, - "seq_len_kv": 8704, - "num_heads": 128, - "head_dim_qk_nope": 448, - "head_dim_qk_rope": 64, - "head_dim_v": 512, - "kv_lora_rank": 1024, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsv4" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 1792, - "seq_len_q": 8192, - "seq_len_kv": 8192, - "num_heads": 128, - "head_dim_qk_nope": 448, - "head_dim_qk_rope": 64, - "head_dim_v": 512, - "kv_lora_rank": 1024, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsv4" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 1792, - "seq_len_q": 1, - "seq_len_kv": 8704, - "num_heads": 128, - "head_dim_qk_nope": 448, - "head_dim_qk_rope": 64, - "head_dim_v": 512, - "kv_lora_rank": 1024, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsv4" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 256, - "seq_len_q": 2, - "seq_len_kv": 8704, - "num_heads": 128, - "head_dim_qk_nope": 448, - "head_dim_qk_rope": 64, - "head_dim_v": 512, - "kv_lora_rank": 1024, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsv4" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 256, - "seq_len_q": 1024, - "seq_len_kv": 1024, - "num_heads": 32, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 256, - "seq_len_q": 1, - "seq_len_kv": 1536, - "num_heads": 32, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 16, - "seq_len_q": 2, - "seq_len_kv": 1536, - "num_heads": 32, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 32, - "seq_len_q": 2, - "seq_len_kv": 1536, - "num_heads": 32, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 64, - "seq_len_q": 2, - "seq_len_kv": 1536, - "num_heads": 32, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 128, - "seq_len_q": 2, - "seq_len_kv": 1536, - "num_heads": 32, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 256, - "seq_len_q": 2, - "seq_len_kv": 1536, - "num_heads": 32, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 256, - "seq_len_q": 2, - "seq_len_kv": 8704, - "num_heads": 16, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 64, - "seq_len_q": 8192, - "seq_len_kv": 8192, - "num_heads": 32, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "fp8", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 64, - "seq_len_q": 1, - "seq_len_kv": 8704, - "num_heads": 32, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "fp8", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mha", - "args": { - "batch_size": 128, - "seq_len_q": 1024, - "seq_len_kv": 1024, - "num_heads": 4, - "num_heads_kv": 1, - "head_dim": 256, - "dtype_q": "bf16", - "dtype_k": "bf16", - "dtype_v": "bf16", - "dtype_o": "bf16", - "causal": true - }, - "name": "qwen3.5" - }, - { - "type": "attention_mha", - "args": { - "batch_size": 128, - "seq_len_q": 1, - "seq_len_kv": 1536, - "num_heads": 4, - "num_heads_kv": 1, - "head_dim": 256, - "dtype_q": "bf16", - "dtype_k": "bf16", - "dtype_v": "bf16", - "dtype_o": "bf16", - "causal": true - }, - "name": "qwen3.5" - }, - { - "type": "attention_mha", - "args": { - "batch_size": 256, - "seq_len_q": 1024, - "seq_len_kv": 1024, - "num_heads": 4, - "num_heads_kv": 1, - "head_dim": 256, - "dtype_q": "bf16", - "dtype_k": "bf16", - "dtype_v": "bf16", - "dtype_o": "bf16", - "causal": true - }, - "name": "qwen3.5" - }, - { - "type": "attention_mha", - "args": { - "batch_size": 256, - "seq_len_q": 1, - "seq_len_kv": 1536, - "num_heads": 4, - "num_heads_kv": 1, - "head_dim": 256, - "dtype_q": "bf16", - "dtype_k": "bf16", - "dtype_v": "bf16", - "dtype_o": "bf16", - "causal": true - }, - "name": "qwen3.5" - }, - { - "type": "attention_mha", - "args": { - "batch_size": 128, - "seq_len_q": 8192, - "seq_len_kv": 8192, - "num_heads": 4, - "num_heads_kv": 1, - "head_dim": 256, - "dtype_q": "bf16", - "dtype_k": "bf16", - "dtype_v": "bf16", - "dtype_o": "bf16", - "causal": true - }, - "name": "qwen3.5" - }, - { - "type": "attention_mha", - "args": { - "batch_size": 128, - "seq_len_q": 1, - "seq_len_kv": 8704, - "num_heads": 4, - "num_heads_kv": 1, - "head_dim": 256, - "dtype_q": "bf16", - "dtype_k": "bf16", - "dtype_v": "bf16", - "dtype_o": "bf16", - "causal": true - }, - "name": "qwen3.5" - }, - { - "type": "attention_mha", - "args": { - "batch_size": 256, - "seq_len_q": 8192, - "seq_len_kv": 8192, - "num_heads": 4, - "num_heads_kv": 1, - "head_dim": 256, - "dtype_q": "bf16", - "dtype_k": "bf16", - "dtype_v": "bf16", - "dtype_o": "bf16", - "causal": true - }, - "name": "qwen3.5" - }, - { - "type": "attention_mha", - "args": { - "batch_size": 256, - "seq_len_q": 1, - "seq_len_kv": 8704, - "num_heads": 4, - "num_heads_kv": 1, - "head_dim": 256, - "dtype_q": "bf16", - "dtype_k": "bf16", - "dtype_v": "bf16", - "dtype_o": "bf16", - "causal": true - }, - "name": "qwen3.5" - }, - { - "type": "attention_mha", - "args": { - "batch_size": 128, - "seq_len_q": 2, - "seq_len_kv": 1536, - "num_heads": 4, - "num_heads_kv": 1, - "head_dim": 256, - "dtype_q": "bf16", - "dtype_k": "bf16", - "dtype_v": "bf16", - "dtype_o": "bf16", - "causal": true - }, - "name": "qwen3.5" - }, - { - "type": "attention_mha", - "args": { - "batch_size": 256, - "seq_len_q": 2, - "seq_len_kv": 1536, - "num_heads": 4, - "num_heads_kv": 1, - "head_dim": 256, - "dtype_q": "bf16", - "dtype_k": "bf16", - "dtype_v": "bf16", - "dtype_o": "bf16", - "causal": true - }, - "name": "qwen3.5" - }, - { - "type": "attention_mha", - "args": { - "batch_size": 128, - "seq_len_q": 2, - "seq_len_kv": 8704, - "num_heads": 4, - "num_heads_kv": 1, - "head_dim": 256, - "dtype_q": "bf16", - "dtype_k": "bf16", - "dtype_v": "bf16", - "dtype_o": "bf16", - "causal": true - }, - "name": "qwen3.5" - }, - { - "type": "attention_mha", - "args": { - "batch_size": 256, - "seq_len_q": 2, - "seq_len_kv": 8704, - "num_heads": 4, - "num_heads_kv": 1, - "head_dim": 256, - "dtype_q": "bf16", - "dtype_k": "bf16", - "dtype_v": "bf16", - "dtype_o": "bf16", - "causal": true - }, - "name": "qwen3.5" - }, - { - "type": "attention_mha", - "args": { - "batch_size": 128, - "seq_len_q": 1, - "seq_len_kv": 1536, - "num_heads": 4, - "num_heads_kv": 1, - "head_dim": 256, - "dtype_q": "fp8", - "dtype_k": "fp8", - "dtype_v": "fp8", - "dtype_o": "bf16", - "causal": true - }, - "name": "qwen3.5" - }, - { - "type": "attention_mha", - "args": { - "batch_size": 256, - "seq_len_q": 1024, - "seq_len_kv": 1024, - "num_heads": 4, - "num_heads_kv": 1, - "head_dim": 256, - "dtype_q": "fp8", - "dtype_k": "fp8", - "dtype_v": "fp8", - "dtype_o": "bf16", - "causal": true - }, - "name": "qwen3.5" - }, - { - "type": "attention_mha", - "args": { - "batch_size": 256, - "seq_len_q": 1, - "seq_len_kv": 1536, - "num_heads": 4, - "num_heads_kv": 1, - "head_dim": 256, - "dtype_q": "fp8", - "dtype_k": "fp8", - "dtype_v": "fp8", - "dtype_o": "bf16", - "causal": true - }, - "name": "qwen3.5" - }, - { - "type": "attention_mha", - "args": { - "batch_size": 128, - "seq_len_q": 1024, - "seq_len_kv": 1024, - "num_heads": 16, - "num_heads_kv": 1, - "head_dim": 256, - "dtype_q": "fp8", - "dtype_k": "fp8", - "dtype_v": "fp8", - "dtype_o": "bf16", - "causal": true - }, - "name": "qwen3.5" - }, - { - "type": "attention_mha", - "args": { - "batch_size": 128, - "seq_len_q": 1, - "seq_len_kv": 1536, - "num_heads": 16, - "num_heads_kv": 1, - "head_dim": 256, - "dtype_q": "fp8", - "dtype_k": "fp8", - "dtype_v": "fp8", - "dtype_o": "bf16", - "causal": true - }, - "name": "qwen3.5" - }, - { - "type": "attention_mha", - "args": { - "batch_size": 256, - "seq_len_q": 1024, - "seq_len_kv": 1024, - "num_heads": 16, - "num_heads_kv": 1, - "head_dim": 256, - "dtype_q": "fp8", - "dtype_k": "fp8", - "dtype_v": "fp8", - "dtype_o": "bf16", - "causal": true - }, - "name": "qwen3.5" - }, - { - "type": "attention_mha", - "args": { - "batch_size": 256, - "seq_len_q": 1, - "seq_len_kv": 1536, - "num_heads": 16, - "num_heads_kv": 1, - "head_dim": 256, - "dtype_q": "fp8", - "dtype_k": "fp8", - "dtype_v": "fp8", - "dtype_o": "bf16", - "causal": true - }, - "name": "qwen3.5" - }, - { - "type": "attention_mha", - "args": { - "batch_size": 4, - "seq_len_q": 8192, - "seq_len_kv": 8192, - "num_heads": 16, - "num_heads_kv": 1, - "head_dim": 256, - "dtype_q": "fp8", - "dtype_k": "fp8", - "dtype_v": "fp8", - "dtype_o": "bf16", - "causal": true - }, - "name": "qwen3.5" - }, - { - "type": "attention_mha", - "args": { - "batch_size": 4, - "seq_len_q": 1, - "seq_len_kv": 8704, - "num_heads": 16, - "num_heads_kv": 1, - "head_dim": 256, - "dtype_q": "fp8", - "dtype_k": "fp8", - "dtype_v": "fp8", - "dtype_o": "bf16", - "causal": true - }, - "name": "qwen3.5" - }, - { - "type": "attention_mha", - "args": { - "batch_size": 8, - "seq_len_q": 8192, - "seq_len_kv": 8192, - "num_heads": 16, - "num_heads_kv": 1, - "head_dim": 256, - "dtype_q": "fp8", - "dtype_k": "fp8", - "dtype_v": "fp8", - "dtype_o": "bf16", - "causal": true - }, - "name": "qwen3.5" - }, - { - "type": "attention_mha", - "args": { - "batch_size": 8, - "seq_len_q": 1, - "seq_len_kv": 8704, - "num_heads": 16, - "num_heads_kv": 1, - "head_dim": 256, - "dtype_q": "fp8", - "dtype_k": "fp8", - "dtype_v": "fp8", - "dtype_o": "bf16", - "causal": true - }, - "name": "qwen3.5" - }, - { - "type": "attention_mha", - "args": { - "batch_size": 16, - "seq_len_q": 8192, - "seq_len_kv": 8192, - "num_heads": 16, - "num_heads_kv": 1, - "head_dim": 256, - "dtype_q": "fp8", - "dtype_k": "fp8", - "dtype_v": "fp8", - "dtype_o": "bf16", - "causal": true - }, - "name": "qwen3.5" - }, - { - "type": "attention_mha", - "args": { - "batch_size": 16, - "seq_len_q": 1, - "seq_len_kv": 8704, - "num_heads": 16, - "num_heads_kv": 1, - "head_dim": 256, - "dtype_q": "fp8", - "dtype_k": "fp8", - "dtype_v": "fp8", - "dtype_o": "bf16", - "causal": true - }, - "name": "qwen3.5" - }, - { - "type": "attention_mha", - "args": { - "batch_size": 32, - "seq_len_q": 8192, - "seq_len_kv": 8192, - "num_heads": 16, - "num_heads_kv": 1, - "head_dim": 256, - "dtype_q": "fp8", - "dtype_k": "fp8", - "dtype_v": "fp8", - "dtype_o": "bf16", - "causal": true - }, - "name": "qwen3.5" - }, - { - "type": "attention_mha", - "args": { - "batch_size": 32, - "seq_len_q": 1, - "seq_len_kv": 8704, - "num_heads": 16, - "num_heads_kv": 1, - "head_dim": 256, - "dtype_q": "fp8", - "dtype_k": "fp8", - "dtype_v": "fp8", - "dtype_o": "bf16", - "causal": true - }, - "name": "qwen3.5" - }, - { - "type": "attention_mha", - "args": { - "batch_size": 256, - "seq_len_q": 2, - "seq_len_kv": 1536, - "num_heads": 4, - "num_heads_kv": 1, - "head_dim": 256, - "dtype_q": "fp8", - "dtype_k": "fp8", - "dtype_v": "fp8", - "dtype_o": "bf16", - "causal": true - }, - "name": "qwen3.5" - }, - { - "type": "attention_mha", - "args": { - "batch_size": 128, - "seq_len_q": 2, - "seq_len_kv": 1536, - "num_heads": 16, - "num_heads_kv": 1, - "head_dim": 256, - "dtype_q": "fp8", - "dtype_k": "fp8", - "dtype_v": "fp8", - "dtype_o": "bf16", - "causal": true - }, - "name": "qwen3.5" - }, - { - "type": "attention_mha", - "args": { - "batch_size": 256, - "seq_len_q": 2, - "seq_len_kv": 1536, - "num_heads": 16, - "num_heads_kv": 1, - "head_dim": 256, - "dtype_q": "fp8", - "dtype_k": "fp8", - "dtype_v": "fp8", - "dtype_o": "bf16", - "causal": true - }, - "name": "qwen3.5" - }, - { - "type": "attention_mha", - "args": { - "batch_size": 4, - "seq_len_q": 2, - "seq_len_kv": 8704, - "num_heads": 16, - "num_heads_kv": 1, - "head_dim": 256, - "dtype_q": "fp8", - "dtype_k": "fp8", - "dtype_v": "fp8", - "dtype_o": "bf16", - "causal": true - }, - "name": "qwen3.5" - }, - { - "type": "attention_mha", - "args": { - "batch_size": 8, - "seq_len_q": 2, - "seq_len_kv": 8704, - "num_heads": 16, - "num_heads_kv": 1, - "head_dim": 256, - "dtype_q": "fp8", - "dtype_k": "fp8", - "dtype_v": "fp8", - "dtype_o": "bf16", - "causal": true - }, - "name": "qwen3.5" - }, - { - "type": "attention_mha", - "args": { - "batch_size": 16, - "seq_len_q": 2, - "seq_len_kv": 8704, - "num_heads": 16, - "num_heads_kv": 1, - "head_dim": 256, - "dtype_q": "fp8", - "dtype_k": "fp8", - "dtype_v": "fp8", - "dtype_o": "bf16", - "causal": true - }, - "name": "qwen3.5" - }, - { - "type": "attention_mha", - "args": { - "batch_size": 32, - "seq_len_q": 2, - "seq_len_kv": 8704, - "num_heads": 16, - "num_heads_kv": 1, - "head_dim": 256, - "dtype_q": "fp8", - "dtype_k": "fp8", - "dtype_v": "fp8", - "dtype_o": "bf16", - "causal": true - }, - "name": "qwen3.5" - }, - { - "type": "attention_mha", - "args": { - "batch_size": 4, - "seq_len_q": 1024, - "seq_len_kv": 1024, - "num_heads": 16, - "num_heads_kv": 1, - "head_dim": 256, - "dtype_q": "fp8", - "dtype_k": "fp8", - "dtype_v": "fp8", - "dtype_o": "bf16", - "causal": true - }, - "name": "qwen3.5" - }, - { - "type": "attention_mha", - "args": { - "batch_size": 4, - "seq_len_q": 1, - "seq_len_kv": 1536, - "num_heads": 16, - "num_heads_kv": 1, - "head_dim": 256, - "dtype_q": "fp8", - "dtype_k": "fp8", - "dtype_v": "fp8", - "dtype_o": "bf16", - "causal": true - }, - "name": "qwen3.5" - }, - { - "type": "attention_mha", - "args": { - "batch_size": 8, - "seq_len_q": 1024, - "seq_len_kv": 1024, - "num_heads": 16, - "num_heads_kv": 1, - "head_dim": 256, - "dtype_q": "fp8", - "dtype_k": "fp8", - "dtype_v": "fp8", - "dtype_o": "bf16", - "causal": true - }, - "name": "qwen3.5" - }, - { - "type": "attention_mha", - "args": { - "batch_size": 8, - "seq_len_q": 1, - "seq_len_kv": 1536, - "num_heads": 16, - "num_heads_kv": 1, - "head_dim": 256, - "dtype_q": "fp8", - "dtype_k": "fp8", - "dtype_v": "fp8", - "dtype_o": "bf16", - "causal": true - }, - "name": "qwen3.5" - }, - { - "type": "attention_mha", - "args": { - "batch_size": 16, - "seq_len_q": 1024, - "seq_len_kv": 1024, - "num_heads": 16, - "num_heads_kv": 1, - "head_dim": 256, - "dtype_q": "fp8", - "dtype_k": "fp8", - "dtype_v": "fp8", - "dtype_o": "bf16", - "causal": true - }, - "name": "qwen3.5" - }, - { - "type": "attention_mha", - "args": { - "batch_size": 16, - "seq_len_q": 1, - "seq_len_kv": 1536, - "num_heads": 16, - "num_heads_kv": 1, - "head_dim": 256, - "dtype_q": "fp8", - "dtype_k": "fp8", - "dtype_v": "fp8", - "dtype_o": "bf16", - "causal": true - }, - "name": "qwen3.5" - }, - { - "type": "attention_mha", - "args": { - "batch_size": 32, - "seq_len_q": 1024, - "seq_len_kv": 1024, - "num_heads": 16, - "num_heads_kv": 1, - "head_dim": 256, - "dtype_q": "fp8", - "dtype_k": "fp8", - "dtype_v": "fp8", - "dtype_o": "bf16", - "causal": true - }, - "name": "qwen3.5" - }, - { - "type": "attention_mha", - "args": { - "batch_size": 32, - "seq_len_q": 1, - "seq_len_kv": 1536, - "num_heads": 16, - "num_heads_kv": 1, - "head_dim": 256, - "dtype_q": "fp8", - "dtype_k": "fp8", - "dtype_v": "fp8", - "dtype_o": "bf16", - "causal": true - }, - "name": "qwen3.5" - }, - { - "type": "attention_mha", - "args": { - "batch_size": 64, - "seq_len_q": 1024, - "seq_len_kv": 1024, - "num_heads": 16, - "num_heads_kv": 1, - "head_dim": 256, - "dtype_q": "fp8", - "dtype_k": "fp8", - "dtype_v": "fp8", - "dtype_o": "bf16", - "causal": true - }, - "name": "qwen3.5" - }, - { - "type": "attention_mha", - "args": { - "batch_size": 64, - "seq_len_q": 1, - "seq_len_kv": 1536, - "num_heads": 16, - "num_heads_kv": 1, - "head_dim": 256, - "dtype_q": "fp8", - "dtype_k": "fp8", - "dtype_v": "fp8", - "dtype_o": "bf16", - "causal": true - }, - "name": "qwen3.5" - }, - { - "type": "attention_mha", - "args": { - "batch_size": 64, - "seq_len_q": 8192, - "seq_len_kv": 8192, - "num_heads": 16, - "num_heads_kv": 1, - "head_dim": 256, - "dtype_q": "fp8", - "dtype_k": "fp8", - "dtype_v": "fp8", - "dtype_o": "bf16", - "causal": true - }, - "name": "qwen3.5" - }, - { - "type": "attention_mha", - "args": { - "batch_size": 64, - "seq_len_q": 1, - "seq_len_kv": 8704, - "num_heads": 16, - "num_heads_kv": 1, - "head_dim": 256, - "dtype_q": "fp8", - "dtype_k": "fp8", - "dtype_v": "fp8", - "dtype_o": "bf16", - "causal": true - }, - "name": "qwen3.5" - }, - { - "type": "attention_mha", - "args": { - "batch_size": 128, - "seq_len_q": 8192, - "seq_len_kv": 8192, - "num_heads": 16, - "num_heads_kv": 1, - "head_dim": 256, - "dtype_q": "fp8", - "dtype_k": "fp8", - "dtype_v": "fp8", - "dtype_o": "bf16", - "causal": true - }, - "name": "qwen3.5" - }, - { - "type": "attention_mha", - "args": { - "batch_size": 128, - "seq_len_q": 1, - "seq_len_kv": 8704, - "num_heads": 16, - "num_heads_kv": 1, - "head_dim": 256, - "dtype_q": "fp8", - "dtype_k": "fp8", - "dtype_v": "fp8", - "dtype_o": "bf16", - "causal": true - }, - "name": "qwen3.5" - }, - { - "type": "attention_mha", - "args": { - "batch_size": 256, - "seq_len_q": 8192, - "seq_len_kv": 8192, - "num_heads": 16, - "num_heads_kv": 1, - "head_dim": 256, - "dtype_q": "fp8", - "dtype_k": "fp8", - "dtype_v": "fp8", - "dtype_o": "bf16", - "causal": true - }, - "name": "qwen3.5" - }, - { - "type": "attention_mha", - "args": { - "batch_size": 256, - "seq_len_q": 1, - "seq_len_kv": 8704, - "num_heads": 16, - "num_heads_kv": 1, - "head_dim": 256, - "dtype_q": "fp8", - "dtype_k": "fp8", - "dtype_v": "fp8", - "dtype_o": "bf16", - "causal": true - }, - "name": "qwen3.5" - }, - { - "type": "attention_mha", - "args": { - "batch_size": 128, - "seq_len_q": 1, - "seq_len_kv": 8704, - "num_heads": 4, - "num_heads_kv": 1, - "head_dim": 256, - "dtype_q": "fp8", - "dtype_k": "fp8", - "dtype_v": "fp8", - "dtype_o": "bf16", - "causal": true - }, - "name": "qwen3.5" - }, - { - "type": "attention_mha", - "args": { - "batch_size": 256, - "seq_len_q": 8192, - "seq_len_kv": 8192, - "num_heads": 4, - "num_heads_kv": 1, - "head_dim": 256, - "dtype_q": "fp8", - "dtype_k": "fp8", - "dtype_v": "fp8", - "dtype_o": "bf16", - "causal": true - }, - "name": "qwen3.5" - }, - { - "type": "attention_mha", - "args": { - "batch_size": 256, - "seq_len_q": 1, - "seq_len_kv": 8704, - "num_heads": 4, - "num_heads_kv": 1, - "head_dim": 256, - "dtype_q": "fp8", - "dtype_k": "fp8", - "dtype_v": "fp8", - "dtype_o": "bf16", - "causal": true - }, - "name": "qwen3.5" - }, - { - "type": "attention_mha", - "args": { - "batch_size": 256, - "seq_len_q": 2, - "seq_len_kv": 8704, - "num_heads": 4, - "num_heads_kv": 1, - "head_dim": 256, - "dtype_q": "fp8", - "dtype_k": "fp8", - "dtype_v": "fp8", - "dtype_o": "bf16", - "causal": true - }, - "name": "qwen3.5" - }, - { - "type": "attention_mha", - "args": { - "batch_size": 256, - "seq_len_q": 1024, - "seq_len_kv": 1024, - "num_heads": 16, - "num_heads_kv": 1, - "head_dim": 256, - "dtype_q": "nvfp4", - "dtype_k": "fp8", - "dtype_v": "fp8", - "dtype_o": "bf16", - "causal": true - }, - "name": "qwen3.5" - }, - { - "type": "attention_mha", - "args": { - "batch_size": 256, - "seq_len_q": 1, - "seq_len_kv": 1536, - "num_heads": 16, - "num_heads_kv": 1, - "head_dim": 256, - "dtype_q": "nvfp4", - "dtype_k": "fp8", - "dtype_v": "fp8", - "dtype_o": "bf16", - "causal": true - }, - "name": "qwen3.5" - }, - { - "type": "attention_mha", - "args": { - "batch_size": 256, - "seq_len_q": 8192, - "seq_len_kv": 8192, - "num_heads": 16, - "num_heads_kv": 1, - "head_dim": 256, - "dtype_q": "nvfp4", - "dtype_k": "fp8", - "dtype_v": "fp8", - "dtype_o": "bf16", - "causal": true - }, - "name": "qwen3.5" - }, - { - "type": "attention_mha", - "args": { - "batch_size": 256, - "seq_len_q": 1, - "seq_len_kv": 8704, - "num_heads": 16, - "num_heads_kv": 1, - "head_dim": 256, - "dtype_q": "nvfp4", - "dtype_k": "fp8", - "dtype_v": "fp8", - "dtype_o": "bf16", - "causal": true - }, - "name": "qwen3.5" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 4, - "seq_len_q": 1024, - "seq_len_kv": 1024, - "num_heads": 16, - "head_dim_qk_nope": 192, - "head_dim_qk_rope": 64, - "head_dim_v": 256, - "kv_lora_rank": 512, - "dtype_q": "fp8", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "glm5" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 4, - "seq_len_q": 2, - "seq_len_kv": 1536, - "num_heads": 16, - "head_dim_qk_nope": 192, - "head_dim_qk_rope": 64, - "head_dim_v": 256, - "kv_lora_rank": 512, - "dtype_q": "fp8", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "glm5" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 8, - "seq_len_q": 1024, - "seq_len_kv": 1024, - "num_heads": 16, - "head_dim_qk_nope": 192, - "head_dim_qk_rope": 64, - "head_dim_v": 256, - "kv_lora_rank": 512, - "dtype_q": "fp8", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "glm5" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 8, - "seq_len_q": 2, - "seq_len_kv": 1536, - "num_heads": 16, - "head_dim_qk_nope": 192, - "head_dim_qk_rope": 64, - "head_dim_v": 256, - "kv_lora_rank": 512, - "dtype_q": "fp8", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "glm5" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 16, - "seq_len_q": 1024, - "seq_len_kv": 1024, - "num_heads": 16, - "head_dim_qk_nope": 192, - "head_dim_qk_rope": 64, - "head_dim_v": 256, - "kv_lora_rank": 512, - "dtype_q": "fp8", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "glm5" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 16, - "seq_len_q": 2, - "seq_len_kv": 1536, - "num_heads": 16, - "head_dim_qk_nope": 192, - "head_dim_qk_rope": 64, - "head_dim_v": 256, - "kv_lora_rank": 512, - "dtype_q": "fp8", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "glm5" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 32, - "seq_len_q": 1024, - "seq_len_kv": 1024, - "num_heads": 16, - "head_dim_qk_nope": 192, - "head_dim_qk_rope": 64, - "head_dim_v": 256, - "kv_lora_rank": 512, - "dtype_q": "fp8", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "glm5" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 32, - "seq_len_q": 2, - "seq_len_kv": 1536, - "num_heads": 16, - "head_dim_qk_nope": 192, - "head_dim_qk_rope": 64, - "head_dim_v": 256, - "kv_lora_rank": 512, - "dtype_q": "fp8", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "glm5" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 64, - "seq_len_q": 1024, - "seq_len_kv": 1024, - "num_heads": 16, - "head_dim_qk_nope": 192, - "head_dim_qk_rope": 64, - "head_dim_v": 256, - "kv_lora_rank": 512, - "dtype_q": "fp8", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "glm5" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 64, - "seq_len_q": 2, - "seq_len_kv": 1536, - "num_heads": 16, - "head_dim_qk_nope": 192, - "head_dim_qk_rope": 64, - "head_dim_v": 256, - "kv_lora_rank": 512, - "dtype_q": "fp8", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "glm5" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 128, - "seq_len_q": 1024, - "seq_len_kv": 1024, - "num_heads": 16, - "head_dim_qk_nope": 192, - "head_dim_qk_rope": 64, - "head_dim_v": 256, - "kv_lora_rank": 512, - "dtype_q": "fp8", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "glm5" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 128, - "seq_len_q": 2, - "seq_len_kv": 1536, - "num_heads": 16, - "head_dim_qk_nope": 192, - "head_dim_qk_rope": 64, - "head_dim_v": 256, - "kv_lora_rank": 512, - "dtype_q": "fp8", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "glm5" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 4, - "seq_len_q": 8192, - "seq_len_kv": 8192, - "num_heads": 16, - "head_dim_qk_nope": 192, - "head_dim_qk_rope": 64, - "head_dim_v": 256, - "kv_lora_rank": 512, - "dtype_q": "fp8", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "glm5" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 4, - "seq_len_q": 2, - "seq_len_kv": 8704, - "num_heads": 16, - "head_dim_qk_nope": 192, - "head_dim_qk_rope": 64, - "head_dim_v": 256, - "kv_lora_rank": 512, - "dtype_q": "fp8", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "glm5" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 8, - "seq_len_q": 8192, - "seq_len_kv": 8192, - "num_heads": 16, - "head_dim_qk_nope": 192, - "head_dim_qk_rope": 64, - "head_dim_v": 256, - "kv_lora_rank": 512, - "dtype_q": "fp8", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "glm5" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 8, - "seq_len_q": 2, - "seq_len_kv": 8704, - "num_heads": 16, - "head_dim_qk_nope": 192, - "head_dim_qk_rope": 64, - "head_dim_v": 256, - "kv_lora_rank": 512, - "dtype_q": "fp8", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "glm5" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 16, - "seq_len_q": 8192, - "seq_len_kv": 8192, - "num_heads": 16, - "head_dim_qk_nope": 192, - "head_dim_qk_rope": 64, - "head_dim_v": 256, - "kv_lora_rank": 512, - "dtype_q": "fp8", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "glm5" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 16, - "seq_len_q": 2, - "seq_len_kv": 8704, - "num_heads": 16, - "head_dim_qk_nope": 192, - "head_dim_qk_rope": 64, - "head_dim_v": 256, - "kv_lora_rank": 512, - "dtype_q": "fp8", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "glm5" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 32, - "seq_len_q": 8192, - "seq_len_kv": 8192, - "num_heads": 16, - "head_dim_qk_nope": 192, - "head_dim_qk_rope": 64, - "head_dim_v": 256, - "kv_lora_rank": 512, - "dtype_q": "fp8", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "glm5" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 32, - "seq_len_q": 2, - "seq_len_kv": 8704, - "num_heads": 16, - "head_dim_qk_nope": 192, - "head_dim_qk_rope": 64, - "head_dim_v": 256, - "kv_lora_rank": 512, - "dtype_q": "fp8", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "glm5" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 64, - "seq_len_q": 8192, - "seq_len_kv": 8192, - "num_heads": 16, - "head_dim_qk_nope": 192, - "head_dim_qk_rope": 64, - "head_dim_v": 256, - "kv_lora_rank": 512, - "dtype_q": "fp8", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "glm5" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 64, - "seq_len_q": 2, - "seq_len_kv": 8704, - "num_heads": 16, - "head_dim_qk_nope": 192, - "head_dim_qk_rope": 64, - "head_dim_v": 256, - "kv_lora_rank": 512, - "dtype_q": "fp8", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "glm5" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 128, - "seq_len_q": 8192, - "seq_len_kv": 8192, - "num_heads": 16, - "head_dim_qk_nope": 192, - "head_dim_qk_rope": 64, - "head_dim_v": 256, - "kv_lora_rank": 512, - "dtype_q": "fp8", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "glm5" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 128, - "seq_len_q": 2, - "seq_len_kv": 8704, - "num_heads": 16, - "head_dim_qk_nope": 192, - "head_dim_qk_rope": 64, - "head_dim_v": 256, - "kv_lora_rank": 512, - "dtype_q": "fp8", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "glm5" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 4, - "seq_len_q": 1, - "seq_len_kv": 1536, - "num_heads": 16, - "head_dim_qk_nope": 192, - "head_dim_qk_rope": 64, - "head_dim_v": 256, - "kv_lora_rank": 512, - "dtype_q": "fp8", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "glm5" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 8, - "seq_len_q": 1, - "seq_len_kv": 1536, - "num_heads": 16, - "head_dim_qk_nope": 192, - "head_dim_qk_rope": 64, - "head_dim_v": 256, - "kv_lora_rank": 512, - "dtype_q": "fp8", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "glm5" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 16, - "seq_len_q": 1, - "seq_len_kv": 1536, - "num_heads": 16, - "head_dim_qk_nope": 192, - "head_dim_qk_rope": 64, - "head_dim_v": 256, - "kv_lora_rank": 512, - "dtype_q": "fp8", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "glm5" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 32, - "seq_len_q": 1, - "seq_len_kv": 1536, - "num_heads": 16, - "head_dim_qk_nope": 192, - "head_dim_qk_rope": 64, - "head_dim_v": 256, - "kv_lora_rank": 512, - "dtype_q": "fp8", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "glm5" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 64, - "seq_len_q": 1, - "seq_len_kv": 1536, - "num_heads": 16, - "head_dim_qk_nope": 192, - "head_dim_qk_rope": 64, - "head_dim_v": 256, - "kv_lora_rank": 512, - "dtype_q": "fp8", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "glm5" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 128, - "seq_len_q": 1, - "seq_len_kv": 1536, - "num_heads": 16, - "head_dim_qk_nope": 192, - "head_dim_qk_rope": 64, - "head_dim_v": 256, - "kv_lora_rank": 512, - "dtype_q": "fp8", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "glm5" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 256, - "seq_len_q": 1024, - "seq_len_kv": 1024, - "num_heads": 16, - "head_dim_qk_nope": 192, - "head_dim_qk_rope": 64, - "head_dim_v": 256, - "kv_lora_rank": 512, - "dtype_q": "fp8", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "glm5" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 256, - "seq_len_q": 1, - "seq_len_kv": 1536, - "num_heads": 16, - "head_dim_qk_nope": 192, - "head_dim_qk_rope": 64, - "head_dim_v": 256, - "kv_lora_rank": 512, - "dtype_q": "fp8", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "glm5" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 4, - "seq_len_q": 1, - "seq_len_kv": 8704, - "num_heads": 16, - "head_dim_qk_nope": 192, - "head_dim_qk_rope": 64, - "head_dim_v": 256, - "kv_lora_rank": 512, - "dtype_q": "fp8", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "glm5" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 8, - "seq_len_q": 1, - "seq_len_kv": 8704, - "num_heads": 16, - "head_dim_qk_nope": 192, - "head_dim_qk_rope": 64, - "head_dim_v": 256, - "kv_lora_rank": 512, - "dtype_q": "fp8", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "glm5" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 16, - "seq_len_q": 1, - "seq_len_kv": 8704, - "num_heads": 16, - "head_dim_qk_nope": 192, - "head_dim_qk_rope": 64, - "head_dim_v": 256, - "kv_lora_rank": 512, - "dtype_q": "fp8", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "glm5" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 32, - "seq_len_q": 1, - "seq_len_kv": 8704, - "num_heads": 16, - "head_dim_qk_nope": 192, - "head_dim_qk_rope": 64, - "head_dim_v": 256, - "kv_lora_rank": 512, - "dtype_q": "fp8", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "glm5" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 64, - "seq_len_q": 1, - "seq_len_kv": 8704, - "num_heads": 16, - "head_dim_qk_nope": 192, - "head_dim_qk_rope": 64, - "head_dim_v": 256, - "kv_lora_rank": 512, - "dtype_q": "fp8", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "glm5" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 128, - "seq_len_q": 1, - "seq_len_kv": 8704, - "num_heads": 16, - "head_dim_qk_nope": 192, - "head_dim_qk_rope": 64, - "head_dim_v": 256, - "kv_lora_rank": 512, - "dtype_q": "fp8", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "glm5" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 256, - "seq_len_q": 8192, - "seq_len_kv": 8192, - "num_heads": 16, - "head_dim_qk_nope": 192, - "head_dim_qk_rope": 64, - "head_dim_v": 256, - "kv_lora_rank": 512, - "dtype_q": "fp8", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "glm5" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 256, - "seq_len_q": 1, - "seq_len_kv": 8704, - "num_heads": 16, - "head_dim_qk_nope": 192, - "head_dim_qk_rope": 64, - "head_dim_v": 256, - "kv_lora_rank": 512, - "dtype_q": "fp8", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "glm5" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 4, - "seq_len_q": 1024, - "seq_len_kv": 1024, - "num_heads": 32, - "head_dim_qk_nope": 192, - "head_dim_qk_rope": 64, - "head_dim_v": 256, - "kv_lora_rank": 512, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "glm5.1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 4, - "seq_len_q": 1, - "seq_len_kv": 1536, - "num_heads": 32, - "head_dim_qk_nope": 192, - "head_dim_qk_rope": 64, - "head_dim_v": 256, - "kv_lora_rank": 512, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "glm5.1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 8, - "seq_len_q": 1024, - "seq_len_kv": 1024, - "num_heads": 32, - "head_dim_qk_nope": 192, - "head_dim_qk_rope": 64, - "head_dim_v": 256, - "kv_lora_rank": 512, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "glm5.1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 8, - "seq_len_q": 1, - "seq_len_kv": 1536, - "num_heads": 32, - "head_dim_qk_nope": 192, - "head_dim_qk_rope": 64, - "head_dim_v": 256, - "kv_lora_rank": 512, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "glm5.1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 16, - "seq_len_q": 1024, - "seq_len_kv": 1024, - "num_heads": 32, - "head_dim_qk_nope": 192, - "head_dim_qk_rope": 64, - "head_dim_v": 256, - "kv_lora_rank": 512, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "glm5.1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 16, - "seq_len_q": 1, - "seq_len_kv": 1536, - "num_heads": 32, - "head_dim_qk_nope": 192, - "head_dim_qk_rope": 64, - "head_dim_v": 256, - "kv_lora_rank": 512, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "glm5.1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 32, - "seq_len_q": 1024, - "seq_len_kv": 1024, - "num_heads": 32, - "head_dim_qk_nope": 192, - "head_dim_qk_rope": 64, - "head_dim_v": 256, - "kv_lora_rank": 512, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "glm5.1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 32, - "seq_len_q": 1, - "seq_len_kv": 1536, - "num_heads": 32, - "head_dim_qk_nope": 192, - "head_dim_qk_rope": 64, - "head_dim_v": 256, - "kv_lora_rank": 512, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "glm5.1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 64, - "seq_len_q": 1024, - "seq_len_kv": 1024, - "num_heads": 32, - "head_dim_qk_nope": 192, - "head_dim_qk_rope": 64, - "head_dim_v": 256, - "kv_lora_rank": 512, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "glm5.1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 64, - "seq_len_q": 1, - "seq_len_kv": 1536, - "num_heads": 32, - "head_dim_qk_nope": 192, - "head_dim_qk_rope": 64, - "head_dim_v": 256, - "kv_lora_rank": 512, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "glm5.1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 128, - "seq_len_q": 1024, - "seq_len_kv": 1024, - "num_heads": 32, - "head_dim_qk_nope": 192, - "head_dim_qk_rope": 64, - "head_dim_v": 256, - "kv_lora_rank": 512, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "glm5.1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 128, - "seq_len_q": 1, - "seq_len_kv": 1536, - "num_heads": 32, - "head_dim_qk_nope": 192, - "head_dim_qk_rope": 64, - "head_dim_v": 256, - "kv_lora_rank": 512, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "glm5.1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 256, - "seq_len_q": 1024, - "seq_len_kv": 1024, - "num_heads": 32, - "head_dim_qk_nope": 192, - "head_dim_qk_rope": 64, - "head_dim_v": 256, - "kv_lora_rank": 512, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "glm5.1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 256, - "seq_len_q": 1, - "seq_len_kv": 1536, - "num_heads": 32, - "head_dim_qk_nope": 192, - "head_dim_qk_rope": 64, - "head_dim_v": 256, - "kv_lora_rank": 512, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "glm5.1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 4, - "seq_len_q": 8192, - "seq_len_kv": 8192, - "num_heads": 32, - "head_dim_qk_nope": 192, - "head_dim_qk_rope": 64, - "head_dim_v": 256, - "kv_lora_rank": 512, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "glm5.1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 4, - "seq_len_q": 1, - "seq_len_kv": 8704, - "num_heads": 32, - "head_dim_qk_nope": 192, - "head_dim_qk_rope": 64, - "head_dim_v": 256, - "kv_lora_rank": 512, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "glm5.1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 8, - "seq_len_q": 8192, - "seq_len_kv": 8192, - "num_heads": 32, - "head_dim_qk_nope": 192, - "head_dim_qk_rope": 64, - "head_dim_v": 256, - "kv_lora_rank": 512, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "glm5.1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 8, - "seq_len_q": 1, - "seq_len_kv": 8704, - "num_heads": 32, - "head_dim_qk_nope": 192, - "head_dim_qk_rope": 64, - "head_dim_v": 256, - "kv_lora_rank": 512, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "glm5.1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 16, - "seq_len_q": 8192, - "seq_len_kv": 8192, - "num_heads": 32, - "head_dim_qk_nope": 192, - "head_dim_qk_rope": 64, - "head_dim_v": 256, - "kv_lora_rank": 512, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "glm5.1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 16, - "seq_len_q": 1, - "seq_len_kv": 8704, - "num_heads": 32, - "head_dim_qk_nope": 192, - "head_dim_qk_rope": 64, - "head_dim_v": 256, - "kv_lora_rank": 512, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "glm5.1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 32, - "seq_len_q": 8192, - "seq_len_kv": 8192, - "num_heads": 32, - "head_dim_qk_nope": 192, - "head_dim_qk_rope": 64, - "head_dim_v": 256, - "kv_lora_rank": 512, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "glm5.1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 32, - "seq_len_q": 1, - "seq_len_kv": 8704, - "num_heads": 32, - "head_dim_qk_nope": 192, - "head_dim_qk_rope": 64, - "head_dim_v": 256, - "kv_lora_rank": 512, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "glm5.1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 64, - "seq_len_q": 8192, - "seq_len_kv": 8192, - "num_heads": 32, - "head_dim_qk_nope": 192, - "head_dim_qk_rope": 64, - "head_dim_v": 256, - "kv_lora_rank": 512, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "glm5.1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 64, - "seq_len_q": 1, - "seq_len_kv": 8704, - "num_heads": 32, - "head_dim_qk_nope": 192, - "head_dim_qk_rope": 64, - "head_dim_v": 256, - "kv_lora_rank": 512, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "glm5.1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 128, - "seq_len_q": 8192, - "seq_len_kv": 8192, - "num_heads": 32, - "head_dim_qk_nope": 192, - "head_dim_qk_rope": 64, - "head_dim_v": 256, - "kv_lora_rank": 512, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "glm5.1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 128, - "seq_len_q": 1, - "seq_len_kv": 8704, - "num_heads": 32, - "head_dim_qk_nope": 192, - "head_dim_qk_rope": 64, - "head_dim_v": 256, - "kv_lora_rank": 512, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "glm5.1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 256, - "seq_len_q": 8192, - "seq_len_kv": 8192, - "num_heads": 32, - "head_dim_qk_nope": 192, - "head_dim_qk_rope": 64, - "head_dim_v": 256, - "kv_lora_rank": 512, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "glm5.1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 256, - "seq_len_q": 1, - "seq_len_kv": 8704, - "num_heads": 32, - "head_dim_qk_nope": 192, - "head_dim_qk_rope": 64, - "head_dim_v": 256, - "kv_lora_rank": 512, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "glm5.1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 8, - "seq_len_q": 1024, - "seq_len_kv": 1024, - "num_heads": 8, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "kimik2.5" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 8, - "seq_len_q": 1, - "seq_len_kv": 1536, - "num_heads": 8, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "kimik2.5" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 16, - "seq_len_q": 1024, - "seq_len_kv": 1024, - "num_heads": 8, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "kimik2.5" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 16, - "seq_len_q": 1, - "seq_len_kv": 1536, - "num_heads": 8, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "kimik2.5" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 32, - "seq_len_q": 1024, - "seq_len_kv": 1024, - "num_heads": 8, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "kimik2.5" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 32, - "seq_len_q": 1, - "seq_len_kv": 1536, - "num_heads": 8, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "kimik2.5" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 64, - "seq_len_q": 1024, - "seq_len_kv": 1024, - "num_heads": 8, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "kimik2.5" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 64, - "seq_len_q": 1, - "seq_len_kv": 1536, - "num_heads": 8, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "kimik2.5" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 8, - "seq_len_q": 8192, - "seq_len_kv": 8192, - "num_heads": 8, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "kimik2.5" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 8, - "seq_len_q": 1, - "seq_len_kv": 8704, - "num_heads": 8, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "kimik2.5" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 16, - "seq_len_q": 8192, - "seq_len_kv": 8192, - "num_heads": 8, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "kimik2.5" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 16, - "seq_len_q": 1, - "seq_len_kv": 8704, - "num_heads": 8, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "kimik2.5" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 32, - "seq_len_q": 8192, - "seq_len_kv": 8192, - "num_heads": 8, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "kimik2.5" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 32, - "seq_len_q": 1, - "seq_len_kv": 8704, - "num_heads": 8, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "kimik2.5" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 64, - "seq_len_q": 8192, - "seq_len_kv": 8192, - "num_heads": 8, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "kimik2.5" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 64, - "seq_len_q": 1, - "seq_len_kv": 8704, - "num_heads": 8, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "kimik2.5" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 128, - "seq_len_q": 1024, - "seq_len_kv": 1024, - "num_heads": 8, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "kimik2.5" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 128, - "seq_len_q": 1, - "seq_len_kv": 1536, - "num_heads": 8, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "kimik2.5" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 128, - "seq_len_q": 8192, - "seq_len_kv": 8192, - "num_heads": 8, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "kimik2.5" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 128, - "seq_len_q": 1, - "seq_len_kv": 8704, - "num_heads": 8, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "kimik2.5" - }, - { - "type": "attention_mha", - "args": { - "batch_size": 2, - "seq_len_q": 1024, - "seq_len_kv": 1024, - "num_heads": 24, - "num_heads_kv": 4, - "head_dim": 128, - "dtype_q": "fp8", - "dtype_k": "fp8", - "dtype_v": "fp8", - "dtype_o": "bf16", - "causal": true - }, - "name": "minimaxm2.5" - }, - { - "type": "attention_mha", - "args": { - "batch_size": 2, - "seq_len_q": 1, - "seq_len_kv": 1536, - "num_heads": 24, - "num_heads_kv": 4, - "head_dim": 128, - "dtype_q": "fp8", - "dtype_k": "fp8", - "dtype_v": "fp8", - "dtype_o": "bf16", - "causal": true - }, - "name": "minimaxm2.5" - }, - { - "type": "attention_mha", - "args": { - "batch_size": 4, - "seq_len_q": 1024, - "seq_len_kv": 1024, - "num_heads": 24, - "num_heads_kv": 4, - "head_dim": 128, - "dtype_q": "fp8", - "dtype_k": "fp8", - "dtype_v": "fp8", - "dtype_o": "bf16", - "causal": true - }, - "name": "minimaxm2.5" - }, - { - "type": "attention_mha", - "args": { - "batch_size": 4, - "seq_len_q": 1, - "seq_len_kv": 1536, - "num_heads": 24, - "num_heads_kv": 4, - "head_dim": 128, - "dtype_q": "fp8", - "dtype_k": "fp8", - "dtype_v": "fp8", - "dtype_o": "bf16", - "causal": true - }, - "name": "minimaxm2.5" - }, - { - "type": "attention_mha", - "args": { - "batch_size": 8, - "seq_len_q": 1024, - "seq_len_kv": 1024, - "num_heads": 24, - "num_heads_kv": 4, - "head_dim": 128, - "dtype_q": "fp8", - "dtype_k": "fp8", - "dtype_v": "fp8", - "dtype_o": "bf16", - "causal": true - }, - "name": "minimaxm2.5" - }, - { - "type": "attention_mha", - "args": { - "batch_size": 8, - "seq_len_q": 1, - "seq_len_kv": 1536, - "num_heads": 24, - "num_heads_kv": 4, - "head_dim": 128, - "dtype_q": "fp8", - "dtype_k": "fp8", - "dtype_v": "fp8", - "dtype_o": "bf16", - "causal": true - }, - "name": "minimaxm2.5" - }, - { - "type": "attention_mha", - "args": { - "batch_size": 16, - "seq_len_q": 1024, - "seq_len_kv": 1024, - "num_heads": 24, - "num_heads_kv": 4, - "head_dim": 128, - "dtype_q": "fp8", - "dtype_k": "fp8", - "dtype_v": "fp8", - "dtype_o": "bf16", - "causal": true - }, - "name": "minimaxm2.5" - }, - { - "type": "attention_mha", - "args": { - "batch_size": 16, - "seq_len_q": 1, - "seq_len_kv": 1536, - "num_heads": 24, - "num_heads_kv": 4, - "head_dim": 128, - "dtype_q": "fp8", - "dtype_k": "fp8", - "dtype_v": "fp8", - "dtype_o": "bf16", - "causal": true - }, - "name": "minimaxm2.5" - }, - { - "type": "attention_mha", - "args": { - "batch_size": 32, - "seq_len_q": 1024, - "seq_len_kv": 1024, - "num_heads": 24, - "num_heads_kv": 4, - "head_dim": 128, - "dtype_q": "fp8", - "dtype_k": "fp8", - "dtype_v": "fp8", - "dtype_o": "bf16", - "causal": true - }, - "name": "minimaxm2.5" - }, - { - "type": "attention_mha", - "args": { - "batch_size": 32, - "seq_len_q": 1, - "seq_len_kv": 1536, - "num_heads": 24, - "num_heads_kv": 4, - "head_dim": 128, - "dtype_q": "fp8", - "dtype_k": "fp8", - "dtype_v": "fp8", - "dtype_o": "bf16", - "causal": true - }, - "name": "minimaxm2.5" - }, - { - "type": "attention_mha", - "args": { - "batch_size": 64, - "seq_len_q": 1024, - "seq_len_kv": 1024, - "num_heads": 24, - "num_heads_kv": 4, - "head_dim": 128, - "dtype_q": "fp8", - "dtype_k": "fp8", - "dtype_v": "fp8", - "dtype_o": "bf16", - "causal": true - }, - "name": "minimaxm2.5" - }, - { - "type": "attention_mha", - "args": { - "batch_size": 64, - "seq_len_q": 1, - "seq_len_kv": 1536, - "num_heads": 24, - "num_heads_kv": 4, - "head_dim": 128, - "dtype_q": "fp8", - "dtype_k": "fp8", - "dtype_v": "fp8", - "dtype_o": "bf16", - "causal": true - }, - "name": "minimaxm2.5" - }, - { - "type": "attention_mha", - "args": { - "batch_size": 128, - "seq_len_q": 1024, - "seq_len_kv": 1024, - "num_heads": 24, - "num_heads_kv": 4, - "head_dim": 128, - "dtype_q": "fp8", - "dtype_k": "fp8", - "dtype_v": "fp8", - "dtype_o": "bf16", - "causal": true - }, - "name": "minimaxm2.5" - }, - { - "type": "attention_mha", - "args": { - "batch_size": 128, - "seq_len_q": 1, - "seq_len_kv": 1536, - "num_heads": 24, - "num_heads_kv": 4, - "head_dim": 128, - "dtype_q": "fp8", - "dtype_k": "fp8", - "dtype_v": "fp8", - "dtype_o": "bf16", - "causal": true - }, - "name": "minimaxm2.5" - }, - { - "type": "attention_mha", - "args": { - "batch_size": 256, - "seq_len_q": 1024, - "seq_len_kv": 1024, - "num_heads": 24, - "num_heads_kv": 4, - "head_dim": 128, - "dtype_q": "fp8", - "dtype_k": "fp8", - "dtype_v": "fp8", - "dtype_o": "bf16", - "causal": true - }, - "name": "minimaxm2.5" - }, - { - "type": "attention_mha", - "args": { - "batch_size": 256, - "seq_len_q": 1, - "seq_len_kv": 1536, - "num_heads": 24, - "num_heads_kv": 4, - "head_dim": 128, - "dtype_q": "fp8", - "dtype_k": "fp8", - "dtype_v": "fp8", - "dtype_o": "bf16", - "causal": true - }, - "name": "minimaxm2.5" - }, - { - "type": "attention_mha", - "args": { - "batch_size": 2, - "seq_len_q": 1024, - "seq_len_kv": 1024, - "num_heads": 6, - "num_heads_kv": 1, - "head_dim": 128, - "dtype_q": "fp8", - "dtype_k": "fp8", - "dtype_v": "fp8", - "dtype_o": "bf16", - "causal": true - }, - "name": "minimaxm2.5" - }, - { - "type": "attention_mha", - "args": { - "batch_size": 2, - "seq_len_q": 1, - "seq_len_kv": 1536, - "num_heads": 6, - "num_heads_kv": 1, - "head_dim": 128, - "dtype_q": "fp8", - "dtype_k": "fp8", - "dtype_v": "fp8", - "dtype_o": "bf16", - "causal": true - }, - "name": "minimaxm2.5" - }, - { - "type": "attention_mha", - "args": { - "batch_size": 2, - "seq_len_q": 8192, - "seq_len_kv": 8192, - "num_heads": 24, - "num_heads_kv": 4, - "head_dim": 128, - "dtype_q": "fp8", - "dtype_k": "fp8", - "dtype_v": "fp8", - "dtype_o": "bf16", - "causal": true - }, - "name": "minimaxm2.5" - }, - { - "type": "attention_mha", - "args": { - "batch_size": 2, - "seq_len_q": 1, - "seq_len_kv": 8704, - "num_heads": 24, - "num_heads_kv": 4, - "head_dim": 128, - "dtype_q": "fp8", - "dtype_k": "fp8", - "dtype_v": "fp8", - "dtype_o": "bf16", - "causal": true - }, - "name": "minimaxm2.5" - }, - { - "type": "attention_mha", - "args": { - "batch_size": 2, - "seq_len_q": 8192, - "seq_len_kv": 8192, - "num_heads": 6, - "num_heads_kv": 1, - "head_dim": 128, - "dtype_q": "fp8", - "dtype_k": "fp8", - "dtype_v": "fp8", - "dtype_o": "bf16", - "causal": true - }, - "name": "minimaxm2.5" - }, - { - "type": "attention_mha", - "args": { - "batch_size": 2, - "seq_len_q": 1, - "seq_len_kv": 8704, - "num_heads": 6, - "num_heads_kv": 1, - "head_dim": 128, - "dtype_q": "fp8", - "dtype_k": "fp8", - "dtype_v": "fp8", - "dtype_o": "bf16", - "causal": true - }, - "name": "minimaxm2.5" - }, - { - "type": "attention_mha", - "args": { - "batch_size": 32, - "seq_len_q": 1024, - "seq_len_kv": 1024, - "num_heads": 48, - "num_heads_kv": 8, - "head_dim": 128, - "dtype_q": "nvfp4", - "dtype_k": "fp8", - "dtype_v": "fp8", - "dtype_o": "bf16", - "causal": true - }, - "name": "minimaxm2.5" - }, - { - "type": "attention_mha", - "args": { - "batch_size": 32, - "seq_len_q": 1, - "seq_len_kv": 1536, - "num_heads": 48, - "num_heads_kv": 8, - "head_dim": 128, - "dtype_q": "nvfp4", - "dtype_k": "fp8", - "dtype_v": "fp8", - "dtype_o": "bf16", - "causal": true - }, - "name": "minimaxm2.5" - }, - { - "type": "attention_mha", - "args": { - "batch_size": 64, - "seq_len_q": 1024, - "seq_len_kv": 1024, - "num_heads": 48, - "num_heads_kv": 8, - "head_dim": 128, - "dtype_q": "nvfp4", - "dtype_k": "fp8", - "dtype_v": "fp8", - "dtype_o": "bf16", - "causal": true - }, - "name": "minimaxm2.5" - }, - { - "type": "attention_mha", - "args": { - "batch_size": 64, - "seq_len_q": 1, - "seq_len_kv": 1536, - "num_heads": 48, - "num_heads_kv": 8, - "head_dim": 128, - "dtype_q": "nvfp4", - "dtype_k": "fp8", - "dtype_v": "fp8", - "dtype_o": "bf16", - "causal": true - }, - "name": "minimaxm2.5" - }, - { - "type": "attention_mha", - "args": { - "batch_size": 4, - "seq_len_q": 1024, - "seq_len_kv": 1024, - "num_heads": 24, - "num_heads_kv": 4, - "head_dim": 128, - "dtype_q": "nvfp4", - "dtype_k": "fp8", - "dtype_v": "fp8", - "dtype_o": "bf16", - "causal": true - }, - "name": "minimaxm2.5" - }, - { - "type": "attention_mha", - "args": { - "batch_size": 4, - "seq_len_q": 1, - "seq_len_kv": 1536, - "num_heads": 24, - "num_heads_kv": 4, - "head_dim": 128, - "dtype_q": "nvfp4", - "dtype_k": "fp8", - "dtype_v": "fp8", - "dtype_o": "bf16", - "causal": true - }, - "name": "minimaxm2.5" - }, - { - "type": "attention_mha", - "args": { - "batch_size": 8, - "seq_len_q": 1024, - "seq_len_kv": 1024, - "num_heads": 24, - "num_heads_kv": 4, - "head_dim": 128, - "dtype_q": "nvfp4", - "dtype_k": "fp8", - "dtype_v": "fp8", - "dtype_o": "bf16", - "causal": true - }, - "name": "minimaxm2.5" - }, - { - "type": "attention_mha", - "args": { - "batch_size": 8, - "seq_len_q": 1, - "seq_len_kv": 1536, - "num_heads": 24, - "num_heads_kv": 4, - "head_dim": 128, - "dtype_q": "nvfp4", - "dtype_k": "fp8", - "dtype_v": "fp8", - "dtype_o": "bf16", - "causal": true - }, - "name": "minimaxm2.5" - }, - { - "type": "attention_mha", - "args": { - "batch_size": 32, - "seq_len_q": 1024, - "seq_len_kv": 1024, - "num_heads": 24, - "num_heads_kv": 4, - "head_dim": 128, - "dtype_q": "nvfp4", - "dtype_k": "fp8", - "dtype_v": "fp8", - "dtype_o": "bf16", - "causal": true - }, - "name": "minimaxm2.5" - }, - { - "type": "attention_mha", - "args": { - "batch_size": 32, - "seq_len_q": 1, - "seq_len_kv": 1536, - "num_heads": 24, - "num_heads_kv": 4, - "head_dim": 128, - "dtype_q": "nvfp4", - "dtype_k": "fp8", - "dtype_v": "fp8", - "dtype_o": "bf16", - "causal": true - }, - "name": "minimaxm2.5" - }, - { - "type": "attention_mha", - "args": { - "batch_size": 64, - "seq_len_q": 1024, - "seq_len_kv": 1024, - "num_heads": 24, - "num_heads_kv": 4, - "head_dim": 128, - "dtype_q": "nvfp4", - "dtype_k": "fp8", - "dtype_v": "fp8", - "dtype_o": "bf16", - "causal": true - }, - "name": "minimaxm2.5" - }, - { - "type": "attention_mha", - "args": { - "batch_size": 64, - "seq_len_q": 1, - "seq_len_kv": 1536, - "num_heads": 24, - "num_heads_kv": 4, - "head_dim": 128, - "dtype_q": "nvfp4", - "dtype_k": "fp8", - "dtype_v": "fp8", - "dtype_o": "bf16", - "causal": true - }, - "name": "minimaxm2.5" - }, - { - "type": "attention_mha", - "args": { - "batch_size": 256, - "seq_len_q": 1024, - "seq_len_kv": 1024, - "num_heads": 24, - "num_heads_kv": 4, - "head_dim": 128, - "dtype_q": "nvfp4", - "dtype_k": "fp8", - "dtype_v": "fp8", - "dtype_o": "bf16", - "causal": true - }, - "name": "minimaxm2.5" - }, - { - "type": "attention_mha", - "args": { - "batch_size": 256, - "seq_len_q": 1, - "seq_len_kv": 1536, - "num_heads": 24, - "num_heads_kv": 4, - "head_dim": 128, - "dtype_q": "nvfp4", - "dtype_k": "fp8", - "dtype_v": "fp8", - "dtype_o": "bf16", - "causal": true - }, - "name": "minimaxm2.5" - }, - { - "type": "attention_mha", - "args": { - "batch_size": 512, - "seq_len_q": 1024, - "seq_len_kv": 1024, - "num_heads": 24, - "num_heads_kv": 4, - "head_dim": 128, - "dtype_q": "nvfp4", - "dtype_k": "fp8", - "dtype_v": "fp8", - "dtype_o": "bf16", - "causal": true - }, - "name": "minimaxm2.5" - }, - { - "type": "attention_mha", - "args": { - "batch_size": 512, - "seq_len_q": 1, - "seq_len_kv": 1536, - "num_heads": 24, - "num_heads_kv": 4, - "head_dim": 128, - "dtype_q": "nvfp4", - "dtype_k": "fp8", - "dtype_v": "fp8", - "dtype_o": "bf16", - "causal": true - }, - "name": "minimaxm2.5" - }, - { - "type": "attention_mha", - "args": { - "batch_size": 1024, - "seq_len_q": 1024, - "seq_len_kv": 1024, - "num_heads": 24, - "num_heads_kv": 4, - "head_dim": 128, - "dtype_q": "nvfp4", - "dtype_k": "fp8", - "dtype_v": "fp8", - "dtype_o": "bf16", - "causal": true - }, - "name": "minimaxm2.5" - }, - { - "type": "attention_mha", - "args": { - "batch_size": 1024, - "seq_len_q": 1, - "seq_len_kv": 1536, - "num_heads": 24, - "num_heads_kv": 4, - "head_dim": 128, - "dtype_q": "nvfp4", - "dtype_k": "fp8", - "dtype_v": "fp8", - "dtype_o": "bf16", - "causal": true - }, - "name": "minimaxm2.5" - }, - { - "type": "attention_mha", - "args": { - "batch_size": 32, - "seq_len_q": 1024, - "seq_len_kv": 1024, - "num_heads": 12, - "num_heads_kv": 2, - "head_dim": 128, - "dtype_q": "nvfp4", - "dtype_k": "fp8", - "dtype_v": "fp8", - "dtype_o": "bf16", - "causal": true - }, - "name": "minimaxm2.5" - }, - { - "type": "attention_mha", - "args": { - "batch_size": 32, - "seq_len_q": 1, - "seq_len_kv": 1536, - "num_heads": 12, - "num_heads_kv": 2, - "head_dim": 128, - "dtype_q": "nvfp4", - "dtype_k": "fp8", - "dtype_v": "fp8", - "dtype_o": "bf16", - "causal": true - }, - "name": "minimaxm2.5" - }, - { - "type": "attention_mha", - "args": { - "batch_size": 16, - "seq_len_q": 1024, - "seq_len_kv": 1024, - "num_heads": 6, - "num_heads_kv": 1, - "head_dim": 128, - "dtype_q": "nvfp4", - "dtype_k": "fp8", - "dtype_v": "fp8", - "dtype_o": "bf16", - "causal": true - }, - "name": "minimaxm2.5" - }, - { - "type": "attention_mha", - "args": { - "batch_size": 16, - "seq_len_q": 1, - "seq_len_kv": 1536, - "num_heads": 6, - "num_heads_kv": 1, - "head_dim": 128, - "dtype_q": "nvfp4", - "dtype_k": "fp8", - "dtype_v": "fp8", - "dtype_o": "bf16", - "causal": true - }, - "name": "minimaxm2.5" - }, - { - "type": "attention_mha", - "args": { - "batch_size": 512, - "seq_len_q": 8192, - "seq_len_kv": 8192, - "num_heads": 48, - "num_heads_kv": 8, - "head_dim": 128, - "dtype_q": "nvfp4", - "dtype_k": "fp8", - "dtype_v": "fp8", - "dtype_o": "bf16", - "causal": true - }, - "name": "minimaxm2.5" - }, - { - "type": "attention_mha", - "args": { - "batch_size": 512, - "seq_len_q": 1, - "seq_len_kv": 8704, - "num_heads": 48, - "num_heads_kv": 8, - "head_dim": 128, - "dtype_q": "nvfp4", - "dtype_k": "fp8", - "dtype_v": "fp8", - "dtype_o": "bf16", - "causal": true - }, - "name": "minimaxm2.5" - }, - { - "type": "attention_mha", - "args": { - "batch_size": 1024, - "seq_len_q": 8192, - "seq_len_kv": 8192, - "num_heads": 48, - "num_heads_kv": 8, - "head_dim": 128, - "dtype_q": "nvfp4", - "dtype_k": "fp8", - "dtype_v": "fp8", - "dtype_o": "bf16", - "causal": true - }, - "name": "minimaxm2.5" - }, - { - "type": "attention_mha", - "args": { - "batch_size": 1024, - "seq_len_q": 1, - "seq_len_kv": 8704, - "num_heads": 48, - "num_heads_kv": 8, - "head_dim": 128, - "dtype_q": "nvfp4", - "dtype_k": "fp8", - "dtype_v": "fp8", - "dtype_o": "bf16", - "causal": true - }, - "name": "minimaxm2.5" - }, - { - "type": "attention_mha", - "args": { - "batch_size": 4, - "seq_len_q": 8192, - "seq_len_kv": 8192, - "num_heads": 24, - "num_heads_kv": 4, - "head_dim": 128, - "dtype_q": "nvfp4", - "dtype_k": "fp8", - "dtype_v": "fp8", - "dtype_o": "bf16", - "causal": true - }, - "name": "minimaxm2.5" - }, - { - "type": "attention_mha", - "args": { - "batch_size": 4, - "seq_len_q": 1, - "seq_len_kv": 8704, - "num_heads": 24, - "num_heads_kv": 4, - "head_dim": 128, - "dtype_q": "nvfp4", - "dtype_k": "fp8", - "dtype_v": "fp8", - "dtype_o": "bf16", - "causal": true - }, - "name": "minimaxm2.5" - }, - { - "type": "attention_mha", - "args": { - "batch_size": 8, - "seq_len_q": 8192, - "seq_len_kv": 8192, - "num_heads": 24, - "num_heads_kv": 4, - "head_dim": 128, - "dtype_q": "nvfp4", - "dtype_k": "fp8", - "dtype_v": "fp8", - "dtype_o": "bf16", - "causal": true - }, - "name": "minimaxm2.5" - }, - { - "type": "attention_mha", - "args": { - "batch_size": 8, - "seq_len_q": 1, - "seq_len_kv": 8704, - "num_heads": 24, - "num_heads_kv": 4, - "head_dim": 128, - "dtype_q": "nvfp4", - "dtype_k": "fp8", - "dtype_v": "fp8", - "dtype_o": "bf16", - "causal": true - }, - "name": "minimaxm2.5" - }, - { - "type": "attention_mha", - "args": { - "batch_size": 16, - "seq_len_q": 8192, - "seq_len_kv": 8192, - "num_heads": 24, - "num_heads_kv": 4, - "head_dim": 128, - "dtype_q": "nvfp4", - "dtype_k": "fp8", - "dtype_v": "fp8", - "dtype_o": "bf16", - "causal": true - }, - "name": "minimaxm2.5" - }, - { - "type": "attention_mha", - "args": { - "batch_size": 16, - "seq_len_q": 1, - "seq_len_kv": 8704, - "num_heads": 24, - "num_heads_kv": 4, - "head_dim": 128, - "dtype_q": "nvfp4", - "dtype_k": "fp8", - "dtype_v": "fp8", - "dtype_o": "bf16", - "causal": true - }, - "name": "minimaxm2.5" - }, - { - "type": "attention_mha", - "args": { - "batch_size": 32, - "seq_len_q": 8192, - "seq_len_kv": 8192, - "num_heads": 24, - "num_heads_kv": 4, - "head_dim": 128, - "dtype_q": "nvfp4", - "dtype_k": "fp8", - "dtype_v": "fp8", - "dtype_o": "bf16", - "causal": true - }, - "name": "minimaxm2.5" - }, - { - "type": "attention_mha", - "args": { - "batch_size": 32, - "seq_len_q": 1, - "seq_len_kv": 8704, - "num_heads": 24, - "num_heads_kv": 4, - "head_dim": 128, - "dtype_q": "nvfp4", - "dtype_k": "fp8", - "dtype_v": "fp8", - "dtype_o": "bf16", - "causal": true - }, - "name": "minimaxm2.5" - }, - { - "type": "attention_mha", - "args": { - "batch_size": 64, - "seq_len_q": 8192, - "seq_len_kv": 8192, - "num_heads": 24, - "num_heads_kv": 4, - "head_dim": 128, - "dtype_q": "nvfp4", - "dtype_k": "fp8", - "dtype_v": "fp8", - "dtype_o": "bf16", - "causal": true - }, - "name": "minimaxm2.5" - }, - { - "type": "attention_mha", - "args": { - "batch_size": 64, - "seq_len_q": 1, - "seq_len_kv": 8704, - "num_heads": 24, - "num_heads_kv": 4, - "head_dim": 128, - "dtype_q": "nvfp4", - "dtype_k": "fp8", - "dtype_v": "fp8", - "dtype_o": "bf16", - "causal": true - }, - "name": "minimaxm2.5" - }, - { - "type": "attention_mha", - "args": { - "batch_size": 1024, - "seq_len_q": 8192, - "seq_len_kv": 8192, - "num_heads": 24, - "num_heads_kv": 4, - "head_dim": 128, - "dtype_q": "nvfp4", - "dtype_k": "fp8", - "dtype_v": "fp8", - "dtype_o": "bf16", - "causal": true - }, - "name": "minimaxm2.5" - }, - { - "type": "attention_mha", - "args": { - "batch_size": 1024, - "seq_len_q": 1, - "seq_len_kv": 8704, - "num_heads": 24, - "num_heads_kv": 4, - "head_dim": 128, - "dtype_q": "nvfp4", - "dtype_k": "fp8", - "dtype_v": "fp8", - "dtype_o": "bf16", - "causal": true - }, - "name": "minimaxm2.5" - }, - { - "type": "attention_mha", - "args": { - "batch_size": 16, - "seq_len_q": 8192, - "seq_len_kv": 8192, - "num_heads": 12, - "num_heads_kv": 2, - "head_dim": 128, - "dtype_q": "nvfp4", - "dtype_k": "fp8", - "dtype_v": "fp8", - "dtype_o": "bf16", - "causal": true - }, - "name": "minimaxm2.5" - }, - { - "type": "attention_mha", - "args": { - "batch_size": 16, - "seq_len_q": 1, - "seq_len_kv": 8704, - "num_heads": 12, - "num_heads_kv": 2, - "head_dim": 128, - "dtype_q": "nvfp4", - "dtype_k": "fp8", - "dtype_v": "fp8", - "dtype_o": "bf16", - "causal": true - }, - "name": "minimaxm2.5" - }, - { - "type": "attention_mha", - "args": { - "batch_size": 32, - "seq_len_q": 8192, - "seq_len_kv": 8192, - "num_heads": 12, - "num_heads_kv": 2, - "head_dim": 128, - "dtype_q": "nvfp4", - "dtype_k": "fp8", - "dtype_v": "fp8", - "dtype_o": "bf16", - "causal": true - }, - "name": "minimaxm2.5" - }, - { - "type": "attention_mha", - "args": { - "batch_size": 32, - "seq_len_q": 1, - "seq_len_kv": 8704, - "num_heads": 12, - "num_heads_kv": 2, - "head_dim": 128, - "dtype_q": "nvfp4", - "dtype_k": "fp8", - "dtype_v": "fp8", - "dtype_o": "bf16", - "causal": true - }, - "name": "minimaxm2.5" - }, - { - "type": "attention_mha", - "args": { - "batch_size": 64, - "seq_len_q": 8192, - "seq_len_kv": 8192, - "num_heads": 12, - "num_heads_kv": 2, - "head_dim": 128, - "dtype_q": "nvfp4", - "dtype_k": "fp8", - "dtype_v": "fp8", - "dtype_o": "bf16", - "causal": true - }, - "name": "minimaxm2.5" - }, - { - "type": "attention_mha", - "args": { - "batch_size": 64, - "seq_len_q": 1, - "seq_len_kv": 8704, - "num_heads": 12, - "num_heads_kv": 2, - "head_dim": 128, - "dtype_q": "nvfp4", - "dtype_k": "fp8", - "dtype_v": "fp8", - "dtype_o": "bf16", - "causal": true - }, - "name": "minimaxm2.5" - }, - { - "type": "attention_mha", - "args": { - "batch_size": 128, - "seq_len_q": 8192, - "seq_len_kv": 8192, - "num_heads": 12, - "num_heads_kv": 2, - "head_dim": 128, - "dtype_q": "nvfp4", - "dtype_k": "fp8", - "dtype_v": "fp8", - "dtype_o": "bf16", - "causal": true - }, - "name": "minimaxm2.5" - }, - { - "type": "attention_mha", - "args": { - "batch_size": 128, - "seq_len_q": 1, - "seq_len_kv": 8704, - "num_heads": 12, - "num_heads_kv": 2, - "head_dim": 128, - "dtype_q": "nvfp4", - "dtype_k": "fp8", - "dtype_v": "fp8", - "dtype_o": "bf16", - "causal": true - }, - "name": "minimaxm2.5" - }, - { - "type": "attention_mha", - "args": { - "batch_size": 8, - "seq_len_q": 8192, - "seq_len_kv": 8192, - "num_heads": 6, - "num_heads_kv": 1, - "head_dim": 128, - "dtype_q": "nvfp4", - "dtype_k": "fp8", - "dtype_v": "fp8", - "dtype_o": "bf16", - "causal": true - }, - "name": "minimaxm2.5" - }, - { - "type": "attention_mha", - "args": { - "batch_size": 8, - "seq_len_q": 1, - "seq_len_kv": 8704, - "num_heads": 6, - "num_heads_kv": 1, - "head_dim": 128, - "dtype_q": "nvfp4", - "dtype_k": "fp8", - "dtype_v": "fp8", - "dtype_o": "bf16", - "causal": true - }, - "name": "minimaxm2.5" - }, - { - "type": "attention_mha", - "args": { - "batch_size": 16, - "seq_len_q": 8192, - "seq_len_kv": 8192, - "num_heads": 6, - "num_heads_kv": 1, - "head_dim": 128, - "dtype_q": "nvfp4", - "dtype_k": "fp8", - "dtype_v": "fp8", - "dtype_o": "bf16", - "causal": true - }, - "name": "minimaxm2.5" - }, - { - "type": "attention_mha", - "args": { - "batch_size": 16, - "seq_len_q": 1, - "seq_len_kv": 8704, - "num_heads": 6, - "num_heads_kv": 1, - "head_dim": 128, - "dtype_q": "nvfp4", - "dtype_k": "fp8", - "dtype_v": "fp8", - "dtype_o": "bf16", - "causal": true - }, - "name": "minimaxm2.5" - }, - { - "type": "attention_mha", - "args": { - "batch_size": 256, - "seq_len_q": 1024, - "seq_len_kv": 1024, - "num_heads": 6, - "num_heads_kv": 1, - "head_dim": 128, - "dtype_q": "fp8", - "dtype_k": "fp8", - "dtype_v": "fp8", - "dtype_o": "bf16", - "causal": true - }, - "name": "minimaxm2.5" - }, - { - "type": "attention_mha", - "args": { - "batch_size": 256, - "seq_len_q": 1, - "seq_len_kv": 1536, - "num_heads": 6, - "num_heads_kv": 1, - "head_dim": 128, - "dtype_q": "fp8", - "dtype_k": "fp8", - "dtype_v": "fp8", - "dtype_o": "bf16", - "causal": true - }, - "name": "minimaxm2.5" - }, - { - "type": "attention_mha", - "args": { - "batch_size": 512, - "seq_len_q": 1024, - "seq_len_kv": 1024, - "num_heads": 6, - "num_heads_kv": 1, - "head_dim": 128, - "dtype_q": "fp8", - "dtype_k": "fp8", - "dtype_v": "fp8", - "dtype_o": "bf16", - "causal": true - }, - "name": "minimaxm2.5" - }, - { - "type": "attention_mha", - "args": { - "batch_size": 512, - "seq_len_q": 1, - "seq_len_kv": 1536, - "num_heads": 6, - "num_heads_kv": 1, - "head_dim": 128, - "dtype_q": "fp8", - "dtype_k": "fp8", - "dtype_v": "fp8", - "dtype_o": "bf16", - "causal": true - }, - "name": "minimaxm2.5" - }, - { - "type": "attention_mha", - "args": { - "batch_size": 256, - "seq_len_q": 8192, - "seq_len_kv": 8192, - "num_heads": 6, - "num_heads_kv": 1, - "head_dim": 128, - "dtype_q": "fp8", - "dtype_k": "fp8", - "dtype_v": "fp8", - "dtype_o": "bf16", - "causal": true - }, - "name": "minimaxm2.5" - }, - { - "type": "attention_mha", - "args": { - "batch_size": 256, - "seq_len_q": 1, - "seq_len_kv": 8704, - "num_heads": 6, - "num_heads_kv": 1, - "head_dim": 128, - "dtype_q": "fp8", - "dtype_k": "fp8", - "dtype_v": "fp8", - "dtype_o": "bf16", - "causal": true - }, - "name": "minimaxm2.5" - }, - { - "type": "attention_mha", - "args": { - "batch_size": 1, - "seq_len_q": 1024, - "seq_len_kv": 1024, - "num_heads": 8, - "num_heads_kv": 1, - "head_dim": 64, - "dtype_q": "nvfp4", - "dtype_k": "fp8", - "dtype_v": "fp8", - "dtype_o": "bf16", - "causal": true, - "sliding_window": 128 - }, - "name": "gptoss" - }, - { - "type": "attention_mha", - "args": { - "batch_size": 1, - "seq_len_q": 1, - "seq_len_kv": 1536, - "num_heads": 8, - "num_heads_kv": 1, - "head_dim": 64, - "dtype_q": "nvfp4", - "dtype_k": "fp8", - "dtype_v": "fp8", - "dtype_o": "bf16", - "causal": true, - "sliding_window": 128 - }, - "name": "gptoss" - }, - { - "type": "attention_mha", - "args": { - "batch_size": 2, - "seq_len_q": 1024, - "seq_len_kv": 1024, - "num_heads": 8, - "num_heads_kv": 1, - "head_dim": 64, - "dtype_q": "nvfp4", - "dtype_k": "fp8", - "dtype_v": "fp8", - "dtype_o": "bf16", - "causal": true, - "sliding_window": 128 - }, - "name": "gptoss" - }, - { - "type": "attention_mha", - "args": { - "batch_size": 2, - "seq_len_q": 1, - "seq_len_kv": 1536, - "num_heads": 8, - "num_heads_kv": 1, - "head_dim": 64, - "dtype_q": "nvfp4", - "dtype_k": "fp8", - "dtype_v": "fp8", - "dtype_o": "bf16", - "causal": true, - "sliding_window": 128 - }, - "name": "gptoss" - }, - { - "type": "attention_mha", - "args": { - "batch_size": 2, - "seq_len_q": 8192, - "seq_len_kv": 8192, - "num_heads": 8, - "num_heads_kv": 1, - "head_dim": 64, - "dtype_q": "nvfp4", - "dtype_k": "fp8", - "dtype_v": "fp8", - "dtype_o": "bf16", - "causal": true, - "sliding_window": 128 - }, - "name": "gptoss" - }, - { - "type": "attention_mha", - "args": { - "batch_size": 2, - "seq_len_q": 1, - "seq_len_kv": 8704, - "num_heads": 8, - "num_heads_kv": 1, - "head_dim": 64, - "dtype_q": "nvfp4", - "dtype_k": "fp8", - "dtype_v": "fp8", - "dtype_o": "bf16", - "causal": true, - "sliding_window": 128 - }, - "name": "gptoss" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 1536, - "seq_len_q": 1024, - "seq_len_kv": 1024, - "num_heads": 16, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "fp8", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 1536, - "seq_len_q": 1, - "seq_len_kv": 1536, - "num_heads": 16, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "fp8", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 64, - "seq_len_q": 1024, - "seq_len_kv": 1024, - "num_heads": 32, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "fp8", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 64, - "seq_len_q": 1, - "seq_len_kv": 1536, - "num_heads": 32, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "fp8", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 2, - "seq_len_q": 1024, - "seq_len_kv": 1024, - "num_heads": 32, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "fp8", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 2, - "seq_len_q": 1, - "seq_len_kv": 1536, - "num_heads": 32, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "fp8", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 1, - "seq_len_q": 1024, - "seq_len_kv": 1024, - "num_heads": 32, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "fp8", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 1, - "seq_len_q": 1, - "seq_len_kv": 1536, - "num_heads": 32, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "fp8", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 2, - "seq_len_q": 1024, - "seq_len_kv": 1024, - "num_heads": 16, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "fp8", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 2, - "seq_len_q": 1, - "seq_len_kv": 1536, - "num_heads": 16, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "fp8", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 2, - "seq_len_q": 8192, - "seq_len_kv": 8192, - "num_heads": 32, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "fp8", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 2, - "seq_len_q": 1, - "seq_len_kv": 8704, - "num_heads": 32, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "fp8", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 1, - "seq_len_q": 8192, - "seq_len_kv": 8192, - "num_heads": 32, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "fp8", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 1, - "seq_len_q": 1, - "seq_len_kv": 8704, - "num_heads": 32, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "fp8", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 2, - "seq_len_q": 1, - "seq_len_kv": 8704, - "num_heads": 16, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "fp8", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 1536, - "seq_len_q": 2, - "seq_len_kv": 1536, - "num_heads": 16, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "fp8", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 192, - "seq_len_q": 2, - "seq_len_kv": 1536, - "num_heads": 128, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "fp8", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 64, - "seq_len_q": 2, - "seq_len_kv": 1536, - "num_heads": 32, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "fp8", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 32, - "seq_len_q": 2, - "seq_len_kv": 1536, - "num_heads": 32, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "fp8", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 16, - "seq_len_q": 2, - "seq_len_kv": 1536, - "num_heads": 32, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "fp8", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 8, - "seq_len_q": 2, - "seq_len_kv": 1536, - "num_heads": 32, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "fp8", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 4, - "seq_len_q": 2, - "seq_len_kv": 1536, - "num_heads": 32, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "fp8", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 2, - "seq_len_q": 2, - "seq_len_kv": 1536, - "num_heads": 32, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "fp8", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 1, - "seq_len_q": 2, - "seq_len_kv": 1536, - "num_heads": 32, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "fp8", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 2, - "seq_len_q": 2, - "seq_len_kv": 1536, - "num_heads": 16, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "fp8", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 256, - "seq_len_q": 2, - "seq_len_kv": 8704, - "num_heads": 128, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "fp8", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 64, - "seq_len_q": 2, - "seq_len_kv": 8704, - "num_heads": 32, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "fp8", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 32, - "seq_len_q": 2, - "seq_len_kv": 8704, - "num_heads": 32, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "fp8", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 16, - "seq_len_q": 2, - "seq_len_kv": 8704, - "num_heads": 32, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "fp8", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 8, - "seq_len_q": 2, - "seq_len_kv": 8704, - "num_heads": 32, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "fp8", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 4, - "seq_len_q": 2, - "seq_len_kv": 8704, - "num_heads": 32, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "fp8", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 2, - "seq_len_q": 2, - "seq_len_kv": 8704, - "num_heads": 32, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "fp8", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 1, - "seq_len_q": 2, - "seq_len_kv": 8704, - "num_heads": 32, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "fp8", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 1, - "seq_len_q": 1024, - "seq_len_kv": 1024, - "num_heads": 16, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 1, - "seq_len_q": 1, - "seq_len_kv": 1536, - "num_heads": 16, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 2, - "seq_len_q": 1024, - "seq_len_kv": 1024, - "num_heads": 16, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 2, - "seq_len_q": 1, - "seq_len_kv": 1536, - "num_heads": 16, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 1, - "seq_len_q": 2, - "seq_len_kv": 1536, - "num_heads": 16, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 2, - "seq_len_q": 2, - "seq_len_kv": 1536, - "num_heads": 16, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 512, - "seq_len_q": 2, - "seq_len_kv": 1536, - "num_heads": 128, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 1024, - "seq_len_q": 2, - "seq_len_kv": 1536, - "num_heads": 128, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 1, - "seq_len_q": 2, - "seq_len_kv": 8704, - "num_heads": 16, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 2, - "seq_len_q": 2, - "seq_len_kv": 8704, - "num_heads": 16, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 128, - "seq_len_q": 2, - "seq_len_kv": 8704, - "num_heads": 32, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 256, - "seq_len_q": 2, - "seq_len_kv": 8704, - "num_heads": 32, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 128, - "seq_len_q": 2, - "seq_len_kv": 8704, - "num_heads": 128, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 256, - "seq_len_q": 2, - "seq_len_kv": 8704, - "num_heads": 128, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 512, - "seq_len_q": 2, - "seq_len_kv": 8704, - "num_heads": 128, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsr1" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 2, - "seq_len_q": 1024, - "seq_len_kv": 1024, - "num_heads": 128, - "head_dim_qk_nope": 448, - "head_dim_qk_rope": 64, - "head_dim_v": 512, - "kv_lora_rank": 1024, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsv4" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 2, - "seq_len_q": 1, - "seq_len_kv": 1536, - "num_heads": 128, - "head_dim_qk_nope": 448, - "head_dim_qk_rope": 64, - "head_dim_v": 512, - "kv_lora_rank": 1024, - "dtype_q": "nvfp4", - "dtype_kv": "fp8", - "dtype_o": "bf16", - "rope": true, - "causal": true - }, - "name": "dsv4" - } -] \ No newline at end of file diff --git a/operatorx/testlists/attention_perf.json b/operatorx/testlists/attention_perf.json deleted file mode 100644 index 65a7d2d750..0000000000 --- a/operatorx/testlists/attention_perf.json +++ /dev/null @@ -1,138 +0,0 @@ -[ - { - "type": "attention_mha", - "args": { - "batch_size": 1, - "seq_len_q": 4096, - "seq_len_kv": 4096, - "num_heads": 32, - "num_heads_kv": 8, - "head_dim": 128, - "dtype_q": "bf16", - "dtype_k": "bf16", - "dtype_v": "bf16", - "dtype_o": "bf16", - "causal": true - }, - "name": "llama3-8b" - }, - { - "type": "attention_mha", - "args": { - "batch_size": 1, - "seq_len_q": 4096, - "seq_len_kv": 4096, - "num_heads": 32, - "num_heads_kv": 8, - "head_dim": 128, - "dtype_q": "fp16", - "dtype_k": "fp16", - "dtype_v": "fp16", - "dtype_o": "fp16", - "causal": true - }, - "name": "llama3-8b" - }, - { - "type": "attention_mha", - "args": { - "batch_size": 1, - "seq_len_q": 1, - "seq_len_kv": 4096, - "num_heads": 32, - "num_heads_kv": 8, - "head_dim": 128, - "dtype_q": "bf16", - "dtype_k": "bf16", - "dtype_v": "bf16", - "dtype_o": "bf16", - "causal": true - }, - "name": "llama3-8b" - }, - { - "type": "attention_mha", - "args": { - "batch_size": 1, - "seq_len_q": 1, - "seq_len_kv": 4096, - "num_heads": 32, - "num_heads_kv": 8, - "head_dim": 128, - "dtype_q": "fp16", - "dtype_k": "fp16", - "dtype_v": "fp16", - "dtype_o": "fp16", - "causal": true - }, - "name": "llama3-8b" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 1, - "seq_len_q": 4096, - "seq_len_kv": 4096, - "num_heads": 128, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "bf16", - "dtype_kv": "bf16", - "dtype_o": "bf16" - }, - "name": "dsv3" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 1, - "seq_len_q": 4096, - "seq_len_kv": 4096, - "num_heads": 128, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "fp16", - "dtype_kv": "fp16", - "dtype_o": "fp16" - }, - "name": "dsv3" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 1, - "seq_len_q": 1, - "seq_len_kv": 4096, - "num_heads": 128, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "bf16", - "dtype_kv": "bf16", - "dtype_o": "bf16" - }, - "name": "dsv3" - }, - { - "type": "attention_mla", - "args": { - "batch_size": 1, - "seq_len_q": 1, - "seq_len_kv": 4096, - "num_heads": 128, - "head_dim_qk_nope": 128, - "head_dim_qk_rope": 64, - "head_dim_v": 128, - "kv_lora_rank": 512, - "dtype_q": "fp16", - "dtype_kv": "fp16", - "dtype_o": "fp16" - }, - "name": "dsv3" - } -] diff --git a/operatorx/testlists/attn_deepseek_v4.json b/operatorx/testlists/attn_deepseek_v4.json new file mode 100644 index 0000000000..b6fa6a9c3a --- /dev/null +++ b/operatorx/testlists/attn_deepseek_v4.json @@ -0,0 +1,44234 @@ +[ + { + "type": "dsv4_attn", + "args": { + "hidden": 7168, + "heads": 128, + "head_dim": 512, + "rope_dim": 64, + "q_lora_rank": 1536, + "window": 128, + "o_groups": 16, + "o_lora_rank": 1024, + "compress_rope_theta": 160000, + "rope_scaling": { + "type": "yarn", + "factor": 16, + "original_max": 65536, + "beta_fast": 32, + "beta_slow": 1 + }, + "kv_cache_dtype": "fp8", + "compress_ratio": 4, + "indexer": "own", + "topk": 1024, + "index_heads": 64, + "index_dim": 128, + "index_cache_dtype": "fp8", + "proj": { + "indexer.wq_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 128 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + } + }, + "wkv": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 128 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + } + }, + "wo_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 128 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + } + }, + "wq_a": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 128 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + } + }, + "wq_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 128 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + } + } + }, + "batch": { + "groups": [ + { + "count": 1, + "q": 1, + "ctx": 1024 + } + ] + } + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro/attn", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn" + ] + }, + { + "type": "dsv4_attn", + "args": { + "hidden": 7168, + "heads": 128, + "head_dim": 512, + "rope_dim": 64, + "q_lora_rank": 1536, + "window": 128, + "o_groups": 16, + "o_lora_rank": 1024, + "compress_rope_theta": 160000, + "rope_scaling": { + "type": "yarn", + "factor": 16, + "original_max": 65536, + "beta_fast": 32, + "beta_slow": 1 + }, + "kv_cache_dtype": "fp8", + "compress_ratio": 4, + "indexer": "own", + "topk": 1024, + "index_heads": 64, + "index_dim": 128, + "index_cache_dtype": "fp8", + "proj": { + "indexer.wq_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 128 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + } + }, + "wkv": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 128 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + } + }, + "wo_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 128 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + } + }, + "wq_a": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 128 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + } + }, + "wq_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 128 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + } + } + }, + "batch": { + "groups": [ + { + "count": 1, + "q": 1, + "ctx": 4096 + } + ] + } + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro/attn", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn" + ] + }, + { + "type": "dsv4_attn", + "args": { + "hidden": 7168, + "heads": 128, + "head_dim": 512, + "rope_dim": 64, + "q_lora_rank": 1536, + "window": 128, + "o_groups": 16, + "o_lora_rank": 1024, + "compress_rope_theta": 160000, + "rope_scaling": { + "type": "yarn", + "factor": 16, + "original_max": 65536, + "beta_fast": 32, + "beta_slow": 1 + }, + "kv_cache_dtype": "fp8", + "compress_ratio": 4, + "indexer": "own", + "topk": 1024, + "index_heads": 64, + "index_dim": 128, + "index_cache_dtype": "fp8", + "proj": { + "indexer.wq_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 128 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + } + }, + "wkv": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 128 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + } + }, + "wo_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 128 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + } + }, + "wq_a": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 128 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + } + }, + "wq_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 128 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + } + } + }, + "batch": { + "groups": [ + { + "count": 1, + "q": 1, + "ctx": 16384 + } + ] + } + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro/attn", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn" + ] + }, + { + "type": "dsv4_attn", + "args": { + "hidden": 7168, + "heads": 128, + "head_dim": 512, + "rope_dim": 64, + "q_lora_rank": 1536, + "window": 128, + "o_groups": 16, + "o_lora_rank": 1024, + "compress_rope_theta": 160000, + "rope_scaling": { + "type": "yarn", + "factor": 16, + "original_max": 65536, + "beta_fast": 32, + "beta_slow": 1 + }, + "kv_cache_dtype": "fp8", + "compress_ratio": 4, + "indexer": "own", + "topk": 1024, + "index_heads": 64, + "index_dim": 128, + "index_cache_dtype": "fp8", + "proj": { + "indexer.wq_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 128 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + } + }, + "wkv": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 128 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + } + }, + "wo_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 128 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + } + }, + "wq_a": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 128 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + } + }, + "wq_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 128 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + } + } + }, + "batch": { + "groups": [ + { + "count": 1, + "q": 1, + "ctx": 65536 + } + ] + } + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro/attn", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn" + ] + }, + { + "type": "dsv4_attn", + "args": { + "hidden": 7168, + "heads": 128, + "head_dim": 512, + "rope_dim": 64, + "q_lora_rank": 1536, + "window": 128, + "o_groups": 16, + "o_lora_rank": 1024, + "compress_rope_theta": 160000, + "rope_scaling": { + "type": "yarn", + "factor": 16, + "original_max": 65536, + "beta_fast": 32, + "beta_slow": 1 + }, + "kv_cache_dtype": "fp8", + "compress_ratio": 4, + "indexer": "own", + "topk": 1024, + "index_heads": 64, + "index_dim": 128, + "index_cache_dtype": "fp8", + "proj": { + "indexer.wq_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 128 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + } + }, + "wkv": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 128 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + } + }, + "wo_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 128 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + } + }, + "wq_a": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 128 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + } + }, + "wq_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 128 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + } + } + }, + "batch": { + "groups": [ + { + "count": 1, + "q": 1, + "ctx": 131072 + } + ] + } + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro/attn", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn" + ] + }, + { + "type": "dsv4_attn", + "args": { + "hidden": 7168, + "heads": 128, + "head_dim": 512, + "rope_dim": 64, + "q_lora_rank": 1536, + "window": 128, + "o_groups": 16, + "o_lora_rank": 1024, + "compress_rope_theta": 160000, + "rope_scaling": { + "type": "yarn", + "factor": 16, + "original_max": 65536, + "beta_fast": 32, + "beta_slow": 1 + }, + "kv_cache_dtype": "fp8", + "compress_ratio": 4, + "indexer": "own", + "topk": 1024, + "index_heads": 64, + "index_dim": 128, + "index_cache_dtype": "fp8", + "proj": { + "indexer.wq_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 128 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + } + }, + "wkv": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 128 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + } + }, + "wo_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 128 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + } + }, + "wq_a": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 128 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + } + }, + "wq_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 128 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + } + } + }, + "batch": { + "groups": [ + { + "count": 4, + "q": 1, + "ctx": 1024 + } + ] + } + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro/attn", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn" + ] + }, + { + "type": "dsv4_attn", + "args": { + "hidden": 7168, + "heads": 128, + "head_dim": 512, + "rope_dim": 64, + "q_lora_rank": 1536, + "window": 128, + "o_groups": 16, + "o_lora_rank": 1024, + "compress_rope_theta": 160000, + "rope_scaling": { + "type": "yarn", + "factor": 16, + "original_max": 65536, + "beta_fast": 32, + "beta_slow": 1 + }, + "kv_cache_dtype": "fp8", + "compress_ratio": 4, + "indexer": "own", + "topk": 1024, + "index_heads": 64, + "index_dim": 128, + "index_cache_dtype": "fp8", + "proj": { + "indexer.wq_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 128 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + } + }, + "wkv": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 128 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + } + }, + "wo_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 128 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + } + }, + "wq_a": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 128 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + } + }, + "wq_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 128 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + } + } + }, + "batch": { + "groups": [ + { + "count": 4, + "q": 1, + "ctx": 4096 + } + ] + } + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro/attn", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn" + ] + }, + { + "type": "dsv4_attn", + "args": { + "hidden": 7168, + "heads": 128, + "head_dim": 512, + "rope_dim": 64, + "q_lora_rank": 1536, + "window": 128, + "o_groups": 16, + "o_lora_rank": 1024, + "compress_rope_theta": 160000, + "rope_scaling": { + "type": "yarn", + "factor": 16, + "original_max": 65536, + "beta_fast": 32, + "beta_slow": 1 + }, + "kv_cache_dtype": "fp8", + "compress_ratio": 4, + "indexer": "own", + "topk": 1024, + "index_heads": 64, + "index_dim": 128, + "index_cache_dtype": "fp8", + "proj": { + "indexer.wq_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 128 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + } + }, + "wkv": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 128 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + } + }, + "wo_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 128 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + } + }, + "wq_a": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 128 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + } + }, + "wq_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 128 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + } + } + }, + "batch": { + "groups": [ + { + "count": 4, + "q": 1, + "ctx": 16384 + } + ] + } + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro/attn", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn" + ] + }, + { + "type": "dsv4_attn", + "args": { + "hidden": 7168, + "heads": 128, + "head_dim": 512, + "rope_dim": 64, + "q_lora_rank": 1536, + "window": 128, + "o_groups": 16, + "o_lora_rank": 1024, + "compress_rope_theta": 160000, + "rope_scaling": { + "type": "yarn", + "factor": 16, + "original_max": 65536, + "beta_fast": 32, + "beta_slow": 1 + }, + "kv_cache_dtype": "fp8", + "compress_ratio": 4, + "indexer": "own", + "topk": 1024, + "index_heads": 64, + "index_dim": 128, + "index_cache_dtype": "fp8", + "proj": { + "indexer.wq_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 128 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + } + }, + "wkv": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 128 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + } + }, + "wo_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 128 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + } + }, + "wq_a": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 128 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + } + }, + "wq_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 128 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + } + } + }, + "batch": { + "groups": [ + { + "count": 4, + "q": 1, + "ctx": 65536 + } + ] + } + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro/attn", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn" + ] + }, + { + "type": "dsv4_attn", + "args": { + "hidden": 7168, + "heads": 128, + "head_dim": 512, + "rope_dim": 64, + "q_lora_rank": 1536, + "window": 128, + "o_groups": 16, + "o_lora_rank": 1024, + "compress_rope_theta": 160000, + "rope_scaling": { + "type": "yarn", + "factor": 16, + "original_max": 65536, + "beta_fast": 32, + "beta_slow": 1 + }, + "kv_cache_dtype": "fp8", + "compress_ratio": 4, + "indexer": "own", + "topk": 1024, + "index_heads": 64, + "index_dim": 128, + "index_cache_dtype": "fp8", + "proj": { + "indexer.wq_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 128 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + } + }, + "wkv": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 128 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + } + }, + "wo_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 128 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + } + }, + "wq_a": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 128 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + } + }, + "wq_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 128 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + } + } + }, + "batch": { + "groups": [ + { + "count": 4, + "q": 1, + "ctx": 131072 + } + ] + } + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro/attn", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn" + ] + }, + { + "type": "dsv4_attn", + "args": { + "hidden": 7168, + "heads": 128, + "head_dim": 512, + "rope_dim": 64, + "q_lora_rank": 1536, + "window": 128, + "o_groups": 16, + "o_lora_rank": 1024, + "compress_rope_theta": 160000, + "rope_scaling": { + "type": "yarn", + "factor": 16, + "original_max": 65536, + "beta_fast": 32, + "beta_slow": 1 + }, + "kv_cache_dtype": "fp8", + "compress_ratio": 4, + "indexer": "own", + "topk": 1024, + "index_heads": 64, + "index_dim": 128, + "index_cache_dtype": "fp8", + "proj": { + "indexer.wq_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 128 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + } + }, + "wkv": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 128 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + } + }, + "wo_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 128 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + } + }, + "wq_a": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 128 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + } + }, + "wq_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 128 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + } + } + }, + "batch": { + "groups": [ + { + "count": 16, + "q": 1, + "ctx": 1024 + } + ] + } + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro/attn", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn" + ] + }, + { + "type": "dsv4_attn", + "args": { + "hidden": 7168, + "heads": 128, + "head_dim": 512, + "rope_dim": 64, + "q_lora_rank": 1536, + "window": 128, + "o_groups": 16, + "o_lora_rank": 1024, + "compress_rope_theta": 160000, + "rope_scaling": { + "type": "yarn", + "factor": 16, + "original_max": 65536, + "beta_fast": 32, + "beta_slow": 1 + }, + "kv_cache_dtype": "fp8", + "compress_ratio": 4, + "indexer": "own", + "topk": 1024, + "index_heads": 64, + "index_dim": 128, + "index_cache_dtype": "fp8", + "proj": { + "indexer.wq_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 128 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + } + }, + "wkv": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 128 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + } + }, + "wo_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 128 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + } + }, + "wq_a": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 128 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + } + }, + "wq_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 128 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + } + } + }, + "batch": { + "groups": [ + { + "count": 16, + "q": 1, + "ctx": 4096 + } + ] + } + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro/attn", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn" + ] + }, + { + "type": "dsv4_attn", + "args": { + "hidden": 7168, + "heads": 128, + "head_dim": 512, + "rope_dim": 64, + "q_lora_rank": 1536, + "window": 128, + "o_groups": 16, + "o_lora_rank": 1024, + "compress_rope_theta": 160000, + "rope_scaling": { + "type": "yarn", + "factor": 16, + "original_max": 65536, + "beta_fast": 32, + "beta_slow": 1 + }, + "kv_cache_dtype": "fp8", + "compress_ratio": 4, + "indexer": "own", + "topk": 1024, + "index_heads": 64, + "index_dim": 128, + "index_cache_dtype": "fp8", + "proj": { + "indexer.wq_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 128 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + } + }, + "wkv": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 128 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + } + }, + "wo_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 128 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + } + }, + "wq_a": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 128 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + } + }, + "wq_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 128 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + } + } + }, + "batch": { + "groups": [ + { + "count": 16, + "q": 1, + "ctx": 16384 + } + ] + } + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro/attn", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn" + ] + }, + { + "type": "dsv4_attn", + "args": { + "hidden": 7168, + "heads": 128, + "head_dim": 512, + "rope_dim": 64, + "q_lora_rank": 1536, + "window": 128, + "o_groups": 16, + "o_lora_rank": 1024, + "compress_rope_theta": 160000, + "rope_scaling": { + "type": "yarn", + "factor": 16, + "original_max": 65536, + "beta_fast": 32, + "beta_slow": 1 + }, + "kv_cache_dtype": "fp8", + "compress_ratio": 4, + "indexer": "own", + "topk": 1024, + "index_heads": 64, + "index_dim": 128, + "index_cache_dtype": "fp8", + "proj": { + "indexer.wq_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 128 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + } + }, + "wkv": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 128 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + } + }, + "wo_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 128 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + } + }, + "wq_a": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 128 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + } + }, + "wq_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 128 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + } + } + }, + "batch": { + "groups": [ + { + "count": 16, + "q": 1, + "ctx": 65536 + } + ] + } + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro/attn", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn" + ] + }, + { + "type": "dsv4_attn", + "args": { + "hidden": 7168, + "heads": 128, + "head_dim": 512, + "rope_dim": 64, + "q_lora_rank": 1536, + "window": 128, + "o_groups": 16, + "o_lora_rank": 1024, + "compress_rope_theta": 160000, + "rope_scaling": { + "type": "yarn", + "factor": 16, + "original_max": 65536, + "beta_fast": 32, + "beta_slow": 1 + }, + "kv_cache_dtype": "fp8", + "compress_ratio": 4, + "indexer": "own", + "topk": 1024, + "index_heads": 64, + "index_dim": 128, + "index_cache_dtype": "fp8", + "proj": { + "indexer.wq_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 128 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + } + }, + "wkv": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 128 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + } + }, + "wo_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 128 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + } + }, + "wq_a": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 128 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + } + }, + "wq_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 128 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + } + } + }, + "batch": { + "groups": [ + { + "count": 16, + "q": 1, + "ctx": 131072 + } + ] + } + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro/attn", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn" + ] + }, + { + "type": "dsv4_attn", + "args": { + "hidden": 7168, + "heads": 128, + "head_dim": 512, + "rope_dim": 64, + "q_lora_rank": 1536, + "window": 128, + "o_groups": 16, + "o_lora_rank": 1024, + "compress_rope_theta": 160000, + "rope_scaling": { + "type": "yarn", + "factor": 16, + "original_max": 65536, + "beta_fast": 32, + "beta_slow": 1 + }, + "kv_cache_dtype": "fp8", + "compress_ratio": 4, + "indexer": "own", + "topk": 1024, + "index_heads": 64, + "index_dim": 128, + "index_cache_dtype": "fp8", + "proj": { + "indexer.wq_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 128 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + } + }, + "wkv": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 128 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + } + }, + "wo_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 128 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + } + }, + "wq_a": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 128 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + } + }, + "wq_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 128 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + } + } + }, + "batch": { + "groups": [ + { + "count": 64, + "q": 1, + "ctx": 1024 + } + ] + } + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro/attn", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn" + ] + }, + { + "type": "dsv4_attn", + "args": { + "hidden": 7168, + "heads": 128, + "head_dim": 512, + "rope_dim": 64, + "q_lora_rank": 1536, + "window": 128, + "o_groups": 16, + "o_lora_rank": 1024, + "compress_rope_theta": 160000, + "rope_scaling": { + "type": "yarn", + "factor": 16, + "original_max": 65536, + "beta_fast": 32, + "beta_slow": 1 + }, + "kv_cache_dtype": "fp8", + "compress_ratio": 4, + "indexer": "own", + "topk": 1024, + "index_heads": 64, + "index_dim": 128, + "index_cache_dtype": "fp8", + "proj": { + "indexer.wq_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 128 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + } + }, + "wkv": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 128 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + } + }, + "wo_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 128 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + } + }, + "wq_a": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 128 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + } + }, + "wq_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 128 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + } + } + }, + "batch": { + "groups": [ + { + "count": 64, + "q": 1, + "ctx": 4096 + } + ] + } + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro/attn", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn" + ] + }, + { + "type": "dsv4_attn", + "args": { + "hidden": 7168, + "heads": 128, + "head_dim": 512, + "rope_dim": 64, + "q_lora_rank": 1536, + "window": 128, + "o_groups": 16, + "o_lora_rank": 1024, + "compress_rope_theta": 160000, + "rope_scaling": { + "type": "yarn", + "factor": 16, + "original_max": 65536, + "beta_fast": 32, + "beta_slow": 1 + }, + "kv_cache_dtype": "fp8", + "compress_ratio": 4, + "indexer": "own", + "topk": 1024, + "index_heads": 64, + "index_dim": 128, + "index_cache_dtype": "fp8", + "proj": { + "indexer.wq_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 128 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + } + }, + "wkv": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 128 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + } + }, + "wo_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 128 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + } + }, + "wq_a": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 128 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + } + }, + "wq_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 128 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + } + } + }, + "batch": { + "groups": [ + { + "count": 64, + "q": 1, + "ctx": 16384 + } + ] + } + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro/attn", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn" + ] + }, + { + "type": "dsv4_attn", + "args": { + "hidden": 7168, + "heads": 128, + "head_dim": 512, + "rope_dim": 64, + "q_lora_rank": 1536, + "window": 128, + "o_groups": 16, + "o_lora_rank": 1024, + "compress_rope_theta": 160000, + "rope_scaling": { + "type": "yarn", + "factor": 16, + "original_max": 65536, + "beta_fast": 32, + "beta_slow": 1 + }, + "kv_cache_dtype": "fp8", + "compress_ratio": 4, + "indexer": "own", + "topk": 1024, + "index_heads": 64, + "index_dim": 128, + "index_cache_dtype": "fp8", + "proj": { + "indexer.wq_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 128 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + } + }, + "wkv": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 128 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + } + }, + "wo_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 128 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + } + }, + "wq_a": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 128 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + } + }, + "wq_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 128 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + } + } + }, + "batch": { + "groups": [ + { + "count": 64, + "q": 1, + "ctx": 65536 + } + ] + } + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro/attn", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn" + ] + }, + { + "type": "dsv4_attn", + "args": { + "hidden": 7168, + "heads": 128, + "head_dim": 512, + "rope_dim": 64, + "q_lora_rank": 1536, + "window": 128, + "o_groups": 16, + "o_lora_rank": 1024, + "compress_rope_theta": 160000, + "rope_scaling": { + "type": "yarn", + "factor": 16, + "original_max": 65536, + "beta_fast": 32, + "beta_slow": 1 + }, + "kv_cache_dtype": "fp8", + "compress_ratio": 4, + "indexer": "own", + "topk": 1024, + "index_heads": 64, + "index_dim": 128, + "index_cache_dtype": "fp8", + "proj": { + "indexer.wq_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 128 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + } + }, + "wkv": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 128 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + } + }, + "wo_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 128 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + } + }, + "wq_a": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 128 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + } + }, + "wq_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 128 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + } + } + }, + "batch": { + "groups": [ + { + "count": 64, + "q": 1, + "ctx": 131072 + } + ] + } + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro/attn", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn" + ] + }, + { + "type": "dsv4_attn", + "args": { + "hidden": 7168, + "heads": 128, + "head_dim": 512, + "rope_dim": 64, + "q_lora_rank": 1536, + "window": 128, + "o_groups": 16, + "o_lora_rank": 1024, + "compress_rope_theta": 160000, + "rope_scaling": { + "type": "yarn", + "factor": 16, + "original_max": 65536, + "beta_fast": 32, + "beta_slow": 1 + }, + "kv_cache_dtype": "fp8", + "compress_ratio": 4, + "indexer": "own", + "topk": 1024, + "index_heads": 64, + "index_dim": 128, + "index_cache_dtype": "fp8", + "proj": { + "indexer.wq_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 128 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + } + }, + "wkv": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 128 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + } + }, + "wo_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 128 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + } + }, + "wq_a": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 128 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + } + }, + "wq_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 128 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + } + } + }, + "batch": { + "groups": [ + { + "count": 256, + "q": 1, + "ctx": 1024 + } + ] + } + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro/attn", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn" + ] + }, + { + "type": "dsv4_attn", + "args": { + "hidden": 7168, + "heads": 128, + "head_dim": 512, + "rope_dim": 64, + "q_lora_rank": 1536, + "window": 128, + "o_groups": 16, + "o_lora_rank": 1024, + "compress_rope_theta": 160000, + "rope_scaling": { + "type": "yarn", + "factor": 16, + "original_max": 65536, + "beta_fast": 32, + "beta_slow": 1 + }, + "kv_cache_dtype": "fp8", + "compress_ratio": 4, + "indexer": "own", + "topk": 1024, + "index_heads": 64, + "index_dim": 128, + "index_cache_dtype": "fp8", + "proj": { + "indexer.wq_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 128 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + } + }, + "wkv": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 128 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + } + }, + "wo_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 128 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + } + }, + "wq_a": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 128 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + } + }, + "wq_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 128 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + } + } + }, + "batch": { + "groups": [ + { + "count": 256, + "q": 1, + "ctx": 4096 + } + ] + } + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro/attn", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn" + ] + }, + { + "type": "dsv4_attn", + "args": { + "hidden": 7168, + "heads": 128, + "head_dim": 512, + "rope_dim": 64, + "q_lora_rank": 1536, + "window": 128, + "o_groups": 16, + "o_lora_rank": 1024, + "compress_rope_theta": 160000, + "rope_scaling": { + "type": "yarn", + "factor": 16, + "original_max": 65536, + "beta_fast": 32, + "beta_slow": 1 + }, + "kv_cache_dtype": "fp8", + "compress_ratio": 4, + "indexer": "own", + "topk": 1024, + "index_heads": 64, + "index_dim": 128, + "index_cache_dtype": "fp8", + "proj": { + "indexer.wq_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 128 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + } + }, + "wkv": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 128 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + } + }, + "wo_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 128 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + } + }, + "wq_a": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 128 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + } + }, + "wq_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 128 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + } + } + }, + "batch": { + "groups": [ + { + "count": 256, + "q": 1, + "ctx": 16384 + } + ] + } + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro/attn", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn" + ] + }, + { + "type": "dsv4_attn", + "args": { + "hidden": 7168, + "heads": 128, + "head_dim": 512, + "rope_dim": 64, + "q_lora_rank": 1536, + "window": 128, + "o_groups": 16, + "o_lora_rank": 1024, + "compress_rope_theta": 160000, + "rope_scaling": { + "type": "yarn", + "factor": 16, + "original_max": 65536, + "beta_fast": 32, + "beta_slow": 1 + }, + "kv_cache_dtype": "fp8", + "compress_ratio": 4, + "indexer": "own", + "topk": 1024, + "index_heads": 64, + "index_dim": 128, + "index_cache_dtype": "fp8", + "proj": { + "indexer.wq_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 128 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + } + }, + "wkv": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 128 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + } + }, + "wo_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 128 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + } + }, + "wq_a": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 128 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + } + }, + "wq_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 128 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + } + } + }, + "batch": { + "groups": [ + { + "count": 256, + "q": 1, + "ctx": 65536 + } + ] + } + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro/attn", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn" + ] + }, + { + "type": "dsv4_attn", + "args": { + "hidden": 7168, + "heads": 128, + "head_dim": 512, + "rope_dim": 64, + "q_lora_rank": 1536, + "window": 128, + "o_groups": 16, + "o_lora_rank": 1024, + "compress_rope_theta": 160000, + "rope_scaling": { + "type": "yarn", + "factor": 16, + "original_max": 65536, + "beta_fast": 32, + "beta_slow": 1 + }, + "kv_cache_dtype": "fp8", + "compress_ratio": 4, + "indexer": "own", + "topk": 1024, + "index_heads": 64, + "index_dim": 128, + "index_cache_dtype": "fp8", + "proj": { + "indexer.wq_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 128 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + } + }, + "wkv": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 128 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + } + }, + "wo_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 128 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + } + }, + "wq_a": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 128 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + } + }, + "wq_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 128 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + } + } + }, + "batch": { + "groups": [ + { + "count": 256, + "q": 1, + "ctx": 131072 + } + ] + } + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro/attn", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn" + ] + }, + { + "type": "dsv4_attn", + "args": { + "hidden": 7168, + "heads": 128, + "head_dim": 512, + "rope_dim": 64, + "q_lora_rank": 1536, + "window": 128, + "o_groups": 16, + "o_lora_rank": 1024, + "compress_rope_theta": 160000, + "rope_scaling": { + "type": "yarn", + "factor": 16, + "original_max": 65536, + "beta_fast": 32, + "beta_slow": 1 + }, + "kv_cache_dtype": "fp8", + "compress_ratio": 4, + "indexer": "own", + "topk": 1024, + "index_heads": 64, + "index_dim": 128, + "index_cache_dtype": "fp8", + "proj": { + "indexer.wq_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 128 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + } + }, + "wkv": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 128 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + } + }, + "wo_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 128 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + } + }, + "wq_a": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 128 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + } + }, + "wq_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 128 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + } + } + }, + "batch": { + "groups": [ + { + "count": 1, + "q": 1024, + "ctx": 0 + } + ] + } + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro/attn", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn" + ] + }, + { + "type": "dsv4_attn", + "args": { + "hidden": 7168, + "heads": 128, + "head_dim": 512, + "rope_dim": 64, + "q_lora_rank": 1536, + "window": 128, + "o_groups": 16, + "o_lora_rank": 1024, + "compress_rope_theta": 160000, + "rope_scaling": { + "type": "yarn", + "factor": 16, + "original_max": 65536, + "beta_fast": 32, + "beta_slow": 1 + }, + "kv_cache_dtype": "fp8", + "compress_ratio": 4, + "indexer": "own", + "topk": 1024, + "index_heads": 64, + "index_dim": 128, + "index_cache_dtype": "fp8", + "proj": { + "indexer.wq_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 128 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + } + }, + "wkv": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 128 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + } + }, + "wo_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 128 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + } + }, + "wq_a": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 128 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + } + }, + "wq_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 128 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + } + } + }, + "batch": { + "groups": [ + { + "count": 1, + "q": 4096, + "ctx": 0 + } + ] + } + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro/attn", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn" + ] + }, + { + "type": "dsv4_attn", + "args": { + "hidden": 7168, + "heads": 128, + "head_dim": 512, + "rope_dim": 64, + "q_lora_rank": 1536, + "window": 128, + "o_groups": 16, + "o_lora_rank": 1024, + "compress_rope_theta": 160000, + "rope_scaling": { + "type": "yarn", + "factor": 16, + "original_max": 65536, + "beta_fast": 32, + "beta_slow": 1 + }, + "kv_cache_dtype": "fp8", + "compress_ratio": 4, + "indexer": "own", + "topk": 1024, + "index_heads": 64, + "index_dim": 128, + "index_cache_dtype": "fp8", + "proj": { + "indexer.wq_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 128 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + } + }, + "wkv": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 128 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + } + }, + "wo_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 128 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + } + }, + "wq_a": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 128 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + } + }, + "wq_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 128 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + } + } + }, + "batch": { + "groups": [ + { + "count": 1, + "q": 16384, + "ctx": 0 + } + ] + } + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro/attn", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn" + ] + }, + { + "type": "dsv4_attn", + "args": { + "hidden": 7168, + "heads": 128, + "head_dim": 512, + "rope_dim": 64, + "q_lora_rank": 1536, + "window": 128, + "o_groups": 16, + "o_lora_rank": 1024, + "compress_rope_theta": 160000, + "rope_scaling": { + "type": "yarn", + "factor": 16, + "original_max": 65536, + "beta_fast": 32, + "beta_slow": 1 + }, + "kv_cache_dtype": "fp8", + "compress_ratio": 4, + "indexer": "own", + "topk": 1024, + "index_heads": 64, + "index_dim": 128, + "index_cache_dtype": "fp8", + "proj": { + "indexer.wq_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 128 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + } + }, + "wkv": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 128 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + } + }, + "wo_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 128 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + } + }, + "wq_a": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 128 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + } + }, + "wq_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 128 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + } + } + }, + "batch": { + "groups": [ + { + "count": 1, + "q": 32768, + "ctx": 0 + } + ] + } + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro/attn", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn" + ] + }, + { + "type": "dsv4_attn", + "args": { + "hidden": 7168, + "heads": 128, + "head_dim": 512, + "rope_dim": 64, + "q_lora_rank": 1536, + "window": 128, + "o_groups": 16, + "o_lora_rank": 1024, + "compress_rope_theta": 160000, + "rope_scaling": { + "type": "yarn", + "factor": 16, + "original_max": 65536, + "beta_fast": 32, + "beta_slow": 1 + }, + "kv_cache_dtype": "fp8", + "compress_ratio": 4, + "indexer": "own", + "topk": 1024, + "index_heads": 64, + "index_dim": 128, + "index_cache_dtype": "fp8", + "proj": { + "indexer.wq_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 128 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + } + }, + "wkv": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 128 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + } + }, + "wo_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 128 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + } + }, + "wq_a": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 128 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + } + }, + "wq_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 128 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + } + } + }, + "batch": { + "groups": [ + { + "count": 1, + "q": 8192, + "ctx": 8192 + } + ] + } + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro/attn", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn" + ] + }, + { + "type": "dsv4_attn", + "args": { + "hidden": 7168, + "heads": 128, + "head_dim": 512, + "rope_dim": 64, + "q_lora_rank": 1536, + "window": 128, + "o_groups": 16, + "o_lora_rank": 1024, + "compress_rope_theta": 160000, + "rope_scaling": { + "type": "yarn", + "factor": 16, + "original_max": 65536, + "beta_fast": 32, + "beta_slow": 1 + }, + "kv_cache_dtype": "fp8", + "compress_ratio": 4, + "indexer": "own", + "topk": 1024, + "index_heads": 64, + "index_dim": 128, + "index_cache_dtype": "fp8", + "proj": { + "indexer.wq_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 128 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + } + }, + "wkv": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 128 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + } + }, + "wo_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 128 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + } + }, + "wq_a": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 128 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + } + }, + "wq_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 128 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + } + } + }, + "batch": { + "groups": [ + { + "count": 1, + "q": 8192, + "ctx": 32768 + } + ] + } + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro/attn", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn" + ] + }, + { + "type": "dsv4_attn", + "args": { + "hidden": 7168, + "heads": 128, + "head_dim": 512, + "rope_dim": 64, + "q_lora_rank": 1536, + "window": 128, + "o_groups": 16, + "o_lora_rank": 1024, + "compress_rope_theta": 160000, + "rope_scaling": { + "type": "yarn", + "factor": 16, + "original_max": 65536, + "beta_fast": 32, + "beta_slow": 1 + }, + "kv_cache_dtype": "fp8", + "compress_ratio": 4, + "indexer": "own", + "topk": 1024, + "index_heads": 64, + "index_dim": 128, + "index_cache_dtype": "fp8", + "proj": { + "indexer.wq_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 128 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + } + }, + "wkv": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 128 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + } + }, + "wo_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 128 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + } + }, + "wq_a": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 128 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + } + }, + "wq_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 128 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + } + } + }, + "batch": { + "groups": [ + { + "count": 1, + "q": 8192, + "ctx": 131072 + } + ] + } + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro/attn", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn" + ] + }, + { + "type": "dsv4_attn", + "args": { + "hidden": 7168, + "heads": 128, + "head_dim": 512, + "rope_dim": 64, + "q_lora_rank": 1536, + "window": 128, + "o_groups": 16, + "o_lora_rank": 1024, + "compress_rope_theta": 160000, + "rope_scaling": { + "type": "yarn", + "factor": 16, + "original_max": 65536, + "beta_fast": 32, + "beta_slow": 1 + }, + "kv_cache_dtype": "fp8", + "compress_ratio": 4, + "indexer": "own", + "topk": 1024, + "index_heads": 64, + "index_dim": 128, + "index_cache_dtype": "fp8", + "proj": { + "indexer.wq_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 128 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + } + }, + "wkv": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 128 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + } + }, + "wo_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 128 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + } + }, + "wq_a": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 128 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + } + }, + "wq_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 128 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + } + } + }, + "batch": { + "groups": [ + { + "count": 1, + "q": 8192, + "ctx": 8192 + }, + { + "count": 64, + "q": 1, + "ctx": 8192 + } + ] + } + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro/attn", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn" + ] + }, + { + "type": "dsv4_attn", + "args": { + "hidden": 7168, + "heads": 128, + "head_dim": 512, + "rope_dim": 64, + "q_lora_rank": 1536, + "window": 128, + "o_groups": 16, + "o_lora_rank": 1024, + "compress_rope_theta": 160000, + "rope_scaling": { + "type": "yarn", + "factor": 16, + "original_max": 65536, + "beta_fast": 32, + "beta_slow": 1 + }, + "kv_cache_dtype": "fp8", + "compress_ratio": 4, + "indexer": "own", + "topk": 1024, + "index_heads": 64, + "index_dim": 128, + "index_cache_dtype": "mxfp4", + "proj": { + "indexer.wq_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 128 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + } + }, + "wkv": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 128 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + } + }, + "wo_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 128 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + } + }, + "wq_a": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 128 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + } + }, + "wq_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 128 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + } + } + }, + "batch": { + "groups": [ + { + "count": 1, + "q": 1, + "ctx": 1024 + } + ] + } + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro/attn", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn" + ] + }, + { + "type": "dsv4_attn", + "args": { + "hidden": 7168, + "heads": 128, + "head_dim": 512, + "rope_dim": 64, + "q_lora_rank": 1536, + "window": 128, + "o_groups": 16, + "o_lora_rank": 1024, + "compress_rope_theta": 160000, + "rope_scaling": { + "type": "yarn", + "factor": 16, + "original_max": 65536, + "beta_fast": 32, + "beta_slow": 1 + }, + "kv_cache_dtype": "fp8", + "compress_ratio": 4, + "indexer": "own", + "topk": 1024, + "index_heads": 64, + "index_dim": 128, + "index_cache_dtype": "mxfp4", + "proj": { + "indexer.wq_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 128 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + } + }, + "wkv": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 128 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + } + }, + "wo_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 128 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + } + }, + "wq_a": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 128 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + } + }, + "wq_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 128 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + } + } + }, + "batch": { + "groups": [ + { + "count": 1, + "q": 1, + "ctx": 4096 + } + ] + } + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro/attn", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn" + ] + }, + { + "type": "dsv4_attn", + "args": { + "hidden": 7168, + "heads": 128, + "head_dim": 512, + "rope_dim": 64, + "q_lora_rank": 1536, + "window": 128, + "o_groups": 16, + "o_lora_rank": 1024, + "compress_rope_theta": 160000, + "rope_scaling": { + "type": "yarn", + "factor": 16, + "original_max": 65536, + "beta_fast": 32, + "beta_slow": 1 + }, + "kv_cache_dtype": "fp8", + "compress_ratio": 4, + "indexer": "own", + "topk": 1024, + "index_heads": 64, + "index_dim": 128, + "index_cache_dtype": "mxfp4", + "proj": { + "indexer.wq_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 128 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + } + }, + "wkv": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 128 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + } + }, + "wo_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 128 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + } + }, + "wq_a": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 128 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + } + }, + "wq_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 128 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + } + } + }, + "batch": { + "groups": [ + { + "count": 1, + "q": 1, + "ctx": 16384 + } + ] + } + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro/attn", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn" + ] + }, + { + "type": "dsv4_attn", + "args": { + "hidden": 7168, + "heads": 128, + "head_dim": 512, + "rope_dim": 64, + "q_lora_rank": 1536, + "window": 128, + "o_groups": 16, + "o_lora_rank": 1024, + "compress_rope_theta": 160000, + "rope_scaling": { + "type": "yarn", + "factor": 16, + "original_max": 65536, + "beta_fast": 32, + "beta_slow": 1 + }, + "kv_cache_dtype": "fp8", + "compress_ratio": 4, + "indexer": "own", + "topk": 1024, + "index_heads": 64, + "index_dim": 128, + "index_cache_dtype": "mxfp4", + "proj": { + "indexer.wq_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 128 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + } + }, + "wkv": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 128 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + } + }, + "wo_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 128 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + } + }, + "wq_a": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 128 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + } + }, + "wq_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 128 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + } + } + }, + "batch": { + "groups": [ + { + "count": 1, + "q": 1, + "ctx": 65536 + } + ] + } + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro/attn", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn" + ] + }, + { + "type": "dsv4_attn", + "args": { + "hidden": 7168, + "heads": 128, + "head_dim": 512, + "rope_dim": 64, + "q_lora_rank": 1536, + "window": 128, + "o_groups": 16, + "o_lora_rank": 1024, + "compress_rope_theta": 160000, + "rope_scaling": { + "type": "yarn", + "factor": 16, + "original_max": 65536, + "beta_fast": 32, + "beta_slow": 1 + }, + "kv_cache_dtype": "fp8", + "compress_ratio": 4, + "indexer": "own", + "topk": 1024, + "index_heads": 64, + "index_dim": 128, + "index_cache_dtype": "mxfp4", + "proj": { + "indexer.wq_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 128 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + } + }, + "wkv": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 128 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + } + }, + "wo_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 128 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + } + }, + "wq_a": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 128 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + } + }, + "wq_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 128 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + } + } + }, + "batch": { + "groups": [ + { + "count": 1, + "q": 1, + "ctx": 131072 + } + ] + } + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro/attn", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn" + ] + }, + { + "type": "dsv4_attn", + "args": { + "hidden": 7168, + "heads": 128, + "head_dim": 512, + "rope_dim": 64, + "q_lora_rank": 1536, + "window": 128, + "o_groups": 16, + "o_lora_rank": 1024, + "compress_rope_theta": 160000, + "rope_scaling": { + "type": "yarn", + "factor": 16, + "original_max": 65536, + "beta_fast": 32, + "beta_slow": 1 + }, + "kv_cache_dtype": "fp8", + "compress_ratio": 4, + "indexer": "own", + "topk": 1024, + "index_heads": 64, + "index_dim": 128, + "index_cache_dtype": "mxfp4", + "proj": { + "indexer.wq_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 128 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + } + }, + "wkv": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 128 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + } + }, + "wo_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 128 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + } + }, + "wq_a": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 128 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + } + }, + "wq_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 128 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + } + } + }, + "batch": { + "groups": [ + { + "count": 4, + "q": 1, + "ctx": 1024 + } + ] + } + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro/attn", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn" + ] + }, + { + "type": "dsv4_attn", + "args": { + "hidden": 7168, + "heads": 128, + "head_dim": 512, + "rope_dim": 64, + "q_lora_rank": 1536, + "window": 128, + "o_groups": 16, + "o_lora_rank": 1024, + "compress_rope_theta": 160000, + "rope_scaling": { + "type": "yarn", + "factor": 16, + "original_max": 65536, + "beta_fast": 32, + "beta_slow": 1 + }, + "kv_cache_dtype": "fp8", + "compress_ratio": 4, + "indexer": "own", + "topk": 1024, + "index_heads": 64, + "index_dim": 128, + "index_cache_dtype": "mxfp4", + "proj": { + "indexer.wq_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 128 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + } + }, + "wkv": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 128 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + } + }, + "wo_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 128 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + } + }, + "wq_a": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 128 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + } + }, + "wq_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 128 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + } + } + }, + "batch": { + "groups": [ + { + "count": 4, + "q": 1, + "ctx": 4096 + } + ] + } + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro/attn", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn" + ] + }, + { + "type": "dsv4_attn", + "args": { + "hidden": 7168, + "heads": 128, + "head_dim": 512, + "rope_dim": 64, + "q_lora_rank": 1536, + "window": 128, + "o_groups": 16, + "o_lora_rank": 1024, + "compress_rope_theta": 160000, + "rope_scaling": { + "type": "yarn", + "factor": 16, + "original_max": 65536, + "beta_fast": 32, + "beta_slow": 1 + }, + "kv_cache_dtype": "fp8", + "compress_ratio": 4, + "indexer": "own", + "topk": 1024, + "index_heads": 64, + "index_dim": 128, + "index_cache_dtype": "mxfp4", + "proj": { + "indexer.wq_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 128 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + } + }, + "wkv": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 128 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + } + }, + "wo_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 128 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + } + }, + "wq_a": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 128 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + } + }, + "wq_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 128 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + } + } + }, + "batch": { + "groups": [ + { + "count": 4, + "q": 1, + "ctx": 16384 + } + ] + } + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro/attn", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn" + ] + }, + { + "type": "dsv4_attn", + "args": { + "hidden": 7168, + "heads": 128, + "head_dim": 512, + "rope_dim": 64, + "q_lora_rank": 1536, + "window": 128, + "o_groups": 16, + "o_lora_rank": 1024, + "compress_rope_theta": 160000, + "rope_scaling": { + "type": "yarn", + "factor": 16, + "original_max": 65536, + "beta_fast": 32, + "beta_slow": 1 + }, + "kv_cache_dtype": "fp8", + "compress_ratio": 4, + "indexer": "own", + "topk": 1024, + "index_heads": 64, + "index_dim": 128, + "index_cache_dtype": "mxfp4", + "proj": { + "indexer.wq_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 128 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + } + }, + "wkv": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 128 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + } + }, + "wo_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 128 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + } + }, + "wq_a": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 128 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + } + }, + "wq_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 128 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + } + } + }, + "batch": { + "groups": [ + { + "count": 4, + "q": 1, + "ctx": 65536 + } + ] + } + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro/attn", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn" + ] + }, + { + "type": "dsv4_attn", + "args": { + "hidden": 7168, + "heads": 128, + "head_dim": 512, + "rope_dim": 64, + "q_lora_rank": 1536, + "window": 128, + "o_groups": 16, + "o_lora_rank": 1024, + "compress_rope_theta": 160000, + "rope_scaling": { + "type": "yarn", + "factor": 16, + "original_max": 65536, + "beta_fast": 32, + "beta_slow": 1 + }, + "kv_cache_dtype": "fp8", + "compress_ratio": 4, + "indexer": "own", + "topk": 1024, + "index_heads": 64, + "index_dim": 128, + "index_cache_dtype": "mxfp4", + "proj": { + "indexer.wq_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 128 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + } + }, + "wkv": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 128 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + } + }, + "wo_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 128 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + } + }, + "wq_a": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 128 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + } + }, + "wq_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 128 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + } + } + }, + "batch": { + "groups": [ + { + "count": 4, + "q": 1, + "ctx": 131072 + } + ] + } + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro/attn", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn" + ] + }, + { + "type": "dsv4_attn", + "args": { + "hidden": 7168, + "heads": 128, + "head_dim": 512, + "rope_dim": 64, + "q_lora_rank": 1536, + "window": 128, + "o_groups": 16, + "o_lora_rank": 1024, + "compress_rope_theta": 160000, + "rope_scaling": { + "type": "yarn", + "factor": 16, + "original_max": 65536, + "beta_fast": 32, + "beta_slow": 1 + }, + "kv_cache_dtype": "fp8", + "compress_ratio": 4, + "indexer": "own", + "topk": 1024, + "index_heads": 64, + "index_dim": 128, + "index_cache_dtype": "mxfp4", + "proj": { + "indexer.wq_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 128 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + } + }, + "wkv": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 128 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + } + }, + "wo_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 128 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + } + }, + "wq_a": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 128 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + } + }, + "wq_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 128 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + } + } + }, + "batch": { + "groups": [ + { + "count": 16, + "q": 1, + "ctx": 1024 + } + ] + } + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro/attn", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn" + ] + }, + { + "type": "dsv4_attn", + "args": { + "hidden": 7168, + "heads": 128, + "head_dim": 512, + "rope_dim": 64, + "q_lora_rank": 1536, + "window": 128, + "o_groups": 16, + "o_lora_rank": 1024, + "compress_rope_theta": 160000, + "rope_scaling": { + "type": "yarn", + "factor": 16, + "original_max": 65536, + "beta_fast": 32, + "beta_slow": 1 + }, + "kv_cache_dtype": "fp8", + "compress_ratio": 4, + "indexer": "own", + "topk": 1024, + "index_heads": 64, + "index_dim": 128, + "index_cache_dtype": "mxfp4", + "proj": { + "indexer.wq_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 128 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + } + }, + "wkv": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 128 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + } + }, + "wo_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 128 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + } + }, + "wq_a": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 128 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + } + }, + "wq_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 128 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + } + } + }, + "batch": { + "groups": [ + { + "count": 16, + "q": 1, + "ctx": 4096 + } + ] + } + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro/attn", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn" + ] + }, + { + "type": "dsv4_attn", + "args": { + "hidden": 7168, + "heads": 128, + "head_dim": 512, + "rope_dim": 64, + "q_lora_rank": 1536, + "window": 128, + "o_groups": 16, + "o_lora_rank": 1024, + "compress_rope_theta": 160000, + "rope_scaling": { + "type": "yarn", + "factor": 16, + "original_max": 65536, + "beta_fast": 32, + "beta_slow": 1 + }, + "kv_cache_dtype": "fp8", + "compress_ratio": 4, + "indexer": "own", + "topk": 1024, + "index_heads": 64, + "index_dim": 128, + "index_cache_dtype": "mxfp4", + "proj": { + "indexer.wq_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 128 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + } + }, + "wkv": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 128 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + } + }, + "wo_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 128 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + } + }, + "wq_a": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 128 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + } + }, + "wq_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 128 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + } + } + }, + "batch": { + "groups": [ + { + "count": 16, + "q": 1, + "ctx": 16384 + } + ] + } + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro/attn", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn" + ] + }, + { + "type": "dsv4_attn", + "args": { + "hidden": 7168, + "heads": 128, + "head_dim": 512, + "rope_dim": 64, + "q_lora_rank": 1536, + "window": 128, + "o_groups": 16, + "o_lora_rank": 1024, + "compress_rope_theta": 160000, + "rope_scaling": { + "type": "yarn", + "factor": 16, + "original_max": 65536, + "beta_fast": 32, + "beta_slow": 1 + }, + "kv_cache_dtype": "fp8", + "compress_ratio": 4, + "indexer": "own", + "topk": 1024, + "index_heads": 64, + "index_dim": 128, + "index_cache_dtype": "mxfp4", + "proj": { + "indexer.wq_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 128 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + } + }, + "wkv": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 128 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + } + }, + "wo_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 128 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + } + }, + "wq_a": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 128 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + } + }, + "wq_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 128 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + } + } + }, + "batch": { + "groups": [ + { + "count": 16, + "q": 1, + "ctx": 65536 + } + ] + } + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro/attn", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn" + ] + }, + { + "type": "dsv4_attn", + "args": { + "hidden": 7168, + "heads": 128, + "head_dim": 512, + "rope_dim": 64, + "q_lora_rank": 1536, + "window": 128, + "o_groups": 16, + "o_lora_rank": 1024, + "compress_rope_theta": 160000, + "rope_scaling": { + "type": "yarn", + "factor": 16, + "original_max": 65536, + "beta_fast": 32, + "beta_slow": 1 + }, + "kv_cache_dtype": "fp8", + "compress_ratio": 4, + "indexer": "own", + "topk": 1024, + "index_heads": 64, + "index_dim": 128, + "index_cache_dtype": "mxfp4", + "proj": { + "indexer.wq_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 128 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + } + }, + "wkv": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 128 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + } + }, + "wo_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 128 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + } + }, + "wq_a": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 128 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + } + }, + "wq_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 128 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + } + } + }, + "batch": { + "groups": [ + { + "count": 16, + "q": 1, + "ctx": 131072 + } + ] + } + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro/attn", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn" + ] + }, + { + "type": "dsv4_attn", + "args": { + "hidden": 7168, + "heads": 128, + "head_dim": 512, + "rope_dim": 64, + "q_lora_rank": 1536, + "window": 128, + "o_groups": 16, + "o_lora_rank": 1024, + "compress_rope_theta": 160000, + "rope_scaling": { + "type": "yarn", + "factor": 16, + "original_max": 65536, + "beta_fast": 32, + "beta_slow": 1 + }, + "kv_cache_dtype": "fp8", + "compress_ratio": 4, + "indexer": "own", + "topk": 1024, + "index_heads": 64, + "index_dim": 128, + "index_cache_dtype": "mxfp4", + "proj": { + "indexer.wq_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 128 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + } + }, + "wkv": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 128 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + } + }, + "wo_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 128 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + } + }, + "wq_a": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 128 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + } + }, + "wq_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 128 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + } + } + }, + "batch": { + "groups": [ + { + "count": 64, + "q": 1, + "ctx": 1024 + } + ] + } + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro/attn", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn" + ] + }, + { + "type": "dsv4_attn", + "args": { + "hidden": 7168, + "heads": 128, + "head_dim": 512, + "rope_dim": 64, + "q_lora_rank": 1536, + "window": 128, + "o_groups": 16, + "o_lora_rank": 1024, + "compress_rope_theta": 160000, + "rope_scaling": { + "type": "yarn", + "factor": 16, + "original_max": 65536, + "beta_fast": 32, + "beta_slow": 1 + }, + "kv_cache_dtype": "fp8", + "compress_ratio": 4, + "indexer": "own", + "topk": 1024, + "index_heads": 64, + "index_dim": 128, + "index_cache_dtype": "mxfp4", + "proj": { + "indexer.wq_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 128 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + } + }, + "wkv": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 128 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + } + }, + "wo_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 128 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + } + }, + "wq_a": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 128 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + } + }, + "wq_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 128 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + } + } + }, + "batch": { + "groups": [ + { + "count": 64, + "q": 1, + "ctx": 4096 + } + ] + } + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro/attn", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn" + ] + }, + { + "type": "dsv4_attn", + "args": { + "hidden": 7168, + "heads": 128, + "head_dim": 512, + "rope_dim": 64, + "q_lora_rank": 1536, + "window": 128, + "o_groups": 16, + "o_lora_rank": 1024, + "compress_rope_theta": 160000, + "rope_scaling": { + "type": "yarn", + "factor": 16, + "original_max": 65536, + "beta_fast": 32, + "beta_slow": 1 + }, + "kv_cache_dtype": "fp8", + "compress_ratio": 4, + "indexer": "own", + "topk": 1024, + "index_heads": 64, + "index_dim": 128, + "index_cache_dtype": "mxfp4", + "proj": { + "indexer.wq_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 128 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + } + }, + "wkv": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 128 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + } + }, + "wo_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 128 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + } + }, + "wq_a": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 128 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + } + }, + "wq_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 128 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + } + } + }, + "batch": { + "groups": [ + { + "count": 64, + "q": 1, + "ctx": 16384 + } + ] + } + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro/attn", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn" + ] + }, + { + "type": "dsv4_attn", + "args": { + "hidden": 7168, + "heads": 128, + "head_dim": 512, + "rope_dim": 64, + "q_lora_rank": 1536, + "window": 128, + "o_groups": 16, + "o_lora_rank": 1024, + "compress_rope_theta": 160000, + "rope_scaling": { + "type": "yarn", + "factor": 16, + "original_max": 65536, + "beta_fast": 32, + "beta_slow": 1 + }, + "kv_cache_dtype": "fp8", + "compress_ratio": 4, + "indexer": "own", + "topk": 1024, + "index_heads": 64, + "index_dim": 128, + "index_cache_dtype": "mxfp4", + "proj": { + "indexer.wq_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 128 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + } + }, + "wkv": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 128 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + } + }, + "wo_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 128 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + } + }, + "wq_a": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 128 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + } + }, + "wq_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 128 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + } + } + }, + "batch": { + "groups": [ + { + "count": 64, + "q": 1, + "ctx": 65536 + } + ] + } + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro/attn", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn" + ] + }, + { + "type": "dsv4_attn", + "args": { + "hidden": 7168, + "heads": 128, + "head_dim": 512, + "rope_dim": 64, + "q_lora_rank": 1536, + "window": 128, + "o_groups": 16, + "o_lora_rank": 1024, + "compress_rope_theta": 160000, + "rope_scaling": { + "type": "yarn", + "factor": 16, + "original_max": 65536, + "beta_fast": 32, + "beta_slow": 1 + }, + "kv_cache_dtype": "fp8", + "compress_ratio": 4, + "indexer": "own", + "topk": 1024, + "index_heads": 64, + "index_dim": 128, + "index_cache_dtype": "mxfp4", + "proj": { + "indexer.wq_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 128 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + } + }, + "wkv": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 128 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + } + }, + "wo_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 128 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + } + }, + "wq_a": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 128 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + } + }, + "wq_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 128 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + } + } + }, + "batch": { + "groups": [ + { + "count": 64, + "q": 1, + "ctx": 131072 + } + ] + } + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro/attn", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn" + ] + }, + { + "type": "dsv4_attn", + "args": { + "hidden": 7168, + "heads": 128, + "head_dim": 512, + "rope_dim": 64, + "q_lora_rank": 1536, + "window": 128, + "o_groups": 16, + "o_lora_rank": 1024, + "compress_rope_theta": 160000, + "rope_scaling": { + "type": "yarn", + "factor": 16, + "original_max": 65536, + "beta_fast": 32, + "beta_slow": 1 + }, + "kv_cache_dtype": "fp8", + "compress_ratio": 4, + "indexer": "own", + "topk": 1024, + "index_heads": 64, + "index_dim": 128, + "index_cache_dtype": "mxfp4", + "proj": { + "indexer.wq_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 128 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + } + }, + "wkv": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 128 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + } + }, + "wo_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 128 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + } + }, + "wq_a": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 128 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + } + }, + "wq_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 128 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + } + } + }, + "batch": { + "groups": [ + { + "count": 256, + "q": 1, + "ctx": 1024 + } + ] + } + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro/attn", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn" + ] + }, + { + "type": "dsv4_attn", + "args": { + "hidden": 7168, + "heads": 128, + "head_dim": 512, + "rope_dim": 64, + "q_lora_rank": 1536, + "window": 128, + "o_groups": 16, + "o_lora_rank": 1024, + "compress_rope_theta": 160000, + "rope_scaling": { + "type": "yarn", + "factor": 16, + "original_max": 65536, + "beta_fast": 32, + "beta_slow": 1 + }, + "kv_cache_dtype": "fp8", + "compress_ratio": 4, + "indexer": "own", + "topk": 1024, + "index_heads": 64, + "index_dim": 128, + "index_cache_dtype": "mxfp4", + "proj": { + "indexer.wq_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 128 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + } + }, + "wkv": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 128 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + } + }, + "wo_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 128 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + } + }, + "wq_a": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 128 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + } + }, + "wq_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 128 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + } + } + }, + "batch": { + "groups": [ + { + "count": 256, + "q": 1, + "ctx": 4096 + } + ] + } + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro/attn", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn" + ] + }, + { + "type": "dsv4_attn", + "args": { + "hidden": 7168, + "heads": 128, + "head_dim": 512, + "rope_dim": 64, + "q_lora_rank": 1536, + "window": 128, + "o_groups": 16, + "o_lora_rank": 1024, + "compress_rope_theta": 160000, + "rope_scaling": { + "type": "yarn", + "factor": 16, + "original_max": 65536, + "beta_fast": 32, + "beta_slow": 1 + }, + "kv_cache_dtype": "fp8", + "compress_ratio": 4, + "indexer": "own", + "topk": 1024, + "index_heads": 64, + "index_dim": 128, + "index_cache_dtype": "mxfp4", + "proj": { + "indexer.wq_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 128 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + } + }, + "wkv": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 128 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + } + }, + "wo_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 128 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + } + }, + "wq_a": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 128 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + } + }, + "wq_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 128 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + } + } + }, + "batch": { + "groups": [ + { + "count": 256, + "q": 1, + "ctx": 16384 + } + ] + } + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro/attn", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn" + ] + }, + { + "type": "dsv4_attn", + "args": { + "hidden": 7168, + "heads": 128, + "head_dim": 512, + "rope_dim": 64, + "q_lora_rank": 1536, + "window": 128, + "o_groups": 16, + "o_lora_rank": 1024, + "compress_rope_theta": 160000, + "rope_scaling": { + "type": "yarn", + "factor": 16, + "original_max": 65536, + "beta_fast": 32, + "beta_slow": 1 + }, + "kv_cache_dtype": "fp8", + "compress_ratio": 4, + "indexer": "own", + "topk": 1024, + "index_heads": 64, + "index_dim": 128, + "index_cache_dtype": "mxfp4", + "proj": { + "indexer.wq_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 128 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + } + }, + "wkv": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 128 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + } + }, + "wo_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 128 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + } + }, + "wq_a": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 128 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + } + }, + "wq_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 128 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + } + } + }, + "batch": { + "groups": [ + { + "count": 256, + "q": 1, + "ctx": 65536 + } + ] + } + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro/attn", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn" + ] + }, + { + "type": "dsv4_attn", + "args": { + "hidden": 7168, + "heads": 128, + "head_dim": 512, + "rope_dim": 64, + "q_lora_rank": 1536, + "window": 128, + "o_groups": 16, + "o_lora_rank": 1024, + "compress_rope_theta": 160000, + "rope_scaling": { + "type": "yarn", + "factor": 16, + "original_max": 65536, + "beta_fast": 32, + "beta_slow": 1 + }, + "kv_cache_dtype": "fp8", + "compress_ratio": 4, + "indexer": "own", + "topk": 1024, + "index_heads": 64, + "index_dim": 128, + "index_cache_dtype": "mxfp4", + "proj": { + "indexer.wq_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 128 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + } + }, + "wkv": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 128 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + } + }, + "wo_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 128 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + } + }, + "wq_a": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 128 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + } + }, + "wq_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 128 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + } + } + }, + "batch": { + "groups": [ + { + "count": 256, + "q": 1, + "ctx": 131072 + } + ] + } + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro/attn", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn" + ] + }, + { + "type": "dsv4_attn", + "args": { + "hidden": 7168, + "heads": 128, + "head_dim": 512, + "rope_dim": 64, + "q_lora_rank": 1536, + "window": 128, + "o_groups": 16, + "o_lora_rank": 1024, + "compress_rope_theta": 160000, + "rope_scaling": { + "type": "yarn", + "factor": 16, + "original_max": 65536, + "beta_fast": 32, + "beta_slow": 1 + }, + "kv_cache_dtype": "fp8", + "compress_ratio": 4, + "indexer": "own", + "topk": 1024, + "index_heads": 64, + "index_dim": 128, + "index_cache_dtype": "mxfp4", + "proj": { + "indexer.wq_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 128 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + } + }, + "wkv": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 128 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + } + }, + "wo_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 128 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + } + }, + "wq_a": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 128 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + } + }, + "wq_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 128 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + } + } + }, + "batch": { + "groups": [ + { + "count": 1, + "q": 1024, + "ctx": 0 + } + ] + } + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro/attn", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn" + ] + }, + { + "type": "dsv4_attn", + "args": { + "hidden": 7168, + "heads": 128, + "head_dim": 512, + "rope_dim": 64, + "q_lora_rank": 1536, + "window": 128, + "o_groups": 16, + "o_lora_rank": 1024, + "compress_rope_theta": 160000, + "rope_scaling": { + "type": "yarn", + "factor": 16, + "original_max": 65536, + "beta_fast": 32, + "beta_slow": 1 + }, + "kv_cache_dtype": "fp8", + "compress_ratio": 4, + "indexer": "own", + "topk": 1024, + "index_heads": 64, + "index_dim": 128, + "index_cache_dtype": "mxfp4", + "proj": { + "indexer.wq_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 128 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + } + }, + "wkv": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 128 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + } + }, + "wo_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 128 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + } + }, + "wq_a": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 128 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + } + }, + "wq_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 128 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + } + } + }, + "batch": { + "groups": [ + { + "count": 1, + "q": 4096, + "ctx": 0 + } + ] + } + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro/attn", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn" + ] + }, + { + "type": "dsv4_attn", + "args": { + "hidden": 7168, + "heads": 128, + "head_dim": 512, + "rope_dim": 64, + "q_lora_rank": 1536, + "window": 128, + "o_groups": 16, + "o_lora_rank": 1024, + "compress_rope_theta": 160000, + "rope_scaling": { + "type": "yarn", + "factor": 16, + "original_max": 65536, + "beta_fast": 32, + "beta_slow": 1 + }, + "kv_cache_dtype": "fp8", + "compress_ratio": 4, + "indexer": "own", + "topk": 1024, + "index_heads": 64, + "index_dim": 128, + "index_cache_dtype": "mxfp4", + "proj": { + "indexer.wq_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 128 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + } + }, + "wkv": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 128 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + } + }, + "wo_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 128 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + } + }, + "wq_a": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 128 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + } + }, + "wq_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 128 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + } + } + }, + "batch": { + "groups": [ + { + "count": 1, + "q": 16384, + "ctx": 0 + } + ] + } + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro/attn", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn" + ] + }, + { + "type": "dsv4_attn", + "args": { + "hidden": 7168, + "heads": 128, + "head_dim": 512, + "rope_dim": 64, + "q_lora_rank": 1536, + "window": 128, + "o_groups": 16, + "o_lora_rank": 1024, + "compress_rope_theta": 160000, + "rope_scaling": { + "type": "yarn", + "factor": 16, + "original_max": 65536, + "beta_fast": 32, + "beta_slow": 1 + }, + "kv_cache_dtype": "fp8", + "compress_ratio": 4, + "indexer": "own", + "topk": 1024, + "index_heads": 64, + "index_dim": 128, + "index_cache_dtype": "mxfp4", + "proj": { + "indexer.wq_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 128 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + } + }, + "wkv": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 128 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + } + }, + "wo_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 128 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + } + }, + "wq_a": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 128 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + } + }, + "wq_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 128 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + } + } + }, + "batch": { + "groups": [ + { + "count": 1, + "q": 32768, + "ctx": 0 + } + ] + } + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro/attn", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn" + ] + }, + { + "type": "dsv4_attn", + "args": { + "hidden": 7168, + "heads": 128, + "head_dim": 512, + "rope_dim": 64, + "q_lora_rank": 1536, + "window": 128, + "o_groups": 16, + "o_lora_rank": 1024, + "compress_rope_theta": 160000, + "rope_scaling": { + "type": "yarn", + "factor": 16, + "original_max": 65536, + "beta_fast": 32, + "beta_slow": 1 + }, + "kv_cache_dtype": "fp8", + "compress_ratio": 4, + "indexer": "own", + "topk": 1024, + "index_heads": 64, + "index_dim": 128, + "index_cache_dtype": "mxfp4", + "proj": { + "indexer.wq_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 128 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + } + }, + "wkv": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 128 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + } + }, + "wo_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 128 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + } + }, + "wq_a": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 128 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + } + }, + "wq_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 128 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + } + } + }, + "batch": { + "groups": [ + { + "count": 1, + "q": 8192, + "ctx": 8192 + } + ] + } + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro/attn", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn" + ] + }, + { + "type": "dsv4_attn", + "args": { + "hidden": 7168, + "heads": 128, + "head_dim": 512, + "rope_dim": 64, + "q_lora_rank": 1536, + "window": 128, + "o_groups": 16, + "o_lora_rank": 1024, + "compress_rope_theta": 160000, + "rope_scaling": { + "type": "yarn", + "factor": 16, + "original_max": 65536, + "beta_fast": 32, + "beta_slow": 1 + }, + "kv_cache_dtype": "fp8", + "compress_ratio": 4, + "indexer": "own", + "topk": 1024, + "index_heads": 64, + "index_dim": 128, + "index_cache_dtype": "mxfp4", + "proj": { + "indexer.wq_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 128 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + } + }, + "wkv": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 128 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + } + }, + "wo_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 128 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + } + }, + "wq_a": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 128 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + } + }, + "wq_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 128 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + } + } + }, + "batch": { + "groups": [ + { + "count": 1, + "q": 8192, + "ctx": 32768 + } + ] + } + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro/attn", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn" + ] + }, + { + "type": "dsv4_attn", + "args": { + "hidden": 7168, + "heads": 128, + "head_dim": 512, + "rope_dim": 64, + "q_lora_rank": 1536, + "window": 128, + "o_groups": 16, + "o_lora_rank": 1024, + "compress_rope_theta": 160000, + "rope_scaling": { + "type": "yarn", + "factor": 16, + "original_max": 65536, + "beta_fast": 32, + "beta_slow": 1 + }, + "kv_cache_dtype": "fp8", + "compress_ratio": 4, + "indexer": "own", + "topk": 1024, + "index_heads": 64, + "index_dim": 128, + "index_cache_dtype": "mxfp4", + "proj": { + "indexer.wq_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 128 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + } + }, + "wkv": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 128 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + } + }, + "wo_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 128 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + } + }, + "wq_a": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 128 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + } + }, + "wq_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 128 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + } + } + }, + "batch": { + "groups": [ + { + "count": 1, + "q": 8192, + "ctx": 131072 + } + ] + } + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro/attn", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn" + ] + }, + { + "type": "dsv4_attn", + "args": { + "hidden": 7168, + "heads": 128, + "head_dim": 512, + "rope_dim": 64, + "q_lora_rank": 1536, + "window": 128, + "o_groups": 16, + "o_lora_rank": 1024, + "compress_rope_theta": 160000, + "rope_scaling": { + "type": "yarn", + "factor": 16, + "original_max": 65536, + "beta_fast": 32, + "beta_slow": 1 + }, + "kv_cache_dtype": "fp8", + "compress_ratio": 4, + "indexer": "own", + "topk": 1024, + "index_heads": 64, + "index_dim": 128, + "index_cache_dtype": "mxfp4", + "proj": { + "indexer.wq_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 128 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + } + }, + "wkv": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 128 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + } + }, + "wo_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 128 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + } + }, + "wq_a": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 128 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + } + }, + "wq_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 128 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + } + } + }, + "batch": { + "groups": [ + { + "count": 1, + "q": 8192, + "ctx": 8192 + }, + { + "count": 64, + "q": 1, + "ctx": 8192 + } + ] + } + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro/attn", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn" + ] + }, + { + "type": "dsv4_attn", + "args": { + "hidden": 7168, + "heads": 128, + "head_dim": 512, + "rope_dim": 64, + "q_lora_rank": 1536, + "window": 128, + "o_groups": 16, + "o_lora_rank": 1024, + "compress_rope_theta": 160000, + "rope_scaling": { + "type": "yarn", + "factor": 16, + "original_max": 65536, + "beta_fast": 32, + "beta_slow": 1 + }, + "kv_cache_dtype": "fp8", + "compress_ratio": 128, + "proj": { + "wkv": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 128 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + } + }, + "wo_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 128 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + } + }, + "wq_a": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 128 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + } + }, + "wq_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 128 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + } + } + }, + "batch": { + "groups": [ + { + "count": 1, + "q": 1, + "ctx": 1024 + } + ] + } + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro/attn", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn" + ] + }, + { + "type": "dsv4_attn", + "args": { + "hidden": 7168, + "heads": 128, + "head_dim": 512, + "rope_dim": 64, + "q_lora_rank": 1536, + "window": 128, + "o_groups": 16, + "o_lora_rank": 1024, + "compress_rope_theta": 160000, + "rope_scaling": { + "type": "yarn", + "factor": 16, + "original_max": 65536, + "beta_fast": 32, + "beta_slow": 1 + }, + "kv_cache_dtype": "fp8", + "compress_ratio": 128, + "proj": { + "wkv": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 128 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + } + }, + "wo_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 128 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + } + }, + "wq_a": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 128 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + } + }, + "wq_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 128 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + } + } + }, + "batch": { + "groups": [ + { + "count": 1, + "q": 1, + "ctx": 4096 + } + ] + } + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro/attn", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn" + ] + }, + { + "type": "dsv4_attn", + "args": { + "hidden": 7168, + "heads": 128, + "head_dim": 512, + "rope_dim": 64, + "q_lora_rank": 1536, + "window": 128, + "o_groups": 16, + "o_lora_rank": 1024, + "compress_rope_theta": 160000, + "rope_scaling": { + "type": "yarn", + "factor": 16, + "original_max": 65536, + "beta_fast": 32, + "beta_slow": 1 + }, + "kv_cache_dtype": "fp8", + "compress_ratio": 128, + "proj": { + "wkv": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 128 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + } + }, + "wo_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 128 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + } + }, + "wq_a": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 128 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + } + }, + "wq_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 128 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + } + } + }, + "batch": { + "groups": [ + { + "count": 1, + "q": 1, + "ctx": 16384 + } + ] + } + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro/attn", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn" + ] + }, + { + "type": "dsv4_attn", + "args": { + "hidden": 7168, + "heads": 128, + "head_dim": 512, + "rope_dim": 64, + "q_lora_rank": 1536, + "window": 128, + "o_groups": 16, + "o_lora_rank": 1024, + "compress_rope_theta": 160000, + "rope_scaling": { + "type": "yarn", + "factor": 16, + "original_max": 65536, + "beta_fast": 32, + "beta_slow": 1 + }, + "kv_cache_dtype": "fp8", + "compress_ratio": 128, + "proj": { + "wkv": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 128 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + } + }, + "wo_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 128 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + } + }, + "wq_a": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 128 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + } + }, + "wq_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 128 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + } + } + }, + "batch": { + "groups": [ + { + "count": 1, + "q": 1, + "ctx": 65536 + } + ] + } + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro/attn", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn" + ] + }, + { + "type": "dsv4_attn", + "args": { + "hidden": 7168, + "heads": 128, + "head_dim": 512, + "rope_dim": 64, + "q_lora_rank": 1536, + "window": 128, + "o_groups": 16, + "o_lora_rank": 1024, + "compress_rope_theta": 160000, + "rope_scaling": { + "type": "yarn", + "factor": 16, + "original_max": 65536, + "beta_fast": 32, + "beta_slow": 1 + }, + "kv_cache_dtype": "fp8", + "compress_ratio": 128, + "proj": { + "wkv": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 128 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + } + }, + "wo_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 128 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + } + }, + "wq_a": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 128 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + } + }, + "wq_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 128 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + } + } + }, + "batch": { + "groups": [ + { + "count": 1, + "q": 1, + "ctx": 131072 + } + ] + } + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro/attn", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn" + ] + }, + { + "type": "dsv4_attn", + "args": { + "hidden": 7168, + "heads": 128, + "head_dim": 512, + "rope_dim": 64, + "q_lora_rank": 1536, + "window": 128, + "o_groups": 16, + "o_lora_rank": 1024, + "compress_rope_theta": 160000, + "rope_scaling": { + "type": "yarn", + "factor": 16, + "original_max": 65536, + "beta_fast": 32, + "beta_slow": 1 + }, + "kv_cache_dtype": "fp8", + "compress_ratio": 128, + "proj": { + "wkv": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 128 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + } + }, + "wo_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 128 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + } + }, + "wq_a": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 128 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + } + }, + "wq_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 128 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + } + } + }, + "batch": { + "groups": [ + { + "count": 4, + "q": 1, + "ctx": 1024 + } + ] + } + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro/attn", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn" + ] + }, + { + "type": "dsv4_attn", + "args": { + "hidden": 7168, + "heads": 128, + "head_dim": 512, + "rope_dim": 64, + "q_lora_rank": 1536, + "window": 128, + "o_groups": 16, + "o_lora_rank": 1024, + "compress_rope_theta": 160000, + "rope_scaling": { + "type": "yarn", + "factor": 16, + "original_max": 65536, + "beta_fast": 32, + "beta_slow": 1 + }, + "kv_cache_dtype": "fp8", + "compress_ratio": 128, + "proj": { + "wkv": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 128 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + } + }, + "wo_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 128 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + } + }, + "wq_a": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 128 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + } + }, + "wq_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 128 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + } + } + }, + "batch": { + "groups": [ + { + "count": 4, + "q": 1, + "ctx": 4096 + } + ] + } + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro/attn", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn" + ] + }, + { + "type": "dsv4_attn", + "args": { + "hidden": 7168, + "heads": 128, + "head_dim": 512, + "rope_dim": 64, + "q_lora_rank": 1536, + "window": 128, + "o_groups": 16, + "o_lora_rank": 1024, + "compress_rope_theta": 160000, + "rope_scaling": { + "type": "yarn", + "factor": 16, + "original_max": 65536, + "beta_fast": 32, + "beta_slow": 1 + }, + "kv_cache_dtype": "fp8", + "compress_ratio": 128, + "proj": { + "wkv": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 128 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + } + }, + "wo_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 128 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + } + }, + "wq_a": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 128 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + } + }, + "wq_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 128 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + } + } + }, + "batch": { + "groups": [ + { + "count": 4, + "q": 1, + "ctx": 16384 + } + ] + } + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro/attn", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn" + ] + }, + { + "type": "dsv4_attn", + "args": { + "hidden": 7168, + "heads": 128, + "head_dim": 512, + "rope_dim": 64, + "q_lora_rank": 1536, + "window": 128, + "o_groups": 16, + "o_lora_rank": 1024, + "compress_rope_theta": 160000, + "rope_scaling": { + "type": "yarn", + "factor": 16, + "original_max": 65536, + "beta_fast": 32, + "beta_slow": 1 + }, + "kv_cache_dtype": "fp8", + "compress_ratio": 128, + "proj": { + "wkv": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 128 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + } + }, + "wo_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 128 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + } + }, + "wq_a": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 128 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + } + }, + "wq_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 128 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + } + } + }, + "batch": { + "groups": [ + { + "count": 4, + "q": 1, + "ctx": 65536 + } + ] + } + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro/attn", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn" + ] + }, + { + "type": "dsv4_attn", + "args": { + "hidden": 7168, + "heads": 128, + "head_dim": 512, + "rope_dim": 64, + "q_lora_rank": 1536, + "window": 128, + "o_groups": 16, + "o_lora_rank": 1024, + "compress_rope_theta": 160000, + "rope_scaling": { + "type": "yarn", + "factor": 16, + "original_max": 65536, + "beta_fast": 32, + "beta_slow": 1 + }, + "kv_cache_dtype": "fp8", + "compress_ratio": 128, + "proj": { + "wkv": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 128 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + } + }, + "wo_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 128 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + } + }, + "wq_a": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 128 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + } + }, + "wq_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 128 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + } + } + }, + "batch": { + "groups": [ + { + "count": 4, + "q": 1, + "ctx": 131072 + } + ] + } + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro/attn", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn" + ] + }, + { + "type": "dsv4_attn", + "args": { + "hidden": 7168, + "heads": 128, + "head_dim": 512, + "rope_dim": 64, + "q_lora_rank": 1536, + "window": 128, + "o_groups": 16, + "o_lora_rank": 1024, + "compress_rope_theta": 160000, + "rope_scaling": { + "type": "yarn", + "factor": 16, + "original_max": 65536, + "beta_fast": 32, + "beta_slow": 1 + }, + "kv_cache_dtype": "fp8", + "compress_ratio": 128, + "proj": { + "wkv": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 128 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + } + }, + "wo_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 128 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + } + }, + "wq_a": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 128 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + } + }, + "wq_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 128 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + } + } + }, + "batch": { + "groups": [ + { + "count": 16, + "q": 1, + "ctx": 1024 + } + ] + } + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro/attn", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn" + ] + }, + { + "type": "dsv4_attn", + "args": { + "hidden": 7168, + "heads": 128, + "head_dim": 512, + "rope_dim": 64, + "q_lora_rank": 1536, + "window": 128, + "o_groups": 16, + "o_lora_rank": 1024, + "compress_rope_theta": 160000, + "rope_scaling": { + "type": "yarn", + "factor": 16, + "original_max": 65536, + "beta_fast": 32, + "beta_slow": 1 + }, + "kv_cache_dtype": "fp8", + "compress_ratio": 128, + "proj": { + "wkv": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 128 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + } + }, + "wo_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 128 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + } + }, + "wq_a": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 128 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + } + }, + "wq_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 128 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + } + } + }, + "batch": { + "groups": [ + { + "count": 16, + "q": 1, + "ctx": 4096 + } + ] + } + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro/attn", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn" + ] + }, + { + "type": "dsv4_attn", + "args": { + "hidden": 7168, + "heads": 128, + "head_dim": 512, + "rope_dim": 64, + "q_lora_rank": 1536, + "window": 128, + "o_groups": 16, + "o_lora_rank": 1024, + "compress_rope_theta": 160000, + "rope_scaling": { + "type": "yarn", + "factor": 16, + "original_max": 65536, + "beta_fast": 32, + "beta_slow": 1 + }, + "kv_cache_dtype": "fp8", + "compress_ratio": 128, + "proj": { + "wkv": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 128 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + } + }, + "wo_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 128 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + } + }, + "wq_a": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 128 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + } + }, + "wq_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 128 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + } + } + }, + "batch": { + "groups": [ + { + "count": 16, + "q": 1, + "ctx": 16384 + } + ] + } + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro/attn", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn" + ] + }, + { + "type": "dsv4_attn", + "args": { + "hidden": 7168, + "heads": 128, + "head_dim": 512, + "rope_dim": 64, + "q_lora_rank": 1536, + "window": 128, + "o_groups": 16, + "o_lora_rank": 1024, + "compress_rope_theta": 160000, + "rope_scaling": { + "type": "yarn", + "factor": 16, + "original_max": 65536, + "beta_fast": 32, + "beta_slow": 1 + }, + "kv_cache_dtype": "fp8", + "compress_ratio": 128, + "proj": { + "wkv": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 128 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + } + }, + "wo_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 128 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + } + }, + "wq_a": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 128 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + } + }, + "wq_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 128 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + } + } + }, + "batch": { + "groups": [ + { + "count": 16, + "q": 1, + "ctx": 65536 + } + ] + } + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro/attn", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn" + ] + }, + { + "type": "dsv4_attn", + "args": { + "hidden": 7168, + "heads": 128, + "head_dim": 512, + "rope_dim": 64, + "q_lora_rank": 1536, + "window": 128, + "o_groups": 16, + "o_lora_rank": 1024, + "compress_rope_theta": 160000, + "rope_scaling": { + "type": "yarn", + "factor": 16, + "original_max": 65536, + "beta_fast": 32, + "beta_slow": 1 + }, + "kv_cache_dtype": "fp8", + "compress_ratio": 128, + "proj": { + "wkv": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 128 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + } + }, + "wo_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 128 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + } + }, + "wq_a": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 128 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + } + }, + "wq_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 128 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + } + } + }, + "batch": { + "groups": [ + { + "count": 16, + "q": 1, + "ctx": 131072 + } + ] + } + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro/attn", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn" + ] + }, + { + "type": "dsv4_attn", + "args": { + "hidden": 7168, + "heads": 128, + "head_dim": 512, + "rope_dim": 64, + "q_lora_rank": 1536, + "window": 128, + "o_groups": 16, + "o_lora_rank": 1024, + "compress_rope_theta": 160000, + "rope_scaling": { + "type": "yarn", + "factor": 16, + "original_max": 65536, + "beta_fast": 32, + "beta_slow": 1 + }, + "kv_cache_dtype": "fp8", + "compress_ratio": 128, + "proj": { + "wkv": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 128 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + } + }, + "wo_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 128 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + } + }, + "wq_a": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 128 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + } + }, + "wq_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 128 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + } + } + }, + "batch": { + "groups": [ + { + "count": 64, + "q": 1, + "ctx": 1024 + } + ] + } + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro/attn", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn" + ] + }, + { + "type": "dsv4_attn", + "args": { + "hidden": 7168, + "heads": 128, + "head_dim": 512, + "rope_dim": 64, + "q_lora_rank": 1536, + "window": 128, + "o_groups": 16, + "o_lora_rank": 1024, + "compress_rope_theta": 160000, + "rope_scaling": { + "type": "yarn", + "factor": 16, + "original_max": 65536, + "beta_fast": 32, + "beta_slow": 1 + }, + "kv_cache_dtype": "fp8", + "compress_ratio": 128, + "proj": { + "wkv": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 128 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + } + }, + "wo_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 128 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + } + }, + "wq_a": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 128 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + } + }, + "wq_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 128 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + } + } + }, + "batch": { + "groups": [ + { + "count": 64, + "q": 1, + "ctx": 4096 + } + ] + } + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro/attn", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn" + ] + }, + { + "type": "dsv4_attn", + "args": { + "hidden": 7168, + "heads": 128, + "head_dim": 512, + "rope_dim": 64, + "q_lora_rank": 1536, + "window": 128, + "o_groups": 16, + "o_lora_rank": 1024, + "compress_rope_theta": 160000, + "rope_scaling": { + "type": "yarn", + "factor": 16, + "original_max": 65536, + "beta_fast": 32, + "beta_slow": 1 + }, + "kv_cache_dtype": "fp8", + "compress_ratio": 128, + "proj": { + "wkv": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 128 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + } + }, + "wo_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 128 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + } + }, + "wq_a": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 128 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + } + }, + "wq_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 128 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + } + } + }, + "batch": { + "groups": [ + { + "count": 64, + "q": 1, + "ctx": 16384 + } + ] + } + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro/attn", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn" + ] + }, + { + "type": "dsv4_attn", + "args": { + "hidden": 7168, + "heads": 128, + "head_dim": 512, + "rope_dim": 64, + "q_lora_rank": 1536, + "window": 128, + "o_groups": 16, + "o_lora_rank": 1024, + "compress_rope_theta": 160000, + "rope_scaling": { + "type": "yarn", + "factor": 16, + "original_max": 65536, + "beta_fast": 32, + "beta_slow": 1 + }, + "kv_cache_dtype": "fp8", + "compress_ratio": 128, + "proj": { + "wkv": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 128 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + } + }, + "wo_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 128 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + } + }, + "wq_a": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 128 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + } + }, + "wq_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 128 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + } + } + }, + "batch": { + "groups": [ + { + "count": 64, + "q": 1, + "ctx": 65536 + } + ] + } + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro/attn", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn" + ] + }, + { + "type": "dsv4_attn", + "args": { + "hidden": 7168, + "heads": 128, + "head_dim": 512, + "rope_dim": 64, + "q_lora_rank": 1536, + "window": 128, + "o_groups": 16, + "o_lora_rank": 1024, + "compress_rope_theta": 160000, + "rope_scaling": { + "type": "yarn", + "factor": 16, + "original_max": 65536, + "beta_fast": 32, + "beta_slow": 1 + }, + "kv_cache_dtype": "fp8", + "compress_ratio": 128, + "proj": { + "wkv": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 128 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + } + }, + "wo_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 128 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + } + }, + "wq_a": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 128 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + } + }, + "wq_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 128 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + } + } + }, + "batch": { + "groups": [ + { + "count": 64, + "q": 1, + "ctx": 131072 + } + ] + } + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro/attn", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn" + ] + }, + { + "type": "dsv4_attn", + "args": { + "hidden": 7168, + "heads": 128, + "head_dim": 512, + "rope_dim": 64, + "q_lora_rank": 1536, + "window": 128, + "o_groups": 16, + "o_lora_rank": 1024, + "compress_rope_theta": 160000, + "rope_scaling": { + "type": "yarn", + "factor": 16, + "original_max": 65536, + "beta_fast": 32, + "beta_slow": 1 + }, + "kv_cache_dtype": "fp8", + "compress_ratio": 128, + "proj": { + "wkv": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 128 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + } + }, + "wo_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 128 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + } + }, + "wq_a": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 128 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + } + }, + "wq_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 128 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + } + } + }, + "batch": { + "groups": [ + { + "count": 256, + "q": 1, + "ctx": 1024 + } + ] + } + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro/attn", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn" + ] + }, + { + "type": "dsv4_attn", + "args": { + "hidden": 7168, + "heads": 128, + "head_dim": 512, + "rope_dim": 64, + "q_lora_rank": 1536, + "window": 128, + "o_groups": 16, + "o_lora_rank": 1024, + "compress_rope_theta": 160000, + "rope_scaling": { + "type": "yarn", + "factor": 16, + "original_max": 65536, + "beta_fast": 32, + "beta_slow": 1 + }, + "kv_cache_dtype": "fp8", + "compress_ratio": 128, + "proj": { + "wkv": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 128 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + } + }, + "wo_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 128 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + } + }, + "wq_a": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 128 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + } + }, + "wq_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 128 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + } + } + }, + "batch": { + "groups": [ + { + "count": 256, + "q": 1, + "ctx": 4096 + } + ] + } + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro/attn", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn" + ] + }, + { + "type": "dsv4_attn", + "args": { + "hidden": 7168, + "heads": 128, + "head_dim": 512, + "rope_dim": 64, + "q_lora_rank": 1536, + "window": 128, + "o_groups": 16, + "o_lora_rank": 1024, + "compress_rope_theta": 160000, + "rope_scaling": { + "type": "yarn", + "factor": 16, + "original_max": 65536, + "beta_fast": 32, + "beta_slow": 1 + }, + "kv_cache_dtype": "fp8", + "compress_ratio": 128, + "proj": { + "wkv": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 128 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + } + }, + "wo_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 128 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + } + }, + "wq_a": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 128 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + } + }, + "wq_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 128 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + } + } + }, + "batch": { + "groups": [ + { + "count": 256, + "q": 1, + "ctx": 16384 + } + ] + } + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro/attn", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn" + ] + }, + { + "type": "dsv4_attn", + "args": { + "hidden": 7168, + "heads": 128, + "head_dim": 512, + "rope_dim": 64, + "q_lora_rank": 1536, + "window": 128, + "o_groups": 16, + "o_lora_rank": 1024, + "compress_rope_theta": 160000, + "rope_scaling": { + "type": "yarn", + "factor": 16, + "original_max": 65536, + "beta_fast": 32, + "beta_slow": 1 + }, + "kv_cache_dtype": "fp8", + "compress_ratio": 128, + "proj": { + "wkv": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 128 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + } + }, + "wo_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 128 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + } + }, + "wq_a": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 128 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + } + }, + "wq_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 128 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + } + } + }, + "batch": { + "groups": [ + { + "count": 256, + "q": 1, + "ctx": 65536 + } + ] + } + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro/attn", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn" + ] + }, + { + "type": "dsv4_attn", + "args": { + "hidden": 7168, + "heads": 128, + "head_dim": 512, + "rope_dim": 64, + "q_lora_rank": 1536, + "window": 128, + "o_groups": 16, + "o_lora_rank": 1024, + "compress_rope_theta": 160000, + "rope_scaling": { + "type": "yarn", + "factor": 16, + "original_max": 65536, + "beta_fast": 32, + "beta_slow": 1 + }, + "kv_cache_dtype": "fp8", + "compress_ratio": 128, + "proj": { + "wkv": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 128 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + } + }, + "wo_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 128 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + } + }, + "wq_a": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 128 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + } + }, + "wq_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 128 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + } + } + }, + "batch": { + "groups": [ + { + "count": 256, + "q": 1, + "ctx": 131072 + } + ] + } + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro/attn", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn" + ] + }, + { + "type": "dsv4_attn", + "args": { + "hidden": 7168, + "heads": 128, + "head_dim": 512, + "rope_dim": 64, + "q_lora_rank": 1536, + "window": 128, + "o_groups": 16, + "o_lora_rank": 1024, + "compress_rope_theta": 160000, + "rope_scaling": { + "type": "yarn", + "factor": 16, + "original_max": 65536, + "beta_fast": 32, + "beta_slow": 1 + }, + "kv_cache_dtype": "fp8", + "compress_ratio": 128, + "proj": { + "wkv": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 128 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + } + }, + "wo_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 128 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + } + }, + "wq_a": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 128 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + } + }, + "wq_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 128 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + } + } + }, + "batch": { + "groups": [ + { + "count": 1, + "q": 1024, + "ctx": 0 + } + ] + } + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro/attn", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn" + ] + }, + { + "type": "dsv4_attn", + "args": { + "hidden": 7168, + "heads": 128, + "head_dim": 512, + "rope_dim": 64, + "q_lora_rank": 1536, + "window": 128, + "o_groups": 16, + "o_lora_rank": 1024, + "compress_rope_theta": 160000, + "rope_scaling": { + "type": "yarn", + "factor": 16, + "original_max": 65536, + "beta_fast": 32, + "beta_slow": 1 + }, + "kv_cache_dtype": "fp8", + "compress_ratio": 128, + "proj": { + "wkv": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 128 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + } + }, + "wo_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 128 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + } + }, + "wq_a": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 128 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + } + }, + "wq_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 128 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + } + } + }, + "batch": { + "groups": [ + { + "count": 1, + "q": 4096, + "ctx": 0 + } + ] + } + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro/attn", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn" + ] + }, + { + "type": "dsv4_attn", + "args": { + "hidden": 7168, + "heads": 128, + "head_dim": 512, + "rope_dim": 64, + "q_lora_rank": 1536, + "window": 128, + "o_groups": 16, + "o_lora_rank": 1024, + "compress_rope_theta": 160000, + "rope_scaling": { + "type": "yarn", + "factor": 16, + "original_max": 65536, + "beta_fast": 32, + "beta_slow": 1 + }, + "kv_cache_dtype": "fp8", + "compress_ratio": 128, + "proj": { + "wkv": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 128 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + } + }, + "wo_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 128 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + } + }, + "wq_a": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 128 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + } + }, + "wq_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 128 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + } + } + }, + "batch": { + "groups": [ + { + "count": 1, + "q": 16384, + "ctx": 0 + } + ] + } + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro/attn", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn" + ] + }, + { + "type": "dsv4_attn", + "args": { + "hidden": 7168, + "heads": 128, + "head_dim": 512, + "rope_dim": 64, + "q_lora_rank": 1536, + "window": 128, + "o_groups": 16, + "o_lora_rank": 1024, + "compress_rope_theta": 160000, + "rope_scaling": { + "type": "yarn", + "factor": 16, + "original_max": 65536, + "beta_fast": 32, + "beta_slow": 1 + }, + "kv_cache_dtype": "fp8", + "compress_ratio": 128, + "proj": { + "wkv": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 128 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + } + }, + "wo_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 128 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + } + }, + "wq_a": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 128 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + } + }, + "wq_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 128 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + } + } + }, + "batch": { + "groups": [ + { + "count": 1, + "q": 32768, + "ctx": 0 + } + ] + } + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro/attn", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn" + ] + }, + { + "type": "dsv4_attn", + "args": { + "hidden": 7168, + "heads": 128, + "head_dim": 512, + "rope_dim": 64, + "q_lora_rank": 1536, + "window": 128, + "o_groups": 16, + "o_lora_rank": 1024, + "compress_rope_theta": 160000, + "rope_scaling": { + "type": "yarn", + "factor": 16, + "original_max": 65536, + "beta_fast": 32, + "beta_slow": 1 + }, + "kv_cache_dtype": "fp8", + "compress_ratio": 128, + "proj": { + "wkv": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 128 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + } + }, + "wo_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 128 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + } + }, + "wq_a": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 128 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + } + }, + "wq_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 128 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + } + } + }, + "batch": { + "groups": [ + { + "count": 1, + "q": 8192, + "ctx": 8192 + } + ] + } + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro/attn", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn" + ] + }, + { + "type": "dsv4_attn", + "args": { + "hidden": 7168, + "heads": 128, + "head_dim": 512, + "rope_dim": 64, + "q_lora_rank": 1536, + "window": 128, + "o_groups": 16, + "o_lora_rank": 1024, + "compress_rope_theta": 160000, + "rope_scaling": { + "type": "yarn", + "factor": 16, + "original_max": 65536, + "beta_fast": 32, + "beta_slow": 1 + }, + "kv_cache_dtype": "fp8", + "compress_ratio": 128, + "proj": { + "wkv": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 128 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + } + }, + "wo_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 128 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + } + }, + "wq_a": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 128 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + } + }, + "wq_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 128 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + } + } + }, + "batch": { + "groups": [ + { + "count": 1, + "q": 8192, + "ctx": 32768 + } + ] + } + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro/attn", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn" + ] + }, + { + "type": "dsv4_attn", + "args": { + "hidden": 7168, + "heads": 128, + "head_dim": 512, + "rope_dim": 64, + "q_lora_rank": 1536, + "window": 128, + "o_groups": 16, + "o_lora_rank": 1024, + "compress_rope_theta": 160000, + "rope_scaling": { + "type": "yarn", + "factor": 16, + "original_max": 65536, + "beta_fast": 32, + "beta_slow": 1 + }, + "kv_cache_dtype": "fp8", + "compress_ratio": 128, + "proj": { + "wkv": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 128 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + } + }, + "wo_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 128 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + } + }, + "wq_a": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 128 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + } + }, + "wq_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 128 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + } + } + }, + "batch": { + "groups": [ + { + "count": 1, + "q": 8192, + "ctx": 131072 + } + ] + } + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro/attn", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn" + ] + }, + { + "type": "dsv4_attn", + "args": { + "hidden": 7168, + "heads": 128, + "head_dim": 512, + "rope_dim": 64, + "q_lora_rank": 1536, + "window": 128, + "o_groups": 16, + "o_lora_rank": 1024, + "compress_rope_theta": 160000, + "rope_scaling": { + "type": "yarn", + "factor": 16, + "original_max": 65536, + "beta_fast": 32, + "beta_slow": 1 + }, + "kv_cache_dtype": "fp8", + "compress_ratio": 128, + "proj": { + "wkv": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 128 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + } + }, + "wo_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 128 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + } + }, + "wq_a": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 128 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + } + }, + "wq_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 128 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + } + } + }, + "batch": { + "groups": [ + { + "count": 1, + "q": 8192, + "ctx": 8192 + }, + { + "count": 64, + "q": 1, + "ctx": 8192 + } + ] + } + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro/attn", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn" + ] + }, + { + "type": "dsv4_attn", + "args": { + "hidden": 5120, + "heads": 64, + "head_dim": 512, + "rope_dim": 64, + "q_lora_rank": 1280, + "window": 128, + "o_groups": 8, + "o_lora_rank": 1024, + "compress_ratio": 0, + "proj": { + "wkv": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + }, + "wo_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + }, + "wq_a": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + }, + "wq_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + } + }, + "batch": { + "groups": [ + { + "count": 1, + "q": 1, + "ctx": 1024 + } + ] + } + }, + "sources": [ + "deepseek-ai/DeepSeek-V4.1-Flash/attn" + ] + }, + { + "type": "dsv4_attn", + "args": { + "hidden": 5120, + "heads": 64, + "head_dim": 512, + "rope_dim": 64, + "q_lora_rank": 1280, + "window": 128, + "o_groups": 8, + "o_lora_rank": 1024, + "compress_ratio": 0, + "proj": { + "wkv": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + }, + "wo_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + }, + "wq_a": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + }, + "wq_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + } + }, + "batch": { + "groups": [ + { + "count": 1, + "q": 1, + "ctx": 4096 + } + ] + } + }, + "sources": [ + "deepseek-ai/DeepSeek-V4.1-Flash/attn" + ] + }, + { + "type": "dsv4_attn", + "args": { + "hidden": 5120, + "heads": 64, + "head_dim": 512, + "rope_dim": 64, + "q_lora_rank": 1280, + "window": 128, + "o_groups": 8, + "o_lora_rank": 1024, + "compress_ratio": 0, + "proj": { + "wkv": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + }, + "wo_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + }, + "wq_a": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + }, + "wq_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + } + }, + "batch": { + "groups": [ + { + "count": 1, + "q": 1, + "ctx": 16384 + } + ] + } + }, + "sources": [ + "deepseek-ai/DeepSeek-V4.1-Flash/attn" + ] + }, + { + "type": "dsv4_attn", + "args": { + "hidden": 5120, + "heads": 64, + "head_dim": 512, + "rope_dim": 64, + "q_lora_rank": 1280, + "window": 128, + "o_groups": 8, + "o_lora_rank": 1024, + "compress_ratio": 0, + "proj": { + "wkv": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + }, + "wo_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + }, + "wq_a": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + }, + "wq_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + } + }, + "batch": { + "groups": [ + { + "count": 1, + "q": 1, + "ctx": 65536 + } + ] + } + }, + "sources": [ + "deepseek-ai/DeepSeek-V4.1-Flash/attn" + ] + }, + { + "type": "dsv4_attn", + "args": { + "hidden": 5120, + "heads": 64, + "head_dim": 512, + "rope_dim": 64, + "q_lora_rank": 1280, + "window": 128, + "o_groups": 8, + "o_lora_rank": 1024, + "compress_ratio": 0, + "proj": { + "wkv": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + }, + "wo_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + }, + "wq_a": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + }, + "wq_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + } + }, + "batch": { + "groups": [ + { + "count": 1, + "q": 1, + "ctx": 131072 + } + ] + } + }, + "sources": [ + "deepseek-ai/DeepSeek-V4.1-Flash/attn" + ] + }, + { + "type": "dsv4_attn", + "args": { + "hidden": 5120, + "heads": 64, + "head_dim": 512, + "rope_dim": 64, + "q_lora_rank": 1280, + "window": 128, + "o_groups": 8, + "o_lora_rank": 1024, + "compress_ratio": 0, + "proj": { + "wkv": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + }, + "wo_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + }, + "wq_a": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + }, + "wq_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + } + }, + "batch": { + "groups": [ + { + "count": 4, + "q": 1, + "ctx": 1024 + } + ] + } + }, + "sources": [ + "deepseek-ai/DeepSeek-V4.1-Flash/attn" + ] + }, + { + "type": "dsv4_attn", + "args": { + "hidden": 5120, + "heads": 64, + "head_dim": 512, + "rope_dim": 64, + "q_lora_rank": 1280, + "window": 128, + "o_groups": 8, + "o_lora_rank": 1024, + "compress_ratio": 0, + "proj": { + "wkv": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + }, + "wo_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + }, + "wq_a": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + }, + "wq_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + } + }, + "batch": { + "groups": [ + { + "count": 4, + "q": 1, + "ctx": 4096 + } + ] + } + }, + "sources": [ + "deepseek-ai/DeepSeek-V4.1-Flash/attn" + ] + }, + { + "type": "dsv4_attn", + "args": { + "hidden": 5120, + "heads": 64, + "head_dim": 512, + "rope_dim": 64, + "q_lora_rank": 1280, + "window": 128, + "o_groups": 8, + "o_lora_rank": 1024, + "compress_ratio": 0, + "proj": { + "wkv": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + }, + "wo_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + }, + "wq_a": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + }, + "wq_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + } + }, + "batch": { + "groups": [ + { + "count": 4, + "q": 1, + "ctx": 16384 + } + ] + } + }, + "sources": [ + "deepseek-ai/DeepSeek-V4.1-Flash/attn" + ] + }, + { + "type": "dsv4_attn", + "args": { + "hidden": 5120, + "heads": 64, + "head_dim": 512, + "rope_dim": 64, + "q_lora_rank": 1280, + "window": 128, + "o_groups": 8, + "o_lora_rank": 1024, + "compress_ratio": 0, + "proj": { + "wkv": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + }, + "wo_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + }, + "wq_a": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + }, + "wq_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + } + }, + "batch": { + "groups": [ + { + "count": 4, + "q": 1, + "ctx": 65536 + } + ] + } + }, + "sources": [ + "deepseek-ai/DeepSeek-V4.1-Flash/attn" + ] + }, + { + "type": "dsv4_attn", + "args": { + "hidden": 5120, + "heads": 64, + "head_dim": 512, + "rope_dim": 64, + "q_lora_rank": 1280, + "window": 128, + "o_groups": 8, + "o_lora_rank": 1024, + "compress_ratio": 0, + "proj": { + "wkv": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + }, + "wo_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + }, + "wq_a": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + }, + "wq_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + } + }, + "batch": { + "groups": [ + { + "count": 4, + "q": 1, + "ctx": 131072 + } + ] + } + }, + "sources": [ + "deepseek-ai/DeepSeek-V4.1-Flash/attn" + ] + }, + { + "type": "dsv4_attn", + "args": { + "hidden": 5120, + "heads": 64, + "head_dim": 512, + "rope_dim": 64, + "q_lora_rank": 1280, + "window": 128, + "o_groups": 8, + "o_lora_rank": 1024, + "compress_ratio": 0, + "proj": { + "wkv": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + }, + "wo_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + }, + "wq_a": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + }, + "wq_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + } + }, + "batch": { + "groups": [ + { + "count": 16, + "q": 1, + "ctx": 1024 + } + ] + } + }, + "sources": [ + "deepseek-ai/DeepSeek-V4.1-Flash/attn" + ] + }, + { + "type": "dsv4_attn", + "args": { + "hidden": 5120, + "heads": 64, + "head_dim": 512, + "rope_dim": 64, + "q_lora_rank": 1280, + "window": 128, + "o_groups": 8, + "o_lora_rank": 1024, + "compress_ratio": 0, + "proj": { + "wkv": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + }, + "wo_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + }, + "wq_a": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + }, + "wq_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + } + }, + "batch": { + "groups": [ + { + "count": 16, + "q": 1, + "ctx": 4096 + } + ] + } + }, + "sources": [ + "deepseek-ai/DeepSeek-V4.1-Flash/attn" + ] + }, + { + "type": "dsv4_attn", + "args": { + "hidden": 5120, + "heads": 64, + "head_dim": 512, + "rope_dim": 64, + "q_lora_rank": 1280, + "window": 128, + "o_groups": 8, + "o_lora_rank": 1024, + "compress_ratio": 0, + "proj": { + "wkv": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + }, + "wo_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + }, + "wq_a": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + }, + "wq_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + } + }, + "batch": { + "groups": [ + { + "count": 16, + "q": 1, + "ctx": 16384 + } + ] + } + }, + "sources": [ + "deepseek-ai/DeepSeek-V4.1-Flash/attn" + ] + }, + { + "type": "dsv4_attn", + "args": { + "hidden": 5120, + "heads": 64, + "head_dim": 512, + "rope_dim": 64, + "q_lora_rank": 1280, + "window": 128, + "o_groups": 8, + "o_lora_rank": 1024, + "compress_ratio": 0, + "proj": { + "wkv": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + }, + "wo_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + }, + "wq_a": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + }, + "wq_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + } + }, + "batch": { + "groups": [ + { + "count": 16, + "q": 1, + "ctx": 65536 + } + ] + } + }, + "sources": [ + "deepseek-ai/DeepSeek-V4.1-Flash/attn" + ] + }, + { + "type": "dsv4_attn", + "args": { + "hidden": 5120, + "heads": 64, + "head_dim": 512, + "rope_dim": 64, + "q_lora_rank": 1280, + "window": 128, + "o_groups": 8, + "o_lora_rank": 1024, + "compress_ratio": 0, + "proj": { + "wkv": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + }, + "wo_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + }, + "wq_a": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + }, + "wq_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + } + }, + "batch": { + "groups": [ + { + "count": 16, + "q": 1, + "ctx": 131072 + } + ] + } + }, + "sources": [ + "deepseek-ai/DeepSeek-V4.1-Flash/attn" + ] + }, + { + "type": "dsv4_attn", + "args": { + "hidden": 5120, + "heads": 64, + "head_dim": 512, + "rope_dim": 64, + "q_lora_rank": 1280, + "window": 128, + "o_groups": 8, + "o_lora_rank": 1024, + "compress_ratio": 0, + "proj": { + "wkv": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + }, + "wo_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + }, + "wq_a": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + }, + "wq_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + } + }, + "batch": { + "groups": [ + { + "count": 64, + "q": 1, + "ctx": 1024 + } + ] + } + }, + "sources": [ + "deepseek-ai/DeepSeek-V4.1-Flash/attn" + ] + }, + { + "type": "dsv4_attn", + "args": { + "hidden": 5120, + "heads": 64, + "head_dim": 512, + "rope_dim": 64, + "q_lora_rank": 1280, + "window": 128, + "o_groups": 8, + "o_lora_rank": 1024, + "compress_ratio": 0, + "proj": { + "wkv": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + }, + "wo_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + }, + "wq_a": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + }, + "wq_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + } + }, + "batch": { + "groups": [ + { + "count": 64, + "q": 1, + "ctx": 4096 + } + ] + } + }, + "sources": [ + "deepseek-ai/DeepSeek-V4.1-Flash/attn" + ] + }, + { + "type": "dsv4_attn", + "args": { + "hidden": 5120, + "heads": 64, + "head_dim": 512, + "rope_dim": 64, + "q_lora_rank": 1280, + "window": 128, + "o_groups": 8, + "o_lora_rank": 1024, + "compress_ratio": 0, + "proj": { + "wkv": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + }, + "wo_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + }, + "wq_a": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + }, + "wq_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + } + }, + "batch": { + "groups": [ + { + "count": 64, + "q": 1, + "ctx": 16384 + } + ] + } + }, + "sources": [ + "deepseek-ai/DeepSeek-V4.1-Flash/attn" + ] + }, + { + "type": "dsv4_attn", + "args": { + "hidden": 5120, + "heads": 64, + "head_dim": 512, + "rope_dim": 64, + "q_lora_rank": 1280, + "window": 128, + "o_groups": 8, + "o_lora_rank": 1024, + "compress_ratio": 0, + "proj": { + "wkv": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + }, + "wo_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + }, + "wq_a": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + }, + "wq_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + } + }, + "batch": { + "groups": [ + { + "count": 64, + "q": 1, + "ctx": 65536 + } + ] + } + }, + "sources": [ + "deepseek-ai/DeepSeek-V4.1-Flash/attn" + ] + }, + { + "type": "dsv4_attn", + "args": { + "hidden": 5120, + "heads": 64, + "head_dim": 512, + "rope_dim": 64, + "q_lora_rank": 1280, + "window": 128, + "o_groups": 8, + "o_lora_rank": 1024, + "compress_ratio": 0, + "proj": { + "wkv": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + }, + "wo_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + }, + "wq_a": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + }, + "wq_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + } + }, + "batch": { + "groups": [ + { + "count": 64, + "q": 1, + "ctx": 131072 + } + ] + } + }, + "sources": [ + "deepseek-ai/DeepSeek-V4.1-Flash/attn" + ] + }, + { + "type": "dsv4_attn", + "args": { + "hidden": 5120, + "heads": 64, + "head_dim": 512, + "rope_dim": 64, + "q_lora_rank": 1280, + "window": 128, + "o_groups": 8, + "o_lora_rank": 1024, + "compress_ratio": 0, + "proj": { + "wkv": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + }, + "wo_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + }, + "wq_a": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + }, + "wq_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + } + }, + "batch": { + "groups": [ + { + "count": 256, + "q": 1, + "ctx": 1024 + } + ] + } + }, + "sources": [ + "deepseek-ai/DeepSeek-V4.1-Flash/attn" + ] + }, + { + "type": "dsv4_attn", + "args": { + "hidden": 5120, + "heads": 64, + "head_dim": 512, + "rope_dim": 64, + "q_lora_rank": 1280, + "window": 128, + "o_groups": 8, + "o_lora_rank": 1024, + "compress_ratio": 0, + "proj": { + "wkv": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + }, + "wo_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + }, + "wq_a": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + }, + "wq_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + } + }, + "batch": { + "groups": [ + { + "count": 256, + "q": 1, + "ctx": 4096 + } + ] + } + }, + "sources": [ + "deepseek-ai/DeepSeek-V4.1-Flash/attn" + ] + }, + { + "type": "dsv4_attn", + "args": { + "hidden": 5120, + "heads": 64, + "head_dim": 512, + "rope_dim": 64, + "q_lora_rank": 1280, + "window": 128, + "o_groups": 8, + "o_lora_rank": 1024, + "compress_ratio": 0, + "proj": { + "wkv": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + }, + "wo_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + }, + "wq_a": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + }, + "wq_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + } + }, + "batch": { + "groups": [ + { + "count": 256, + "q": 1, + "ctx": 16384 + } + ] + } + }, + "sources": [ + "deepseek-ai/DeepSeek-V4.1-Flash/attn" + ] + }, + { + "type": "dsv4_attn", + "args": { + "hidden": 5120, + "heads": 64, + "head_dim": 512, + "rope_dim": 64, + "q_lora_rank": 1280, + "window": 128, + "o_groups": 8, + "o_lora_rank": 1024, + "compress_ratio": 0, + "proj": { + "wkv": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + }, + "wo_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + }, + "wq_a": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + }, + "wq_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + } + }, + "batch": { + "groups": [ + { + "count": 256, + "q": 1, + "ctx": 65536 + } + ] + } + }, + "sources": [ + "deepseek-ai/DeepSeek-V4.1-Flash/attn" + ] + }, + { + "type": "dsv4_attn", + "args": { + "hidden": 5120, + "heads": 64, + "head_dim": 512, + "rope_dim": 64, + "q_lora_rank": 1280, + "window": 128, + "o_groups": 8, + "o_lora_rank": 1024, + "compress_ratio": 0, + "proj": { + "wkv": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + }, + "wo_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + }, + "wq_a": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + }, + "wq_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + } + }, + "batch": { + "groups": [ + { + "count": 256, + "q": 1, + "ctx": 131072 + } + ] + } + }, + "sources": [ + "deepseek-ai/DeepSeek-V4.1-Flash/attn" + ] + }, + { + "type": "dsv4_attn", + "args": { + "hidden": 5120, + "heads": 64, + "head_dim": 512, + "rope_dim": 64, + "q_lora_rank": 1280, + "window": 128, + "o_groups": 8, + "o_lora_rank": 1024, + "compress_ratio": 0, + "proj": { + "wkv": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + }, + "wo_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + }, + "wq_a": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + }, + "wq_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + } + }, + "batch": { + "groups": [ + { + "count": 1, + "q": 1024, + "ctx": 0 + } + ] + } + }, + "sources": [ + "deepseek-ai/DeepSeek-V4.1-Flash/attn" + ] + }, + { + "type": "dsv4_attn", + "args": { + "hidden": 5120, + "heads": 64, + "head_dim": 512, + "rope_dim": 64, + "q_lora_rank": 1280, + "window": 128, + "o_groups": 8, + "o_lora_rank": 1024, + "compress_ratio": 0, + "proj": { + "wkv": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + }, + "wo_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + }, + "wq_a": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + }, + "wq_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + } + }, + "batch": { + "groups": [ + { + "count": 1, + "q": 4096, + "ctx": 0 + } + ] + } + }, + "sources": [ + "deepseek-ai/DeepSeek-V4.1-Flash/attn" + ] + }, + { + "type": "dsv4_attn", + "args": { + "hidden": 5120, + "heads": 64, + "head_dim": 512, + "rope_dim": 64, + "q_lora_rank": 1280, + "window": 128, + "o_groups": 8, + "o_lora_rank": 1024, + "compress_ratio": 0, + "proj": { + "wkv": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + }, + "wo_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + }, + "wq_a": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + }, + "wq_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + } + }, + "batch": { + "groups": [ + { + "count": 1, + "q": 16384, + "ctx": 0 + } + ] + } + }, + "sources": [ + "deepseek-ai/DeepSeek-V4.1-Flash/attn" + ] + }, + { + "type": "dsv4_attn", + "args": { + "hidden": 5120, + "heads": 64, + "head_dim": 512, + "rope_dim": 64, + "q_lora_rank": 1280, + "window": 128, + "o_groups": 8, + "o_lora_rank": 1024, + "compress_ratio": 0, + "proj": { + "wkv": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + }, + "wo_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + }, + "wq_a": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + }, + "wq_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + } + }, + "batch": { + "groups": [ + { + "count": 1, + "q": 32768, + "ctx": 0 + } + ] + } + }, + "sources": [ + "deepseek-ai/DeepSeek-V4.1-Flash/attn" + ] + }, + { + "type": "dsv4_attn", + "args": { + "hidden": 5120, + "heads": 64, + "head_dim": 512, + "rope_dim": 64, + "q_lora_rank": 1280, + "window": 128, + "o_groups": 8, + "o_lora_rank": 1024, + "compress_ratio": 0, + "proj": { + "wkv": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + }, + "wo_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + }, + "wq_a": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + }, + "wq_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + } + }, + "batch": { + "groups": [ + { + "count": 1, + "q": 8192, + "ctx": 8192 + } + ] + } + }, + "sources": [ + "deepseek-ai/DeepSeek-V4.1-Flash/attn" + ] + }, + { + "type": "dsv4_attn", + "args": { + "hidden": 5120, + "heads": 64, + "head_dim": 512, + "rope_dim": 64, + "q_lora_rank": 1280, + "window": 128, + "o_groups": 8, + "o_lora_rank": 1024, + "compress_ratio": 0, + "proj": { + "wkv": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + }, + "wo_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + }, + "wq_a": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + }, + "wq_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + } + }, + "batch": { + "groups": [ + { + "count": 1, + "q": 8192, + "ctx": 32768 + } + ] + } + }, + "sources": [ + "deepseek-ai/DeepSeek-V4.1-Flash/attn" + ] + }, + { + "type": "dsv4_attn", + "args": { + "hidden": 5120, + "heads": 64, + "head_dim": 512, + "rope_dim": 64, + "q_lora_rank": 1280, + "window": 128, + "o_groups": 8, + "o_lora_rank": 1024, + "compress_ratio": 0, + "proj": { + "wkv": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + }, + "wo_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + }, + "wq_a": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + }, + "wq_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + } + }, + "batch": { + "groups": [ + { + "count": 1, + "q": 8192, + "ctx": 131072 + } + ] + } + }, + "sources": [ + "deepseek-ai/DeepSeek-V4.1-Flash/attn" + ] + }, + { + "type": "dsv4_attn", + "args": { + "hidden": 5120, + "heads": 64, + "head_dim": 512, + "rope_dim": 64, + "q_lora_rank": 1280, + "window": 128, + "o_groups": 8, + "o_lora_rank": 1024, + "compress_ratio": 0, + "proj": { + "wkv": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + }, + "wo_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + }, + "wq_a": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + }, + "wq_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + } + }, + "batch": { + "groups": [ + { + "count": 1, + "q": 8192, + "ctx": 8192 + }, + { + "count": 64, + "q": 1, + "ctx": 8192 + } + ] + } + }, + "sources": [ + "deepseek-ai/DeepSeek-V4.1-Flash/attn" + ] + }, + { + "type": "dsv4_attn", + "args": { + "hidden": 5120, + "heads": 64, + "head_dim": 512, + "rope_dim": 64, + "q_lora_rank": 1280, + "window": 128, + "o_groups": 8, + "o_lora_rank": 1024, + "compress_ratio": 2, + "indexer": "own", + "compress_rope_theta": 160000, + "rope_scaling": { + "type": "yarn", + "factor": 16, + "original_max": 65536, + "beta_fast": 32, + "beta_slow": 1 + }, + "topk": 512, + "index_heads": 32, + "index_dim": 128, + "proj": { + "indexer.wq_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + }, + "wkv": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + }, + "wo_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + }, + "wq_a": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + }, + "wq_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + } + }, + "batch": { + "groups": [ + { + "count": 1, + "q": 1, + "ctx": 1024 + } + ] + } + }, + "sources": [ + "deepseek-ai/DeepSeek-V4.1-Flash/attn" + ] + }, + { + "type": "dsv4_attn", + "args": { + "hidden": 5120, + "heads": 64, + "head_dim": 512, + "rope_dim": 64, + "q_lora_rank": 1280, + "window": 128, + "o_groups": 8, + "o_lora_rank": 1024, + "compress_ratio": 2, + "indexer": "own", + "compress_rope_theta": 160000, + "rope_scaling": { + "type": "yarn", + "factor": 16, + "original_max": 65536, + "beta_fast": 32, + "beta_slow": 1 + }, + "topk": 512, + "index_heads": 32, + "index_dim": 128, + "proj": { + "indexer.wq_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + }, + "wkv": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + }, + "wo_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + }, + "wq_a": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + }, + "wq_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + } + }, + "batch": { + "groups": [ + { + "count": 1, + "q": 1, + "ctx": 4096 + } + ] + } + }, + "sources": [ + "deepseek-ai/DeepSeek-V4.1-Flash/attn" + ] + }, + { + "type": "dsv4_attn", + "args": { + "hidden": 5120, + "heads": 64, + "head_dim": 512, + "rope_dim": 64, + "q_lora_rank": 1280, + "window": 128, + "o_groups": 8, + "o_lora_rank": 1024, + "compress_ratio": 2, + "indexer": "own", + "compress_rope_theta": 160000, + "rope_scaling": { + "type": "yarn", + "factor": 16, + "original_max": 65536, + "beta_fast": 32, + "beta_slow": 1 + }, + "topk": 512, + "index_heads": 32, + "index_dim": 128, + "proj": { + "indexer.wq_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + }, + "wkv": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + }, + "wo_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + }, + "wq_a": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + }, + "wq_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + } + }, + "batch": { + "groups": [ + { + "count": 1, + "q": 1, + "ctx": 16384 + } + ] + } + }, + "sources": [ + "deepseek-ai/DeepSeek-V4.1-Flash/attn" + ] + }, + { + "type": "dsv4_attn", + "args": { + "hidden": 5120, + "heads": 64, + "head_dim": 512, + "rope_dim": 64, + "q_lora_rank": 1280, + "window": 128, + "o_groups": 8, + "o_lora_rank": 1024, + "compress_ratio": 2, + "indexer": "own", + "compress_rope_theta": 160000, + "rope_scaling": { + "type": "yarn", + "factor": 16, + "original_max": 65536, + "beta_fast": 32, + "beta_slow": 1 + }, + "topk": 512, + "index_heads": 32, + "index_dim": 128, + "proj": { + "indexer.wq_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + }, + "wkv": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + }, + "wo_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + }, + "wq_a": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + }, + "wq_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + } + }, + "batch": { + "groups": [ + { + "count": 1, + "q": 1, + "ctx": 65536 + } + ] + } + }, + "sources": [ + "deepseek-ai/DeepSeek-V4.1-Flash/attn" + ] + }, + { + "type": "dsv4_attn", + "args": { + "hidden": 5120, + "heads": 64, + "head_dim": 512, + "rope_dim": 64, + "q_lora_rank": 1280, + "window": 128, + "o_groups": 8, + "o_lora_rank": 1024, + "compress_ratio": 2, + "indexer": "own", + "compress_rope_theta": 160000, + "rope_scaling": { + "type": "yarn", + "factor": 16, + "original_max": 65536, + "beta_fast": 32, + "beta_slow": 1 + }, + "topk": 512, + "index_heads": 32, + "index_dim": 128, + "proj": { + "indexer.wq_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + }, + "wkv": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + }, + "wo_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + }, + "wq_a": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + }, + "wq_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + } + }, + "batch": { + "groups": [ + { + "count": 1, + "q": 1, + "ctx": 131072 + } + ] + } + }, + "sources": [ + "deepseek-ai/DeepSeek-V4.1-Flash/attn" + ] + }, + { + "type": "dsv4_attn", + "args": { + "hidden": 5120, + "heads": 64, + "head_dim": 512, + "rope_dim": 64, + "q_lora_rank": 1280, + "window": 128, + "o_groups": 8, + "o_lora_rank": 1024, + "compress_ratio": 2, + "indexer": "own", + "compress_rope_theta": 160000, + "rope_scaling": { + "type": "yarn", + "factor": 16, + "original_max": 65536, + "beta_fast": 32, + "beta_slow": 1 + }, + "topk": 512, + "index_heads": 32, + "index_dim": 128, + "proj": { + "indexer.wq_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + }, + "wkv": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + }, + "wo_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + }, + "wq_a": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + }, + "wq_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + } + }, + "batch": { + "groups": [ + { + "count": 4, + "q": 1, + "ctx": 1024 + } + ] + } + }, + "sources": [ + "deepseek-ai/DeepSeek-V4.1-Flash/attn" + ] + }, + { + "type": "dsv4_attn", + "args": { + "hidden": 5120, + "heads": 64, + "head_dim": 512, + "rope_dim": 64, + "q_lora_rank": 1280, + "window": 128, + "o_groups": 8, + "o_lora_rank": 1024, + "compress_ratio": 2, + "indexer": "own", + "compress_rope_theta": 160000, + "rope_scaling": { + "type": "yarn", + "factor": 16, + "original_max": 65536, + "beta_fast": 32, + "beta_slow": 1 + }, + "topk": 512, + "index_heads": 32, + "index_dim": 128, + "proj": { + "indexer.wq_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + }, + "wkv": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + }, + "wo_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + }, + "wq_a": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + }, + "wq_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + } + }, + "batch": { + "groups": [ + { + "count": 4, + "q": 1, + "ctx": 4096 + } + ] + } + }, + "sources": [ + "deepseek-ai/DeepSeek-V4.1-Flash/attn" + ] + }, + { + "type": "dsv4_attn", + "args": { + "hidden": 5120, + "heads": 64, + "head_dim": 512, + "rope_dim": 64, + "q_lora_rank": 1280, + "window": 128, + "o_groups": 8, + "o_lora_rank": 1024, + "compress_ratio": 2, + "indexer": "own", + "compress_rope_theta": 160000, + "rope_scaling": { + "type": "yarn", + "factor": 16, + "original_max": 65536, + "beta_fast": 32, + "beta_slow": 1 + }, + "topk": 512, + "index_heads": 32, + "index_dim": 128, + "proj": { + "indexer.wq_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + }, + "wkv": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + }, + "wo_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + }, + "wq_a": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + }, + "wq_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + } + }, + "batch": { + "groups": [ + { + "count": 4, + "q": 1, + "ctx": 16384 + } + ] + } + }, + "sources": [ + "deepseek-ai/DeepSeek-V4.1-Flash/attn" + ] + }, + { + "type": "dsv4_attn", + "args": { + "hidden": 5120, + "heads": 64, + "head_dim": 512, + "rope_dim": 64, + "q_lora_rank": 1280, + "window": 128, + "o_groups": 8, + "o_lora_rank": 1024, + "compress_ratio": 2, + "indexer": "own", + "compress_rope_theta": 160000, + "rope_scaling": { + "type": "yarn", + "factor": 16, + "original_max": 65536, + "beta_fast": 32, + "beta_slow": 1 + }, + "topk": 512, + "index_heads": 32, + "index_dim": 128, + "proj": { + "indexer.wq_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + }, + "wkv": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + }, + "wo_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + }, + "wq_a": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + }, + "wq_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + } + }, + "batch": { + "groups": [ + { + "count": 4, + "q": 1, + "ctx": 65536 + } + ] + } + }, + "sources": [ + "deepseek-ai/DeepSeek-V4.1-Flash/attn" + ] + }, + { + "type": "dsv4_attn", + "args": { + "hidden": 5120, + "heads": 64, + "head_dim": 512, + "rope_dim": 64, + "q_lora_rank": 1280, + "window": 128, + "o_groups": 8, + "o_lora_rank": 1024, + "compress_ratio": 2, + "indexer": "own", + "compress_rope_theta": 160000, + "rope_scaling": { + "type": "yarn", + "factor": 16, + "original_max": 65536, + "beta_fast": 32, + "beta_slow": 1 + }, + "topk": 512, + "index_heads": 32, + "index_dim": 128, + "proj": { + "indexer.wq_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + }, + "wkv": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + }, + "wo_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + }, + "wq_a": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + }, + "wq_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + } + }, + "batch": { + "groups": [ + { + "count": 4, + "q": 1, + "ctx": 131072 + } + ] + } + }, + "sources": [ + "deepseek-ai/DeepSeek-V4.1-Flash/attn" + ] + }, + { + "type": "dsv4_attn", + "args": { + "hidden": 5120, + "heads": 64, + "head_dim": 512, + "rope_dim": 64, + "q_lora_rank": 1280, + "window": 128, + "o_groups": 8, + "o_lora_rank": 1024, + "compress_ratio": 2, + "indexer": "own", + "compress_rope_theta": 160000, + "rope_scaling": { + "type": "yarn", + "factor": 16, + "original_max": 65536, + "beta_fast": 32, + "beta_slow": 1 + }, + "topk": 512, + "index_heads": 32, + "index_dim": 128, + "proj": { + "indexer.wq_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + }, + "wkv": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + }, + "wo_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + }, + "wq_a": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + }, + "wq_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + } + }, + "batch": { + "groups": [ + { + "count": 16, + "q": 1, + "ctx": 1024 + } + ] + } + }, + "sources": [ + "deepseek-ai/DeepSeek-V4.1-Flash/attn" + ] + }, + { + "type": "dsv4_attn", + "args": { + "hidden": 5120, + "heads": 64, + "head_dim": 512, + "rope_dim": 64, + "q_lora_rank": 1280, + "window": 128, + "o_groups": 8, + "o_lora_rank": 1024, + "compress_ratio": 2, + "indexer": "own", + "compress_rope_theta": 160000, + "rope_scaling": { + "type": "yarn", + "factor": 16, + "original_max": 65536, + "beta_fast": 32, + "beta_slow": 1 + }, + "topk": 512, + "index_heads": 32, + "index_dim": 128, + "proj": { + "indexer.wq_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + }, + "wkv": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + }, + "wo_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + }, + "wq_a": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + }, + "wq_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + } + }, + "batch": { + "groups": [ + { + "count": 16, + "q": 1, + "ctx": 4096 + } + ] + } + }, + "sources": [ + "deepseek-ai/DeepSeek-V4.1-Flash/attn" + ] + }, + { + "type": "dsv4_attn", + "args": { + "hidden": 5120, + "heads": 64, + "head_dim": 512, + "rope_dim": 64, + "q_lora_rank": 1280, + "window": 128, + "o_groups": 8, + "o_lora_rank": 1024, + "compress_ratio": 2, + "indexer": "own", + "compress_rope_theta": 160000, + "rope_scaling": { + "type": "yarn", + "factor": 16, + "original_max": 65536, + "beta_fast": 32, + "beta_slow": 1 + }, + "topk": 512, + "index_heads": 32, + "index_dim": 128, + "proj": { + "indexer.wq_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + }, + "wkv": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + }, + "wo_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + }, + "wq_a": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + }, + "wq_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + } + }, + "batch": { + "groups": [ + { + "count": 16, + "q": 1, + "ctx": 16384 + } + ] + } + }, + "sources": [ + "deepseek-ai/DeepSeek-V4.1-Flash/attn" + ] + }, + { + "type": "dsv4_attn", + "args": { + "hidden": 5120, + "heads": 64, + "head_dim": 512, + "rope_dim": 64, + "q_lora_rank": 1280, + "window": 128, + "o_groups": 8, + "o_lora_rank": 1024, + "compress_ratio": 2, + "indexer": "own", + "compress_rope_theta": 160000, + "rope_scaling": { + "type": "yarn", + "factor": 16, + "original_max": 65536, + "beta_fast": 32, + "beta_slow": 1 + }, + "topk": 512, + "index_heads": 32, + "index_dim": 128, + "proj": { + "indexer.wq_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + }, + "wkv": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + }, + "wo_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + }, + "wq_a": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + }, + "wq_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + } + }, + "batch": { + "groups": [ + { + "count": 16, + "q": 1, + "ctx": 65536 + } + ] + } + }, + "sources": [ + "deepseek-ai/DeepSeek-V4.1-Flash/attn" + ] + }, + { + "type": "dsv4_attn", + "args": { + "hidden": 5120, + "heads": 64, + "head_dim": 512, + "rope_dim": 64, + "q_lora_rank": 1280, + "window": 128, + "o_groups": 8, + "o_lora_rank": 1024, + "compress_ratio": 2, + "indexer": "own", + "compress_rope_theta": 160000, + "rope_scaling": { + "type": "yarn", + "factor": 16, + "original_max": 65536, + "beta_fast": 32, + "beta_slow": 1 + }, + "topk": 512, + "index_heads": 32, + "index_dim": 128, + "proj": { + "indexer.wq_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + }, + "wkv": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + }, + "wo_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + }, + "wq_a": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + }, + "wq_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + } + }, + "batch": { + "groups": [ + { + "count": 16, + "q": 1, + "ctx": 131072 + } + ] + } + }, + "sources": [ + "deepseek-ai/DeepSeek-V4.1-Flash/attn" + ] + }, + { + "type": "dsv4_attn", + "args": { + "hidden": 5120, + "heads": 64, + "head_dim": 512, + "rope_dim": 64, + "q_lora_rank": 1280, + "window": 128, + "o_groups": 8, + "o_lora_rank": 1024, + "compress_ratio": 2, + "indexer": "own", + "compress_rope_theta": 160000, + "rope_scaling": { + "type": "yarn", + "factor": 16, + "original_max": 65536, + "beta_fast": 32, + "beta_slow": 1 + }, + "topk": 512, + "index_heads": 32, + "index_dim": 128, + "proj": { + "indexer.wq_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + }, + "wkv": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + }, + "wo_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + }, + "wq_a": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + }, + "wq_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + } + }, + "batch": { + "groups": [ + { + "count": 64, + "q": 1, + "ctx": 1024 + } + ] + } + }, + "sources": [ + "deepseek-ai/DeepSeek-V4.1-Flash/attn" + ] + }, + { + "type": "dsv4_attn", + "args": { + "hidden": 5120, + "heads": 64, + "head_dim": 512, + "rope_dim": 64, + "q_lora_rank": 1280, + "window": 128, + "o_groups": 8, + "o_lora_rank": 1024, + "compress_ratio": 2, + "indexer": "own", + "compress_rope_theta": 160000, + "rope_scaling": { + "type": "yarn", + "factor": 16, + "original_max": 65536, + "beta_fast": 32, + "beta_slow": 1 + }, + "topk": 512, + "index_heads": 32, + "index_dim": 128, + "proj": { + "indexer.wq_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + }, + "wkv": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + }, + "wo_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + }, + "wq_a": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + }, + "wq_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + } + }, + "batch": { + "groups": [ + { + "count": 64, + "q": 1, + "ctx": 4096 + } + ] + } + }, + "sources": [ + "deepseek-ai/DeepSeek-V4.1-Flash/attn" + ] + }, + { + "type": "dsv4_attn", + "args": { + "hidden": 5120, + "heads": 64, + "head_dim": 512, + "rope_dim": 64, + "q_lora_rank": 1280, + "window": 128, + "o_groups": 8, + "o_lora_rank": 1024, + "compress_ratio": 2, + "indexer": "own", + "compress_rope_theta": 160000, + "rope_scaling": { + "type": "yarn", + "factor": 16, + "original_max": 65536, + "beta_fast": 32, + "beta_slow": 1 + }, + "topk": 512, + "index_heads": 32, + "index_dim": 128, + "proj": { + "indexer.wq_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + }, + "wkv": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + }, + "wo_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + }, + "wq_a": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + }, + "wq_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + } + }, + "batch": { + "groups": [ + { + "count": 64, + "q": 1, + "ctx": 16384 + } + ] + } + }, + "sources": [ + "deepseek-ai/DeepSeek-V4.1-Flash/attn" + ] + }, + { + "type": "dsv4_attn", + "args": { + "hidden": 5120, + "heads": 64, + "head_dim": 512, + "rope_dim": 64, + "q_lora_rank": 1280, + "window": 128, + "o_groups": 8, + "o_lora_rank": 1024, + "compress_ratio": 2, + "indexer": "own", + "compress_rope_theta": 160000, + "rope_scaling": { + "type": "yarn", + "factor": 16, + "original_max": 65536, + "beta_fast": 32, + "beta_slow": 1 + }, + "topk": 512, + "index_heads": 32, + "index_dim": 128, + "proj": { + "indexer.wq_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + }, + "wkv": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + }, + "wo_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + }, + "wq_a": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + }, + "wq_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + } + }, + "batch": { + "groups": [ + { + "count": 64, + "q": 1, + "ctx": 65536 + } + ] + } + }, + "sources": [ + "deepseek-ai/DeepSeek-V4.1-Flash/attn" + ] + }, + { + "type": "dsv4_attn", + "args": { + "hidden": 5120, + "heads": 64, + "head_dim": 512, + "rope_dim": 64, + "q_lora_rank": 1280, + "window": 128, + "o_groups": 8, + "o_lora_rank": 1024, + "compress_ratio": 2, + "indexer": "own", + "compress_rope_theta": 160000, + "rope_scaling": { + "type": "yarn", + "factor": 16, + "original_max": 65536, + "beta_fast": 32, + "beta_slow": 1 + }, + "topk": 512, + "index_heads": 32, + "index_dim": 128, + "proj": { + "indexer.wq_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + }, + "wkv": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + }, + "wo_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + }, + "wq_a": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + }, + "wq_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + } + }, + "batch": { + "groups": [ + { + "count": 64, + "q": 1, + "ctx": 131072 + } + ] + } + }, + "sources": [ + "deepseek-ai/DeepSeek-V4.1-Flash/attn" + ] + }, + { + "type": "dsv4_attn", + "args": { + "hidden": 5120, + "heads": 64, + "head_dim": 512, + "rope_dim": 64, + "q_lora_rank": 1280, + "window": 128, + "o_groups": 8, + "o_lora_rank": 1024, + "compress_ratio": 2, + "indexer": "own", + "compress_rope_theta": 160000, + "rope_scaling": { + "type": "yarn", + "factor": 16, + "original_max": 65536, + "beta_fast": 32, + "beta_slow": 1 + }, + "topk": 512, + "index_heads": 32, + "index_dim": 128, + "proj": { + "indexer.wq_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + }, + "wkv": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + }, + "wo_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + }, + "wq_a": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + }, + "wq_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + } + }, + "batch": { + "groups": [ + { + "count": 256, + "q": 1, + "ctx": 1024 + } + ] + } + }, + "sources": [ + "deepseek-ai/DeepSeek-V4.1-Flash/attn" + ] + }, + { + "type": "dsv4_attn", + "args": { + "hidden": 5120, + "heads": 64, + "head_dim": 512, + "rope_dim": 64, + "q_lora_rank": 1280, + "window": 128, + "o_groups": 8, + "o_lora_rank": 1024, + "compress_ratio": 2, + "indexer": "own", + "compress_rope_theta": 160000, + "rope_scaling": { + "type": "yarn", + "factor": 16, + "original_max": 65536, + "beta_fast": 32, + "beta_slow": 1 + }, + "topk": 512, + "index_heads": 32, + "index_dim": 128, + "proj": { + "indexer.wq_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + }, + "wkv": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + }, + "wo_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + }, + "wq_a": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + }, + "wq_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + } + }, + "batch": { + "groups": [ + { + "count": 256, + "q": 1, + "ctx": 4096 + } + ] + } + }, + "sources": [ + "deepseek-ai/DeepSeek-V4.1-Flash/attn" + ] + }, + { + "type": "dsv4_attn", + "args": { + "hidden": 5120, + "heads": 64, + "head_dim": 512, + "rope_dim": 64, + "q_lora_rank": 1280, + "window": 128, + "o_groups": 8, + "o_lora_rank": 1024, + "compress_ratio": 2, + "indexer": "own", + "compress_rope_theta": 160000, + "rope_scaling": { + "type": "yarn", + "factor": 16, + "original_max": 65536, + "beta_fast": 32, + "beta_slow": 1 + }, + "topk": 512, + "index_heads": 32, + "index_dim": 128, + "proj": { + "indexer.wq_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + }, + "wkv": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + }, + "wo_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + }, + "wq_a": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + }, + "wq_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + } + }, + "batch": { + "groups": [ + { + "count": 256, + "q": 1, + "ctx": 16384 + } + ] + } + }, + "sources": [ + "deepseek-ai/DeepSeek-V4.1-Flash/attn" + ] + }, + { + "type": "dsv4_attn", + "args": { + "hidden": 5120, + "heads": 64, + "head_dim": 512, + "rope_dim": 64, + "q_lora_rank": 1280, + "window": 128, + "o_groups": 8, + "o_lora_rank": 1024, + "compress_ratio": 2, + "indexer": "own", + "compress_rope_theta": 160000, + "rope_scaling": { + "type": "yarn", + "factor": 16, + "original_max": 65536, + "beta_fast": 32, + "beta_slow": 1 + }, + "topk": 512, + "index_heads": 32, + "index_dim": 128, + "proj": { + "indexer.wq_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + }, + "wkv": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + }, + "wo_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + }, + "wq_a": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + }, + "wq_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + } + }, + "batch": { + "groups": [ + { + "count": 256, + "q": 1, + "ctx": 65536 + } + ] + } + }, + "sources": [ + "deepseek-ai/DeepSeek-V4.1-Flash/attn" + ] + }, + { + "type": "dsv4_attn", + "args": { + "hidden": 5120, + "heads": 64, + "head_dim": 512, + "rope_dim": 64, + "q_lora_rank": 1280, + "window": 128, + "o_groups": 8, + "o_lora_rank": 1024, + "compress_ratio": 2, + "indexer": "own", + "compress_rope_theta": 160000, + "rope_scaling": { + "type": "yarn", + "factor": 16, + "original_max": 65536, + "beta_fast": 32, + "beta_slow": 1 + }, + "topk": 512, + "index_heads": 32, + "index_dim": 128, + "proj": { + "indexer.wq_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + }, + "wkv": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + }, + "wo_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + }, + "wq_a": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + }, + "wq_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + } + }, + "batch": { + "groups": [ + { + "count": 256, + "q": 1, + "ctx": 131072 + } + ] + } + }, + "sources": [ + "deepseek-ai/DeepSeek-V4.1-Flash/attn" + ] + }, + { + "type": "dsv4_attn", + "args": { + "hidden": 5120, + "heads": 64, + "head_dim": 512, + "rope_dim": 64, + "q_lora_rank": 1280, + "window": 128, + "o_groups": 8, + "o_lora_rank": 1024, + "compress_ratio": 2, + "indexer": "own", + "compress_rope_theta": 160000, + "rope_scaling": { + "type": "yarn", + "factor": 16, + "original_max": 65536, + "beta_fast": 32, + "beta_slow": 1 + }, + "topk": 512, + "index_heads": 32, + "index_dim": 128, + "proj": { + "indexer.wq_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + }, + "wkv": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + }, + "wo_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + }, + "wq_a": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + }, + "wq_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + } + }, + "batch": { + "groups": [ + { + "count": 1, + "q": 1024, + "ctx": 0 + } + ] + } + }, + "sources": [ + "deepseek-ai/DeepSeek-V4.1-Flash/attn" + ] + }, + { + "type": "dsv4_attn", + "args": { + "hidden": 5120, + "heads": 64, + "head_dim": 512, + "rope_dim": 64, + "q_lora_rank": 1280, + "window": 128, + "o_groups": 8, + "o_lora_rank": 1024, + "compress_ratio": 2, + "indexer": "own", + "compress_rope_theta": 160000, + "rope_scaling": { + "type": "yarn", + "factor": 16, + "original_max": 65536, + "beta_fast": 32, + "beta_slow": 1 + }, + "topk": 512, + "index_heads": 32, + "index_dim": 128, + "proj": { + "indexer.wq_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + }, + "wkv": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + }, + "wo_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + }, + "wq_a": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + }, + "wq_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + } + }, + "batch": { + "groups": [ + { + "count": 1, + "q": 4096, + "ctx": 0 + } + ] + } + }, + "sources": [ + "deepseek-ai/DeepSeek-V4.1-Flash/attn" + ] + }, + { + "type": "dsv4_attn", + "args": { + "hidden": 5120, + "heads": 64, + "head_dim": 512, + "rope_dim": 64, + "q_lora_rank": 1280, + "window": 128, + "o_groups": 8, + "o_lora_rank": 1024, + "compress_ratio": 2, + "indexer": "own", + "compress_rope_theta": 160000, + "rope_scaling": { + "type": "yarn", + "factor": 16, + "original_max": 65536, + "beta_fast": 32, + "beta_slow": 1 + }, + "topk": 512, + "index_heads": 32, + "index_dim": 128, + "proj": { + "indexer.wq_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + }, + "wkv": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + }, + "wo_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + }, + "wq_a": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + }, + "wq_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + } + }, + "batch": { + "groups": [ + { + "count": 1, + "q": 16384, + "ctx": 0 + } + ] + } + }, + "sources": [ + "deepseek-ai/DeepSeek-V4.1-Flash/attn" + ] + }, + { + "type": "dsv4_attn", + "args": { + "hidden": 5120, + "heads": 64, + "head_dim": 512, + "rope_dim": 64, + "q_lora_rank": 1280, + "window": 128, + "o_groups": 8, + "o_lora_rank": 1024, + "compress_ratio": 2, + "indexer": "own", + "compress_rope_theta": 160000, + "rope_scaling": { + "type": "yarn", + "factor": 16, + "original_max": 65536, + "beta_fast": 32, + "beta_slow": 1 + }, + "topk": 512, + "index_heads": 32, + "index_dim": 128, + "proj": { + "indexer.wq_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + }, + "wkv": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + }, + "wo_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + }, + "wq_a": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + }, + "wq_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + } + }, + "batch": { + "groups": [ + { + "count": 1, + "q": 32768, + "ctx": 0 + } + ] + } + }, + "sources": [ + "deepseek-ai/DeepSeek-V4.1-Flash/attn" + ] + }, + { + "type": "dsv4_attn", + "args": { + "hidden": 5120, + "heads": 64, + "head_dim": 512, + "rope_dim": 64, + "q_lora_rank": 1280, + "window": 128, + "o_groups": 8, + "o_lora_rank": 1024, + "compress_ratio": 2, + "indexer": "own", + "compress_rope_theta": 160000, + "rope_scaling": { + "type": "yarn", + "factor": 16, + "original_max": 65536, + "beta_fast": 32, + "beta_slow": 1 + }, + "topk": 512, + "index_heads": 32, + "index_dim": 128, + "proj": { + "indexer.wq_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + }, + "wkv": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + }, + "wo_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + }, + "wq_a": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + }, + "wq_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + } + }, + "batch": { + "groups": [ + { + "count": 1, + "q": 8192, + "ctx": 8192 + } + ] + } + }, + "sources": [ + "deepseek-ai/DeepSeek-V4.1-Flash/attn" + ] + }, + { + "type": "dsv4_attn", + "args": { + "hidden": 5120, + "heads": 64, + "head_dim": 512, + "rope_dim": 64, + "q_lora_rank": 1280, + "window": 128, + "o_groups": 8, + "o_lora_rank": 1024, + "compress_ratio": 2, + "indexer": "own", + "compress_rope_theta": 160000, + "rope_scaling": { + "type": "yarn", + "factor": 16, + "original_max": 65536, + "beta_fast": 32, + "beta_slow": 1 + }, + "topk": 512, + "index_heads": 32, + "index_dim": 128, + "proj": { + "indexer.wq_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + }, + "wkv": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + }, + "wo_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + }, + "wq_a": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + }, + "wq_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + } + }, + "batch": { + "groups": [ + { + "count": 1, + "q": 8192, + "ctx": 32768 + } + ] + } + }, + "sources": [ + "deepseek-ai/DeepSeek-V4.1-Flash/attn" + ] + }, + { + "type": "dsv4_attn", + "args": { + "hidden": 5120, + "heads": 64, + "head_dim": 512, + "rope_dim": 64, + "q_lora_rank": 1280, + "window": 128, + "o_groups": 8, + "o_lora_rank": 1024, + "compress_ratio": 2, + "indexer": "own", + "compress_rope_theta": 160000, + "rope_scaling": { + "type": "yarn", + "factor": 16, + "original_max": 65536, + "beta_fast": 32, + "beta_slow": 1 + }, + "topk": 512, + "index_heads": 32, + "index_dim": 128, + "proj": { + "indexer.wq_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + }, + "wkv": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + }, + "wo_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + }, + "wq_a": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + }, + "wq_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + } + }, + "batch": { + "groups": [ + { + "count": 1, + "q": 8192, + "ctx": 131072 + } + ] + } + }, + "sources": [ + "deepseek-ai/DeepSeek-V4.1-Flash/attn" + ] + }, + { + "type": "dsv4_attn", + "args": { + "hidden": 5120, + "heads": 64, + "head_dim": 512, + "rope_dim": 64, + "q_lora_rank": 1280, + "window": 128, + "o_groups": 8, + "o_lora_rank": 1024, + "compress_ratio": 2, + "indexer": "own", + "compress_rope_theta": 160000, + "rope_scaling": { + "type": "yarn", + "factor": 16, + "original_max": 65536, + "beta_fast": 32, + "beta_slow": 1 + }, + "topk": 512, + "index_heads": 32, + "index_dim": 128, + "proj": { + "indexer.wq_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + }, + "wkv": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + }, + "wo_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + }, + "wq_a": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + }, + "wq_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + } + }, + "batch": { + "groups": [ + { + "count": 1, + "q": 8192, + "ctx": 8192 + }, + { + "count": 64, + "q": 1, + "ctx": 8192 + } + ] + } + }, + "sources": [ + "deepseek-ai/DeepSeek-V4.1-Flash/attn" + ] + }, + { + "type": "dsv4_attn", + "args": { + "hidden": 5120, + "heads": 64, + "head_dim": 512, + "rope_dim": 64, + "q_lora_rank": 1280, + "window": 128, + "o_groups": 8, + "o_lora_rank": 1024, + "compress_ratio": 2, + "source": false, + "indexer": "reuse", + "compress_rope_theta": 160000, + "rope_scaling": { + "type": "yarn", + "factor": 16, + "original_max": 65536, + "beta_fast": 32, + "beta_slow": 1 + }, + "topk": 512, + "index_heads": 32, + "index_dim": 128, + "proj": { + "wkv": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + }, + "wo_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + }, + "wq_a": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + }, + "wq_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + } + }, + "batch": { + "groups": [ + { + "count": 1, + "q": 1, + "ctx": 1024 + } + ] + } + }, + "sources": [ + "deepseek-ai/DeepSeek-V4.1-Flash/attn" + ] + }, + { + "type": "dsv4_attn", + "args": { + "hidden": 5120, + "heads": 64, + "head_dim": 512, + "rope_dim": 64, + "q_lora_rank": 1280, + "window": 128, + "o_groups": 8, + "o_lora_rank": 1024, + "compress_ratio": 2, + "source": false, + "indexer": "reuse", + "compress_rope_theta": 160000, + "rope_scaling": { + "type": "yarn", + "factor": 16, + "original_max": 65536, + "beta_fast": 32, + "beta_slow": 1 + }, + "topk": 512, + "index_heads": 32, + "index_dim": 128, + "proj": { + "wkv": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + }, + "wo_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + }, + "wq_a": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + }, + "wq_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + } + }, + "batch": { + "groups": [ + { + "count": 1, + "q": 1, + "ctx": 4096 + } + ] + } + }, + "sources": [ + "deepseek-ai/DeepSeek-V4.1-Flash/attn" + ] + }, + { + "type": "dsv4_attn", + "args": { + "hidden": 5120, + "heads": 64, + "head_dim": 512, + "rope_dim": 64, + "q_lora_rank": 1280, + "window": 128, + "o_groups": 8, + "o_lora_rank": 1024, + "compress_ratio": 2, + "source": false, + "indexer": "reuse", + "compress_rope_theta": 160000, + "rope_scaling": { + "type": "yarn", + "factor": 16, + "original_max": 65536, + "beta_fast": 32, + "beta_slow": 1 + }, + "topk": 512, + "index_heads": 32, + "index_dim": 128, + "proj": { + "wkv": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + }, + "wo_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + }, + "wq_a": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + }, + "wq_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + } + }, + "batch": { + "groups": [ + { + "count": 1, + "q": 1, + "ctx": 16384 + } + ] + } + }, + "sources": [ + "deepseek-ai/DeepSeek-V4.1-Flash/attn" + ] + }, + { + "type": "dsv4_attn", + "args": { + "hidden": 5120, + "heads": 64, + "head_dim": 512, + "rope_dim": 64, + "q_lora_rank": 1280, + "window": 128, + "o_groups": 8, + "o_lora_rank": 1024, + "compress_ratio": 2, + "source": false, + "indexer": "reuse", + "compress_rope_theta": 160000, + "rope_scaling": { + "type": "yarn", + "factor": 16, + "original_max": 65536, + "beta_fast": 32, + "beta_slow": 1 + }, + "topk": 512, + "index_heads": 32, + "index_dim": 128, + "proj": { + "wkv": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + }, + "wo_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + }, + "wq_a": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + }, + "wq_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + } + }, + "batch": { + "groups": [ + { + "count": 1, + "q": 1, + "ctx": 65536 + } + ] + } + }, + "sources": [ + "deepseek-ai/DeepSeek-V4.1-Flash/attn" + ] + }, + { + "type": "dsv4_attn", + "args": { + "hidden": 5120, + "heads": 64, + "head_dim": 512, + "rope_dim": 64, + "q_lora_rank": 1280, + "window": 128, + "o_groups": 8, + "o_lora_rank": 1024, + "compress_ratio": 2, + "source": false, + "indexer": "reuse", + "compress_rope_theta": 160000, + "rope_scaling": { + "type": "yarn", + "factor": 16, + "original_max": 65536, + "beta_fast": 32, + "beta_slow": 1 + }, + "topk": 512, + "index_heads": 32, + "index_dim": 128, + "proj": { + "wkv": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + }, + "wo_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + }, + "wq_a": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + }, + "wq_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + } + }, + "batch": { + "groups": [ + { + "count": 1, + "q": 1, + "ctx": 131072 + } + ] + } + }, + "sources": [ + "deepseek-ai/DeepSeek-V4.1-Flash/attn" + ] + }, + { + "type": "dsv4_attn", + "args": { + "hidden": 5120, + "heads": 64, + "head_dim": 512, + "rope_dim": 64, + "q_lora_rank": 1280, + "window": 128, + "o_groups": 8, + "o_lora_rank": 1024, + "compress_ratio": 2, + "source": false, + "indexer": "reuse", + "compress_rope_theta": 160000, + "rope_scaling": { + "type": "yarn", + "factor": 16, + "original_max": 65536, + "beta_fast": 32, + "beta_slow": 1 + }, + "topk": 512, + "index_heads": 32, + "index_dim": 128, + "proj": { + "wkv": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + }, + "wo_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + }, + "wq_a": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + }, + "wq_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + } + }, + "batch": { + "groups": [ + { + "count": 4, + "q": 1, + "ctx": 1024 + } + ] + } + }, + "sources": [ + "deepseek-ai/DeepSeek-V4.1-Flash/attn" + ] + }, + { + "type": "dsv4_attn", + "args": { + "hidden": 5120, + "heads": 64, + "head_dim": 512, + "rope_dim": 64, + "q_lora_rank": 1280, + "window": 128, + "o_groups": 8, + "o_lora_rank": 1024, + "compress_ratio": 2, + "source": false, + "indexer": "reuse", + "compress_rope_theta": 160000, + "rope_scaling": { + "type": "yarn", + "factor": 16, + "original_max": 65536, + "beta_fast": 32, + "beta_slow": 1 + }, + "topk": 512, + "index_heads": 32, + "index_dim": 128, + "proj": { + "wkv": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + }, + "wo_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + }, + "wq_a": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + }, + "wq_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + } + }, + "batch": { + "groups": [ + { + "count": 4, + "q": 1, + "ctx": 4096 + } + ] + } + }, + "sources": [ + "deepseek-ai/DeepSeek-V4.1-Flash/attn" + ] + }, + { + "type": "dsv4_attn", + "args": { + "hidden": 5120, + "heads": 64, + "head_dim": 512, + "rope_dim": 64, + "q_lora_rank": 1280, + "window": 128, + "o_groups": 8, + "o_lora_rank": 1024, + "compress_ratio": 2, + "source": false, + "indexer": "reuse", + "compress_rope_theta": 160000, + "rope_scaling": { + "type": "yarn", + "factor": 16, + "original_max": 65536, + "beta_fast": 32, + "beta_slow": 1 + }, + "topk": 512, + "index_heads": 32, + "index_dim": 128, + "proj": { + "wkv": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + }, + "wo_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + }, + "wq_a": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + }, + "wq_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + } + }, + "batch": { + "groups": [ + { + "count": 4, + "q": 1, + "ctx": 16384 + } + ] + } + }, + "sources": [ + "deepseek-ai/DeepSeek-V4.1-Flash/attn" + ] + }, + { + "type": "dsv4_attn", + "args": { + "hidden": 5120, + "heads": 64, + "head_dim": 512, + "rope_dim": 64, + "q_lora_rank": 1280, + "window": 128, + "o_groups": 8, + "o_lora_rank": 1024, + "compress_ratio": 2, + "source": false, + "indexer": "reuse", + "compress_rope_theta": 160000, + "rope_scaling": { + "type": "yarn", + "factor": 16, + "original_max": 65536, + "beta_fast": 32, + "beta_slow": 1 + }, + "topk": 512, + "index_heads": 32, + "index_dim": 128, + "proj": { + "wkv": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + }, + "wo_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + }, + "wq_a": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + }, + "wq_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + } + }, + "batch": { + "groups": [ + { + "count": 4, + "q": 1, + "ctx": 65536 + } + ] + } + }, + "sources": [ + "deepseek-ai/DeepSeek-V4.1-Flash/attn" + ] + }, + { + "type": "dsv4_attn", + "args": { + "hidden": 5120, + "heads": 64, + "head_dim": 512, + "rope_dim": 64, + "q_lora_rank": 1280, + "window": 128, + "o_groups": 8, + "o_lora_rank": 1024, + "compress_ratio": 2, + "source": false, + "indexer": "reuse", + "compress_rope_theta": 160000, + "rope_scaling": { + "type": "yarn", + "factor": 16, + "original_max": 65536, + "beta_fast": 32, + "beta_slow": 1 + }, + "topk": 512, + "index_heads": 32, + "index_dim": 128, + "proj": { + "wkv": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + }, + "wo_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + }, + "wq_a": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + }, + "wq_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + } + }, + "batch": { + "groups": [ + { + "count": 4, + "q": 1, + "ctx": 131072 + } + ] + } + }, + "sources": [ + "deepseek-ai/DeepSeek-V4.1-Flash/attn" + ] + }, + { + "type": "dsv4_attn", + "args": { + "hidden": 5120, + "heads": 64, + "head_dim": 512, + "rope_dim": 64, + "q_lora_rank": 1280, + "window": 128, + "o_groups": 8, + "o_lora_rank": 1024, + "compress_ratio": 2, + "source": false, + "indexer": "reuse", + "compress_rope_theta": 160000, + "rope_scaling": { + "type": "yarn", + "factor": 16, + "original_max": 65536, + "beta_fast": 32, + "beta_slow": 1 + }, + "topk": 512, + "index_heads": 32, + "index_dim": 128, + "proj": { + "wkv": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + }, + "wo_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + }, + "wq_a": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + }, + "wq_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + } + }, + "batch": { + "groups": [ + { + "count": 16, + "q": 1, + "ctx": 1024 + } + ] + } + }, + "sources": [ + "deepseek-ai/DeepSeek-V4.1-Flash/attn" + ] + }, + { + "type": "dsv4_attn", + "args": { + "hidden": 5120, + "heads": 64, + "head_dim": 512, + "rope_dim": 64, + "q_lora_rank": 1280, + "window": 128, + "o_groups": 8, + "o_lora_rank": 1024, + "compress_ratio": 2, + "source": false, + "indexer": "reuse", + "compress_rope_theta": 160000, + "rope_scaling": { + "type": "yarn", + "factor": 16, + "original_max": 65536, + "beta_fast": 32, + "beta_slow": 1 + }, + "topk": 512, + "index_heads": 32, + "index_dim": 128, + "proj": { + "wkv": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + }, + "wo_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + }, + "wq_a": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + }, + "wq_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + } + }, + "batch": { + "groups": [ + { + "count": 16, + "q": 1, + "ctx": 4096 + } + ] + } + }, + "sources": [ + "deepseek-ai/DeepSeek-V4.1-Flash/attn" + ] + }, + { + "type": "dsv4_attn", + "args": { + "hidden": 5120, + "heads": 64, + "head_dim": 512, + "rope_dim": 64, + "q_lora_rank": 1280, + "window": 128, + "o_groups": 8, + "o_lora_rank": 1024, + "compress_ratio": 2, + "source": false, + "indexer": "reuse", + "compress_rope_theta": 160000, + "rope_scaling": { + "type": "yarn", + "factor": 16, + "original_max": 65536, + "beta_fast": 32, + "beta_slow": 1 + }, + "topk": 512, + "index_heads": 32, + "index_dim": 128, + "proj": { + "wkv": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + }, + "wo_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + }, + "wq_a": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + }, + "wq_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + } + }, + "batch": { + "groups": [ + { + "count": 16, + "q": 1, + "ctx": 16384 + } + ] + } + }, + "sources": [ + "deepseek-ai/DeepSeek-V4.1-Flash/attn" + ] + }, + { + "type": "dsv4_attn", + "args": { + "hidden": 5120, + "heads": 64, + "head_dim": 512, + "rope_dim": 64, + "q_lora_rank": 1280, + "window": 128, + "o_groups": 8, + "o_lora_rank": 1024, + "compress_ratio": 2, + "source": false, + "indexer": "reuse", + "compress_rope_theta": 160000, + "rope_scaling": { + "type": "yarn", + "factor": 16, + "original_max": 65536, + "beta_fast": 32, + "beta_slow": 1 + }, + "topk": 512, + "index_heads": 32, + "index_dim": 128, + "proj": { + "wkv": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + }, + "wo_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + }, + "wq_a": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + }, + "wq_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + } + }, + "batch": { + "groups": [ + { + "count": 16, + "q": 1, + "ctx": 65536 + } + ] + } + }, + "sources": [ + "deepseek-ai/DeepSeek-V4.1-Flash/attn" + ] + }, + { + "type": "dsv4_attn", + "args": { + "hidden": 5120, + "heads": 64, + "head_dim": 512, + "rope_dim": 64, + "q_lora_rank": 1280, + "window": 128, + "o_groups": 8, + "o_lora_rank": 1024, + "compress_ratio": 2, + "source": false, + "indexer": "reuse", + "compress_rope_theta": 160000, + "rope_scaling": { + "type": "yarn", + "factor": 16, + "original_max": 65536, + "beta_fast": 32, + "beta_slow": 1 + }, + "topk": 512, + "index_heads": 32, + "index_dim": 128, + "proj": { + "wkv": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + }, + "wo_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + }, + "wq_a": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + }, + "wq_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + } + }, + "batch": { + "groups": [ + { + "count": 16, + "q": 1, + "ctx": 131072 + } + ] + } + }, + "sources": [ + "deepseek-ai/DeepSeek-V4.1-Flash/attn" + ] + }, + { + "type": "dsv4_attn", + "args": { + "hidden": 5120, + "heads": 64, + "head_dim": 512, + "rope_dim": 64, + "q_lora_rank": 1280, + "window": 128, + "o_groups": 8, + "o_lora_rank": 1024, + "compress_ratio": 2, + "source": false, + "indexer": "reuse", + "compress_rope_theta": 160000, + "rope_scaling": { + "type": "yarn", + "factor": 16, + "original_max": 65536, + "beta_fast": 32, + "beta_slow": 1 + }, + "topk": 512, + "index_heads": 32, + "index_dim": 128, + "proj": { + "wkv": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + }, + "wo_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + }, + "wq_a": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + }, + "wq_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + } + }, + "batch": { + "groups": [ + { + "count": 64, + "q": 1, + "ctx": 1024 + } + ] + } + }, + "sources": [ + "deepseek-ai/DeepSeek-V4.1-Flash/attn" + ] + }, + { + "type": "dsv4_attn", + "args": { + "hidden": 5120, + "heads": 64, + "head_dim": 512, + "rope_dim": 64, + "q_lora_rank": 1280, + "window": 128, + "o_groups": 8, + "o_lora_rank": 1024, + "compress_ratio": 2, + "source": false, + "indexer": "reuse", + "compress_rope_theta": 160000, + "rope_scaling": { + "type": "yarn", + "factor": 16, + "original_max": 65536, + "beta_fast": 32, + "beta_slow": 1 + }, + "topk": 512, + "index_heads": 32, + "index_dim": 128, + "proj": { + "wkv": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + }, + "wo_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + }, + "wq_a": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + }, + "wq_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + } + }, + "batch": { + "groups": [ + { + "count": 64, + "q": 1, + "ctx": 4096 + } + ] + } + }, + "sources": [ + "deepseek-ai/DeepSeek-V4.1-Flash/attn" + ] + }, + { + "type": "dsv4_attn", + "args": { + "hidden": 5120, + "heads": 64, + "head_dim": 512, + "rope_dim": 64, + "q_lora_rank": 1280, + "window": 128, + "o_groups": 8, + "o_lora_rank": 1024, + "compress_ratio": 2, + "source": false, + "indexer": "reuse", + "compress_rope_theta": 160000, + "rope_scaling": { + "type": "yarn", + "factor": 16, + "original_max": 65536, + "beta_fast": 32, + "beta_slow": 1 + }, + "topk": 512, + "index_heads": 32, + "index_dim": 128, + "proj": { + "wkv": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + }, + "wo_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + }, + "wq_a": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + }, + "wq_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + } + }, + "batch": { + "groups": [ + { + "count": 64, + "q": 1, + "ctx": 16384 + } + ] + } + }, + "sources": [ + "deepseek-ai/DeepSeek-V4.1-Flash/attn" + ] + }, + { + "type": "dsv4_attn", + "args": { + "hidden": 5120, + "heads": 64, + "head_dim": 512, + "rope_dim": 64, + "q_lora_rank": 1280, + "window": 128, + "o_groups": 8, + "o_lora_rank": 1024, + "compress_ratio": 2, + "source": false, + "indexer": "reuse", + "compress_rope_theta": 160000, + "rope_scaling": { + "type": "yarn", + "factor": 16, + "original_max": 65536, + "beta_fast": 32, + "beta_slow": 1 + }, + "topk": 512, + "index_heads": 32, + "index_dim": 128, + "proj": { + "wkv": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + }, + "wo_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + }, + "wq_a": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + }, + "wq_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + } + }, + "batch": { + "groups": [ + { + "count": 64, + "q": 1, + "ctx": 65536 + } + ] + } + }, + "sources": [ + "deepseek-ai/DeepSeek-V4.1-Flash/attn" + ] + }, + { + "type": "dsv4_attn", + "args": { + "hidden": 5120, + "heads": 64, + "head_dim": 512, + "rope_dim": 64, + "q_lora_rank": 1280, + "window": 128, + "o_groups": 8, + "o_lora_rank": 1024, + "compress_ratio": 2, + "source": false, + "indexer": "reuse", + "compress_rope_theta": 160000, + "rope_scaling": { + "type": "yarn", + "factor": 16, + "original_max": 65536, + "beta_fast": 32, + "beta_slow": 1 + }, + "topk": 512, + "index_heads": 32, + "index_dim": 128, + "proj": { + "wkv": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + }, + "wo_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + }, + "wq_a": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + }, + "wq_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + } + }, + "batch": { + "groups": [ + { + "count": 64, + "q": 1, + "ctx": 131072 + } + ] + } + }, + "sources": [ + "deepseek-ai/DeepSeek-V4.1-Flash/attn" + ] + }, + { + "type": "dsv4_attn", + "args": { + "hidden": 5120, + "heads": 64, + "head_dim": 512, + "rope_dim": 64, + "q_lora_rank": 1280, + "window": 128, + "o_groups": 8, + "o_lora_rank": 1024, + "compress_ratio": 2, + "source": false, + "indexer": "reuse", + "compress_rope_theta": 160000, + "rope_scaling": { + "type": "yarn", + "factor": 16, + "original_max": 65536, + "beta_fast": 32, + "beta_slow": 1 + }, + "topk": 512, + "index_heads": 32, + "index_dim": 128, + "proj": { + "wkv": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + }, + "wo_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + }, + "wq_a": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + }, + "wq_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + } + }, + "batch": { + "groups": [ + { + "count": 256, + "q": 1, + "ctx": 1024 + } + ] + } + }, + "sources": [ + "deepseek-ai/DeepSeek-V4.1-Flash/attn" + ] + }, + { + "type": "dsv4_attn", + "args": { + "hidden": 5120, + "heads": 64, + "head_dim": 512, + "rope_dim": 64, + "q_lora_rank": 1280, + "window": 128, + "o_groups": 8, + "o_lora_rank": 1024, + "compress_ratio": 2, + "source": false, + "indexer": "reuse", + "compress_rope_theta": 160000, + "rope_scaling": { + "type": "yarn", + "factor": 16, + "original_max": 65536, + "beta_fast": 32, + "beta_slow": 1 + }, + "topk": 512, + "index_heads": 32, + "index_dim": 128, + "proj": { + "wkv": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + }, + "wo_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + }, + "wq_a": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + }, + "wq_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + } + }, + "batch": { + "groups": [ + { + "count": 256, + "q": 1, + "ctx": 4096 + } + ] + } + }, + "sources": [ + "deepseek-ai/DeepSeek-V4.1-Flash/attn" + ] + }, + { + "type": "dsv4_attn", + "args": { + "hidden": 5120, + "heads": 64, + "head_dim": 512, + "rope_dim": 64, + "q_lora_rank": 1280, + "window": 128, + "o_groups": 8, + "o_lora_rank": 1024, + "compress_ratio": 2, + "source": false, + "indexer": "reuse", + "compress_rope_theta": 160000, + "rope_scaling": { + "type": "yarn", + "factor": 16, + "original_max": 65536, + "beta_fast": 32, + "beta_slow": 1 + }, + "topk": 512, + "index_heads": 32, + "index_dim": 128, + "proj": { + "wkv": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + }, + "wo_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + }, + "wq_a": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + }, + "wq_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + } + }, + "batch": { + "groups": [ + { + "count": 256, + "q": 1, + "ctx": 16384 + } + ] + } + }, + "sources": [ + "deepseek-ai/DeepSeek-V4.1-Flash/attn" + ] + }, + { + "type": "dsv4_attn", + "args": { + "hidden": 5120, + "heads": 64, + "head_dim": 512, + "rope_dim": 64, + "q_lora_rank": 1280, + "window": 128, + "o_groups": 8, + "o_lora_rank": 1024, + "compress_ratio": 2, + "source": false, + "indexer": "reuse", + "compress_rope_theta": 160000, + "rope_scaling": { + "type": "yarn", + "factor": 16, + "original_max": 65536, + "beta_fast": 32, + "beta_slow": 1 + }, + "topk": 512, + "index_heads": 32, + "index_dim": 128, + "proj": { + "wkv": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + }, + "wo_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + }, + "wq_a": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + }, + "wq_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + } + }, + "batch": { + "groups": [ + { + "count": 256, + "q": 1, + "ctx": 65536 + } + ] + } + }, + "sources": [ + "deepseek-ai/DeepSeek-V4.1-Flash/attn" + ] + }, + { + "type": "dsv4_attn", + "args": { + "hidden": 5120, + "heads": 64, + "head_dim": 512, + "rope_dim": 64, + "q_lora_rank": 1280, + "window": 128, + "o_groups": 8, + "o_lora_rank": 1024, + "compress_ratio": 2, + "source": false, + "indexer": "reuse", + "compress_rope_theta": 160000, + "rope_scaling": { + "type": "yarn", + "factor": 16, + "original_max": 65536, + "beta_fast": 32, + "beta_slow": 1 + }, + "topk": 512, + "index_heads": 32, + "index_dim": 128, + "proj": { + "wkv": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + }, + "wo_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + }, + "wq_a": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + }, + "wq_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + } + }, + "batch": { + "groups": [ + { + "count": 256, + "q": 1, + "ctx": 131072 + } + ] + } + }, + "sources": [ + "deepseek-ai/DeepSeek-V4.1-Flash/attn" + ] + }, + { + "type": "dsv4_attn", + "args": { + "hidden": 5120, + "heads": 64, + "head_dim": 512, + "rope_dim": 64, + "q_lora_rank": 1280, + "window": 128, + "o_groups": 8, + "o_lora_rank": 1024, + "compress_ratio": 2, + "source": false, + "indexer": "reuse", + "compress_rope_theta": 160000, + "rope_scaling": { + "type": "yarn", + "factor": 16, + "original_max": 65536, + "beta_fast": 32, + "beta_slow": 1 + }, + "topk": 512, + "index_heads": 32, + "index_dim": 128, + "proj": { + "wkv": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + }, + "wo_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + }, + "wq_a": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + }, + "wq_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + } + }, + "batch": { + "groups": [ + { + "count": 1, + "q": 1024, + "ctx": 0 + } + ] + } + }, + "sources": [ + "deepseek-ai/DeepSeek-V4.1-Flash/attn" + ] + }, + { + "type": "dsv4_attn", + "args": { + "hidden": 5120, + "heads": 64, + "head_dim": 512, + "rope_dim": 64, + "q_lora_rank": 1280, + "window": 128, + "o_groups": 8, + "o_lora_rank": 1024, + "compress_ratio": 2, + "source": false, + "indexer": "reuse", + "compress_rope_theta": 160000, + "rope_scaling": { + "type": "yarn", + "factor": 16, + "original_max": 65536, + "beta_fast": 32, + "beta_slow": 1 + }, + "topk": 512, + "index_heads": 32, + "index_dim": 128, + "proj": { + "wkv": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + }, + "wo_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + }, + "wq_a": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + }, + "wq_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + } + }, + "batch": { + "groups": [ + { + "count": 1, + "q": 4096, + "ctx": 0 + } + ] + } + }, + "sources": [ + "deepseek-ai/DeepSeek-V4.1-Flash/attn" + ] + }, + { + "type": "dsv4_attn", + "args": { + "hidden": 5120, + "heads": 64, + "head_dim": 512, + "rope_dim": 64, + "q_lora_rank": 1280, + "window": 128, + "o_groups": 8, + "o_lora_rank": 1024, + "compress_ratio": 2, + "source": false, + "indexer": "reuse", + "compress_rope_theta": 160000, + "rope_scaling": { + "type": "yarn", + "factor": 16, + "original_max": 65536, + "beta_fast": 32, + "beta_slow": 1 + }, + "topk": 512, + "index_heads": 32, + "index_dim": 128, + "proj": { + "wkv": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + }, + "wo_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + }, + "wq_a": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + }, + "wq_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + } + }, + "batch": { + "groups": [ + { + "count": 1, + "q": 16384, + "ctx": 0 + } + ] + } + }, + "sources": [ + "deepseek-ai/DeepSeek-V4.1-Flash/attn" + ] + }, + { + "type": "dsv4_attn", + "args": { + "hidden": 5120, + "heads": 64, + "head_dim": 512, + "rope_dim": 64, + "q_lora_rank": 1280, + "window": 128, + "o_groups": 8, + "o_lora_rank": 1024, + "compress_ratio": 2, + "source": false, + "indexer": "reuse", + "compress_rope_theta": 160000, + "rope_scaling": { + "type": "yarn", + "factor": 16, + "original_max": 65536, + "beta_fast": 32, + "beta_slow": 1 + }, + "topk": 512, + "index_heads": 32, + "index_dim": 128, + "proj": { + "wkv": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + }, + "wo_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + }, + "wq_a": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + }, + "wq_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + } + }, + "batch": { + "groups": [ + { + "count": 1, + "q": 32768, + "ctx": 0 + } + ] + } + }, + "sources": [ + "deepseek-ai/DeepSeek-V4.1-Flash/attn" + ] + }, + { + "type": "dsv4_attn", + "args": { + "hidden": 5120, + "heads": 64, + "head_dim": 512, + "rope_dim": 64, + "q_lora_rank": 1280, + "window": 128, + "o_groups": 8, + "o_lora_rank": 1024, + "compress_ratio": 2, + "source": false, + "indexer": "reuse", + "compress_rope_theta": 160000, + "rope_scaling": { + "type": "yarn", + "factor": 16, + "original_max": 65536, + "beta_fast": 32, + "beta_slow": 1 + }, + "topk": 512, + "index_heads": 32, + "index_dim": 128, + "proj": { + "wkv": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + }, + "wo_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + }, + "wq_a": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + }, + "wq_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + } + }, + "batch": { + "groups": [ + { + "count": 1, + "q": 8192, + "ctx": 8192 + } + ] + } + }, + "sources": [ + "deepseek-ai/DeepSeek-V4.1-Flash/attn" + ] + }, + { + "type": "dsv4_attn", + "args": { + "hidden": 5120, + "heads": 64, + "head_dim": 512, + "rope_dim": 64, + "q_lora_rank": 1280, + "window": 128, + "o_groups": 8, + "o_lora_rank": 1024, + "compress_ratio": 2, + "source": false, + "indexer": "reuse", + "compress_rope_theta": 160000, + "rope_scaling": { + "type": "yarn", + "factor": 16, + "original_max": 65536, + "beta_fast": 32, + "beta_slow": 1 + }, + "topk": 512, + "index_heads": 32, + "index_dim": 128, + "proj": { + "wkv": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + }, + "wo_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + }, + "wq_a": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + }, + "wq_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + } + }, + "batch": { + "groups": [ + { + "count": 1, + "q": 8192, + "ctx": 32768 + } + ] + } + }, + "sources": [ + "deepseek-ai/DeepSeek-V4.1-Flash/attn" + ] + }, + { + "type": "dsv4_attn", + "args": { + "hidden": 5120, + "heads": 64, + "head_dim": 512, + "rope_dim": 64, + "q_lora_rank": 1280, + "window": 128, + "o_groups": 8, + "o_lora_rank": 1024, + "compress_ratio": 2, + "source": false, + "indexer": "reuse", + "compress_rope_theta": 160000, + "rope_scaling": { + "type": "yarn", + "factor": 16, + "original_max": 65536, + "beta_fast": 32, + "beta_slow": 1 + }, + "topk": 512, + "index_heads": 32, + "index_dim": 128, + "proj": { + "wkv": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + }, + "wo_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + }, + "wq_a": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + }, + "wq_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + } + }, + "batch": { + "groups": [ + { + "count": 1, + "q": 8192, + "ctx": 131072 + } + ] + } + }, + "sources": [ + "deepseek-ai/DeepSeek-V4.1-Flash/attn" + ] + }, + { + "type": "dsv4_attn", + "args": { + "hidden": 5120, + "heads": 64, + "head_dim": 512, + "rope_dim": 64, + "q_lora_rank": 1280, + "window": 128, + "o_groups": 8, + "o_lora_rank": 1024, + "compress_ratio": 2, + "source": false, + "indexer": "reuse", + "compress_rope_theta": 160000, + "rope_scaling": { + "type": "yarn", + "factor": 16, + "original_max": 65536, + "beta_fast": 32, + "beta_slow": 1 + }, + "topk": 512, + "index_heads": 32, + "index_dim": 128, + "proj": { + "wkv": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + }, + "wo_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + }, + "wq_a": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + }, + "wq_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + } + }, + "batch": { + "groups": [ + { + "count": 1, + "q": 8192, + "ctx": 8192 + }, + { + "count": 64, + "q": 1, + "ctx": 8192 + } + ] + } + }, + "sources": [ + "deepseek-ai/DeepSeek-V4.1-Flash/attn" + ] + }, + { + "type": "dsv4_attn", + "args": { + "hidden": 5120, + "heads": 64, + "head_dim": 512, + "rope_dim": 64, + "q_lora_rank": 1280, + "window": 128, + "o_groups": 8, + "o_lora_rank": 1024, + "compress_ratio": 1, + "indexer": "own", + "compress_rope_theta": 160000, + "rope_scaling": { + "type": "yarn", + "factor": 16, + "original_max": 65536, + "beta_fast": 32, + "beta_slow": 1 + }, + "topk": 512, + "index_heads": 32, + "index_dim": 128, + "proj": { + "indexer.wq_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + }, + "wkv": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + }, + "wo_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + }, + "wq_a": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + }, + "wq_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + } + }, + "batch": { + "groups": [ + { + "count": 1, + "q": 1, + "ctx": 1024 + } + ] + } + }, + "sources": [ + "deepseek-ai/DeepSeek-V4.1-Flash/attn" + ] + }, + { + "type": "dsv4_attn", + "args": { + "hidden": 5120, + "heads": 64, + "head_dim": 512, + "rope_dim": 64, + "q_lora_rank": 1280, + "window": 128, + "o_groups": 8, + "o_lora_rank": 1024, + "compress_ratio": 1, + "indexer": "own", + "compress_rope_theta": 160000, + "rope_scaling": { + "type": "yarn", + "factor": 16, + "original_max": 65536, + "beta_fast": 32, + "beta_slow": 1 + }, + "topk": 512, + "index_heads": 32, + "index_dim": 128, + "proj": { + "indexer.wq_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + }, + "wkv": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + }, + "wo_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + }, + "wq_a": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + }, + "wq_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + } + }, + "batch": { + "groups": [ + { + "count": 1, + "q": 1, + "ctx": 4096 + } + ] + } + }, + "sources": [ + "deepseek-ai/DeepSeek-V4.1-Flash/attn" + ] + }, + { + "type": "dsv4_attn", + "args": { + "hidden": 5120, + "heads": 64, + "head_dim": 512, + "rope_dim": 64, + "q_lora_rank": 1280, + "window": 128, + "o_groups": 8, + "o_lora_rank": 1024, + "compress_ratio": 1, + "indexer": "own", + "compress_rope_theta": 160000, + "rope_scaling": { + "type": "yarn", + "factor": 16, + "original_max": 65536, + "beta_fast": 32, + "beta_slow": 1 + }, + "topk": 512, + "index_heads": 32, + "index_dim": 128, + "proj": { + "indexer.wq_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + }, + "wkv": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + }, + "wo_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + }, + "wq_a": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + }, + "wq_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + } + }, + "batch": { + "groups": [ + { + "count": 1, + "q": 1, + "ctx": 16384 + } + ] + } + }, + "sources": [ + "deepseek-ai/DeepSeek-V4.1-Flash/attn" + ] + }, + { + "type": "dsv4_attn", + "args": { + "hidden": 5120, + "heads": 64, + "head_dim": 512, + "rope_dim": 64, + "q_lora_rank": 1280, + "window": 128, + "o_groups": 8, + "o_lora_rank": 1024, + "compress_ratio": 1, + "indexer": "own", + "compress_rope_theta": 160000, + "rope_scaling": { + "type": "yarn", + "factor": 16, + "original_max": 65536, + "beta_fast": 32, + "beta_slow": 1 + }, + "topk": 512, + "index_heads": 32, + "index_dim": 128, + "proj": { + "indexer.wq_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + }, + "wkv": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + }, + "wo_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + }, + "wq_a": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + }, + "wq_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + } + }, + "batch": { + "groups": [ + { + "count": 1, + "q": 1, + "ctx": 65536 + } + ] + } + }, + "sources": [ + "deepseek-ai/DeepSeek-V4.1-Flash/attn" + ] + }, + { + "type": "dsv4_attn", + "args": { + "hidden": 5120, + "heads": 64, + "head_dim": 512, + "rope_dim": 64, + "q_lora_rank": 1280, + "window": 128, + "o_groups": 8, + "o_lora_rank": 1024, + "compress_ratio": 1, + "indexer": "own", + "compress_rope_theta": 160000, + "rope_scaling": { + "type": "yarn", + "factor": 16, + "original_max": 65536, + "beta_fast": 32, + "beta_slow": 1 + }, + "topk": 512, + "index_heads": 32, + "index_dim": 128, + "proj": { + "indexer.wq_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + }, + "wkv": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + }, + "wo_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + }, + "wq_a": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + }, + "wq_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + } + }, + "batch": { + "groups": [ + { + "count": 1, + "q": 1, + "ctx": 131072 + } + ] + } + }, + "sources": [ + "deepseek-ai/DeepSeek-V4.1-Flash/attn" + ] + }, + { + "type": "dsv4_attn", + "args": { + "hidden": 5120, + "heads": 64, + "head_dim": 512, + "rope_dim": 64, + "q_lora_rank": 1280, + "window": 128, + "o_groups": 8, + "o_lora_rank": 1024, + "compress_ratio": 1, + "indexer": "own", + "compress_rope_theta": 160000, + "rope_scaling": { + "type": "yarn", + "factor": 16, + "original_max": 65536, + "beta_fast": 32, + "beta_slow": 1 + }, + "topk": 512, + "index_heads": 32, + "index_dim": 128, + "proj": { + "indexer.wq_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + }, + "wkv": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + }, + "wo_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + }, + "wq_a": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + }, + "wq_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + } + }, + "batch": { + "groups": [ + { + "count": 4, + "q": 1, + "ctx": 1024 + } + ] + } + }, + "sources": [ + "deepseek-ai/DeepSeek-V4.1-Flash/attn" + ] + }, + { + "type": "dsv4_attn", + "args": { + "hidden": 5120, + "heads": 64, + "head_dim": 512, + "rope_dim": 64, + "q_lora_rank": 1280, + "window": 128, + "o_groups": 8, + "o_lora_rank": 1024, + "compress_ratio": 1, + "indexer": "own", + "compress_rope_theta": 160000, + "rope_scaling": { + "type": "yarn", + "factor": 16, + "original_max": 65536, + "beta_fast": 32, + "beta_slow": 1 + }, + "topk": 512, + "index_heads": 32, + "index_dim": 128, + "proj": { + "indexer.wq_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + }, + "wkv": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + }, + "wo_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + }, + "wq_a": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + }, + "wq_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + } + }, + "batch": { + "groups": [ + { + "count": 4, + "q": 1, + "ctx": 4096 + } + ] + } + }, + "sources": [ + "deepseek-ai/DeepSeek-V4.1-Flash/attn" + ] + }, + { + "type": "dsv4_attn", + "args": { + "hidden": 5120, + "heads": 64, + "head_dim": 512, + "rope_dim": 64, + "q_lora_rank": 1280, + "window": 128, + "o_groups": 8, + "o_lora_rank": 1024, + "compress_ratio": 1, + "indexer": "own", + "compress_rope_theta": 160000, + "rope_scaling": { + "type": "yarn", + "factor": 16, + "original_max": 65536, + "beta_fast": 32, + "beta_slow": 1 + }, + "topk": 512, + "index_heads": 32, + "index_dim": 128, + "proj": { + "indexer.wq_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + }, + "wkv": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + }, + "wo_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + }, + "wq_a": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + }, + "wq_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + } + }, + "batch": { + "groups": [ + { + "count": 4, + "q": 1, + "ctx": 16384 + } + ] + } + }, + "sources": [ + "deepseek-ai/DeepSeek-V4.1-Flash/attn" + ] + }, + { + "type": "dsv4_attn", + "args": { + "hidden": 5120, + "heads": 64, + "head_dim": 512, + "rope_dim": 64, + "q_lora_rank": 1280, + "window": 128, + "o_groups": 8, + "o_lora_rank": 1024, + "compress_ratio": 1, + "indexer": "own", + "compress_rope_theta": 160000, + "rope_scaling": { + "type": "yarn", + "factor": 16, + "original_max": 65536, + "beta_fast": 32, + "beta_slow": 1 + }, + "topk": 512, + "index_heads": 32, + "index_dim": 128, + "proj": { + "indexer.wq_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + }, + "wkv": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + }, + "wo_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + }, + "wq_a": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + }, + "wq_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + } + }, + "batch": { + "groups": [ + { + "count": 4, + "q": 1, + "ctx": 65536 + } + ] + } + }, + "sources": [ + "deepseek-ai/DeepSeek-V4.1-Flash/attn" + ] + }, + { + "type": "dsv4_attn", + "args": { + "hidden": 5120, + "heads": 64, + "head_dim": 512, + "rope_dim": 64, + "q_lora_rank": 1280, + "window": 128, + "o_groups": 8, + "o_lora_rank": 1024, + "compress_ratio": 1, + "indexer": "own", + "compress_rope_theta": 160000, + "rope_scaling": { + "type": "yarn", + "factor": 16, + "original_max": 65536, + "beta_fast": 32, + "beta_slow": 1 + }, + "topk": 512, + "index_heads": 32, + "index_dim": 128, + "proj": { + "indexer.wq_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + }, + "wkv": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + }, + "wo_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + }, + "wq_a": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + }, + "wq_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + } + }, + "batch": { + "groups": [ + { + "count": 4, + "q": 1, + "ctx": 131072 + } + ] + } + }, + "sources": [ + "deepseek-ai/DeepSeek-V4.1-Flash/attn" + ] + }, + { + "type": "dsv4_attn", + "args": { + "hidden": 5120, + "heads": 64, + "head_dim": 512, + "rope_dim": 64, + "q_lora_rank": 1280, + "window": 128, + "o_groups": 8, + "o_lora_rank": 1024, + "compress_ratio": 1, + "indexer": "own", + "compress_rope_theta": 160000, + "rope_scaling": { + "type": "yarn", + "factor": 16, + "original_max": 65536, + "beta_fast": 32, + "beta_slow": 1 + }, + "topk": 512, + "index_heads": 32, + "index_dim": 128, + "proj": { + "indexer.wq_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + }, + "wkv": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + }, + "wo_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + }, + "wq_a": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + }, + "wq_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + } + }, + "batch": { + "groups": [ + { + "count": 16, + "q": 1, + "ctx": 1024 + } + ] + } + }, + "sources": [ + "deepseek-ai/DeepSeek-V4.1-Flash/attn" + ] + }, + { + "type": "dsv4_attn", + "args": { + "hidden": 5120, + "heads": 64, + "head_dim": 512, + "rope_dim": 64, + "q_lora_rank": 1280, + "window": 128, + "o_groups": 8, + "o_lora_rank": 1024, + "compress_ratio": 1, + "indexer": "own", + "compress_rope_theta": 160000, + "rope_scaling": { + "type": "yarn", + "factor": 16, + "original_max": 65536, + "beta_fast": 32, + "beta_slow": 1 + }, + "topk": 512, + "index_heads": 32, + "index_dim": 128, + "proj": { + "indexer.wq_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + }, + "wkv": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + }, + "wo_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + }, + "wq_a": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + }, + "wq_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + } + }, + "batch": { + "groups": [ + { + "count": 16, + "q": 1, + "ctx": 4096 + } + ] + } + }, + "sources": [ + "deepseek-ai/DeepSeek-V4.1-Flash/attn" + ] + }, + { + "type": "dsv4_attn", + "args": { + "hidden": 5120, + "heads": 64, + "head_dim": 512, + "rope_dim": 64, + "q_lora_rank": 1280, + "window": 128, + "o_groups": 8, + "o_lora_rank": 1024, + "compress_ratio": 1, + "indexer": "own", + "compress_rope_theta": 160000, + "rope_scaling": { + "type": "yarn", + "factor": 16, + "original_max": 65536, + "beta_fast": 32, + "beta_slow": 1 + }, + "topk": 512, + "index_heads": 32, + "index_dim": 128, + "proj": { + "indexer.wq_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + }, + "wkv": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + }, + "wo_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + }, + "wq_a": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + }, + "wq_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + } + }, + "batch": { + "groups": [ + { + "count": 16, + "q": 1, + "ctx": 16384 + } + ] + } + }, + "sources": [ + "deepseek-ai/DeepSeek-V4.1-Flash/attn" + ] + }, + { + "type": "dsv4_attn", + "args": { + "hidden": 5120, + "heads": 64, + "head_dim": 512, + "rope_dim": 64, + "q_lora_rank": 1280, + "window": 128, + "o_groups": 8, + "o_lora_rank": 1024, + "compress_ratio": 1, + "indexer": "own", + "compress_rope_theta": 160000, + "rope_scaling": { + "type": "yarn", + "factor": 16, + "original_max": 65536, + "beta_fast": 32, + "beta_slow": 1 + }, + "topk": 512, + "index_heads": 32, + "index_dim": 128, + "proj": { + "indexer.wq_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + }, + "wkv": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + }, + "wo_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + }, + "wq_a": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + }, + "wq_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + } + }, + "batch": { + "groups": [ + { + "count": 16, + "q": 1, + "ctx": 65536 + } + ] + } + }, + "sources": [ + "deepseek-ai/DeepSeek-V4.1-Flash/attn" + ] + }, + { + "type": "dsv4_attn", + "args": { + "hidden": 5120, + "heads": 64, + "head_dim": 512, + "rope_dim": 64, + "q_lora_rank": 1280, + "window": 128, + "o_groups": 8, + "o_lora_rank": 1024, + "compress_ratio": 1, + "indexer": "own", + "compress_rope_theta": 160000, + "rope_scaling": { + "type": "yarn", + "factor": 16, + "original_max": 65536, + "beta_fast": 32, + "beta_slow": 1 + }, + "topk": 512, + "index_heads": 32, + "index_dim": 128, + "proj": { + "indexer.wq_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + }, + "wkv": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + }, + "wo_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + }, + "wq_a": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + }, + "wq_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + } + }, + "batch": { + "groups": [ + { + "count": 16, + "q": 1, + "ctx": 131072 + } + ] + } + }, + "sources": [ + "deepseek-ai/DeepSeek-V4.1-Flash/attn" + ] + }, + { + "type": "dsv4_attn", + "args": { + "hidden": 5120, + "heads": 64, + "head_dim": 512, + "rope_dim": 64, + "q_lora_rank": 1280, + "window": 128, + "o_groups": 8, + "o_lora_rank": 1024, + "compress_ratio": 1, + "indexer": "own", + "compress_rope_theta": 160000, + "rope_scaling": { + "type": "yarn", + "factor": 16, + "original_max": 65536, + "beta_fast": 32, + "beta_slow": 1 + }, + "topk": 512, + "index_heads": 32, + "index_dim": 128, + "proj": { + "indexer.wq_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + }, + "wkv": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + }, + "wo_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + }, + "wq_a": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + }, + "wq_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + } + }, + "batch": { + "groups": [ + { + "count": 64, + "q": 1, + "ctx": 1024 + } + ] + } + }, + "sources": [ + "deepseek-ai/DeepSeek-V4.1-Flash/attn" + ] + }, + { + "type": "dsv4_attn", + "args": { + "hidden": 5120, + "heads": 64, + "head_dim": 512, + "rope_dim": 64, + "q_lora_rank": 1280, + "window": 128, + "o_groups": 8, + "o_lora_rank": 1024, + "compress_ratio": 1, + "indexer": "own", + "compress_rope_theta": 160000, + "rope_scaling": { + "type": "yarn", + "factor": 16, + "original_max": 65536, + "beta_fast": 32, + "beta_slow": 1 + }, + "topk": 512, + "index_heads": 32, + "index_dim": 128, + "proj": { + "indexer.wq_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + }, + "wkv": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + }, + "wo_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + }, + "wq_a": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + }, + "wq_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + } + }, + "batch": { + "groups": [ + { + "count": 64, + "q": 1, + "ctx": 4096 + } + ] + } + }, + "sources": [ + "deepseek-ai/DeepSeek-V4.1-Flash/attn" + ] + }, + { + "type": "dsv4_attn", + "args": { + "hidden": 5120, + "heads": 64, + "head_dim": 512, + "rope_dim": 64, + "q_lora_rank": 1280, + "window": 128, + "o_groups": 8, + "o_lora_rank": 1024, + "compress_ratio": 1, + "indexer": "own", + "compress_rope_theta": 160000, + "rope_scaling": { + "type": "yarn", + "factor": 16, + "original_max": 65536, + "beta_fast": 32, + "beta_slow": 1 + }, + "topk": 512, + "index_heads": 32, + "index_dim": 128, + "proj": { + "indexer.wq_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + }, + "wkv": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + }, + "wo_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + }, + "wq_a": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + }, + "wq_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + } + }, + "batch": { + "groups": [ + { + "count": 64, + "q": 1, + "ctx": 16384 + } + ] + } + }, + "sources": [ + "deepseek-ai/DeepSeek-V4.1-Flash/attn" + ] + }, + { + "type": "dsv4_attn", + "args": { + "hidden": 5120, + "heads": 64, + "head_dim": 512, + "rope_dim": 64, + "q_lora_rank": 1280, + "window": 128, + "o_groups": 8, + "o_lora_rank": 1024, + "compress_ratio": 1, + "indexer": "own", + "compress_rope_theta": 160000, + "rope_scaling": { + "type": "yarn", + "factor": 16, + "original_max": 65536, + "beta_fast": 32, + "beta_slow": 1 + }, + "topk": 512, + "index_heads": 32, + "index_dim": 128, + "proj": { + "indexer.wq_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + }, + "wkv": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + }, + "wo_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + }, + "wq_a": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + }, + "wq_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + } + }, + "batch": { + "groups": [ + { + "count": 64, + "q": 1, + "ctx": 65536 + } + ] + } + }, + "sources": [ + "deepseek-ai/DeepSeek-V4.1-Flash/attn" + ] + }, + { + "type": "dsv4_attn", + "args": { + "hidden": 5120, + "heads": 64, + "head_dim": 512, + "rope_dim": 64, + "q_lora_rank": 1280, + "window": 128, + "o_groups": 8, + "o_lora_rank": 1024, + "compress_ratio": 1, + "indexer": "own", + "compress_rope_theta": 160000, + "rope_scaling": { + "type": "yarn", + "factor": 16, + "original_max": 65536, + "beta_fast": 32, + "beta_slow": 1 + }, + "topk": 512, + "index_heads": 32, + "index_dim": 128, + "proj": { + "indexer.wq_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + }, + "wkv": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + }, + "wo_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + }, + "wq_a": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + }, + "wq_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + } + }, + "batch": { + "groups": [ + { + "count": 64, + "q": 1, + "ctx": 131072 + } + ] + } + }, + "sources": [ + "deepseek-ai/DeepSeek-V4.1-Flash/attn" + ] + }, + { + "type": "dsv4_attn", + "args": { + "hidden": 5120, + "heads": 64, + "head_dim": 512, + "rope_dim": 64, + "q_lora_rank": 1280, + "window": 128, + "o_groups": 8, + "o_lora_rank": 1024, + "compress_ratio": 1, + "indexer": "own", + "compress_rope_theta": 160000, + "rope_scaling": { + "type": "yarn", + "factor": 16, + "original_max": 65536, + "beta_fast": 32, + "beta_slow": 1 + }, + "topk": 512, + "index_heads": 32, + "index_dim": 128, + "proj": { + "indexer.wq_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + }, + "wkv": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + }, + "wo_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + }, + "wq_a": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + }, + "wq_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + } + }, + "batch": { + "groups": [ + { + "count": 256, + "q": 1, + "ctx": 1024 + } + ] + } + }, + "sources": [ + "deepseek-ai/DeepSeek-V4.1-Flash/attn" + ] + }, + { + "type": "dsv4_attn", + "args": { + "hidden": 5120, + "heads": 64, + "head_dim": 512, + "rope_dim": 64, + "q_lora_rank": 1280, + "window": 128, + "o_groups": 8, + "o_lora_rank": 1024, + "compress_ratio": 1, + "indexer": "own", + "compress_rope_theta": 160000, + "rope_scaling": { + "type": "yarn", + "factor": 16, + "original_max": 65536, + "beta_fast": 32, + "beta_slow": 1 + }, + "topk": 512, + "index_heads": 32, + "index_dim": 128, + "proj": { + "indexer.wq_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + }, + "wkv": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + }, + "wo_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + }, + "wq_a": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + }, + "wq_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + } + }, + "batch": { + "groups": [ + { + "count": 256, + "q": 1, + "ctx": 4096 + } + ] + } + }, + "sources": [ + "deepseek-ai/DeepSeek-V4.1-Flash/attn" + ] + }, + { + "type": "dsv4_attn", + "args": { + "hidden": 5120, + "heads": 64, + "head_dim": 512, + "rope_dim": 64, + "q_lora_rank": 1280, + "window": 128, + "o_groups": 8, + "o_lora_rank": 1024, + "compress_ratio": 1, + "indexer": "own", + "compress_rope_theta": 160000, + "rope_scaling": { + "type": "yarn", + "factor": 16, + "original_max": 65536, + "beta_fast": 32, + "beta_slow": 1 + }, + "topk": 512, + "index_heads": 32, + "index_dim": 128, + "proj": { + "indexer.wq_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + }, + "wkv": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + }, + "wo_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + }, + "wq_a": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + }, + "wq_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + } + }, + "batch": { + "groups": [ + { + "count": 256, + "q": 1, + "ctx": 16384 + } + ] + } + }, + "sources": [ + "deepseek-ai/DeepSeek-V4.1-Flash/attn" + ] + }, + { + "type": "dsv4_attn", + "args": { + "hidden": 5120, + "heads": 64, + "head_dim": 512, + "rope_dim": 64, + "q_lora_rank": 1280, + "window": 128, + "o_groups": 8, + "o_lora_rank": 1024, + "compress_ratio": 1, + "indexer": "own", + "compress_rope_theta": 160000, + "rope_scaling": { + "type": "yarn", + "factor": 16, + "original_max": 65536, + "beta_fast": 32, + "beta_slow": 1 + }, + "topk": 512, + "index_heads": 32, + "index_dim": 128, + "proj": { + "indexer.wq_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + }, + "wkv": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + }, + "wo_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + }, + "wq_a": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + }, + "wq_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + } + }, + "batch": { + "groups": [ + { + "count": 256, + "q": 1, + "ctx": 65536 + } + ] + } + }, + "sources": [ + "deepseek-ai/DeepSeek-V4.1-Flash/attn" + ] + }, + { + "type": "dsv4_attn", + "args": { + "hidden": 5120, + "heads": 64, + "head_dim": 512, + "rope_dim": 64, + "q_lora_rank": 1280, + "window": 128, + "o_groups": 8, + "o_lora_rank": 1024, + "compress_ratio": 1, + "indexer": "own", + "compress_rope_theta": 160000, + "rope_scaling": { + "type": "yarn", + "factor": 16, + "original_max": 65536, + "beta_fast": 32, + "beta_slow": 1 + }, + "topk": 512, + "index_heads": 32, + "index_dim": 128, + "proj": { + "indexer.wq_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + }, + "wkv": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + }, + "wo_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + }, + "wq_a": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + }, + "wq_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + } + }, + "batch": { + "groups": [ + { + "count": 256, + "q": 1, + "ctx": 131072 + } + ] + } + }, + "sources": [ + "deepseek-ai/DeepSeek-V4.1-Flash/attn" + ] + }, + { + "type": "dsv4_attn", + "args": { + "hidden": 5120, + "heads": 64, + "head_dim": 512, + "rope_dim": 64, + "q_lora_rank": 1280, + "window": 128, + "o_groups": 8, + "o_lora_rank": 1024, + "compress_ratio": 1, + "indexer": "own", + "compress_rope_theta": 160000, + "rope_scaling": { + "type": "yarn", + "factor": 16, + "original_max": 65536, + "beta_fast": 32, + "beta_slow": 1 + }, + "topk": 512, + "index_heads": 32, + "index_dim": 128, + "proj": { + "indexer.wq_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + }, + "wkv": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + }, + "wo_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + }, + "wq_a": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + }, + "wq_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + } + }, + "batch": { + "groups": [ + { + "count": 1, + "q": 1024, + "ctx": 0 + } + ] + } + }, + "sources": [ + "deepseek-ai/DeepSeek-V4.1-Flash/attn" + ] + }, + { + "type": "dsv4_attn", + "args": { + "hidden": 5120, + "heads": 64, + "head_dim": 512, + "rope_dim": 64, + "q_lora_rank": 1280, + "window": 128, + "o_groups": 8, + "o_lora_rank": 1024, + "compress_ratio": 1, + "indexer": "own", + "compress_rope_theta": 160000, + "rope_scaling": { + "type": "yarn", + "factor": 16, + "original_max": 65536, + "beta_fast": 32, + "beta_slow": 1 + }, + "topk": 512, + "index_heads": 32, + "index_dim": 128, + "proj": { + "indexer.wq_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + }, + "wkv": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + }, + "wo_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + }, + "wq_a": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + }, + "wq_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + } + }, + "batch": { + "groups": [ + { + "count": 1, + "q": 4096, + "ctx": 0 + } + ] + } + }, + "sources": [ + "deepseek-ai/DeepSeek-V4.1-Flash/attn" + ] + }, + { + "type": "dsv4_attn", + "args": { + "hidden": 5120, + "heads": 64, + "head_dim": 512, + "rope_dim": 64, + "q_lora_rank": 1280, + "window": 128, + "o_groups": 8, + "o_lora_rank": 1024, + "compress_ratio": 1, + "indexer": "own", + "compress_rope_theta": 160000, + "rope_scaling": { + "type": "yarn", + "factor": 16, + "original_max": 65536, + "beta_fast": 32, + "beta_slow": 1 + }, + "topk": 512, + "index_heads": 32, + "index_dim": 128, + "proj": { + "indexer.wq_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + }, + "wkv": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + }, + "wo_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + }, + "wq_a": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + }, + "wq_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + } + }, + "batch": { + "groups": [ + { + "count": 1, + "q": 16384, + "ctx": 0 + } + ] + } + }, + "sources": [ + "deepseek-ai/DeepSeek-V4.1-Flash/attn" + ] + }, + { + "type": "dsv4_attn", + "args": { + "hidden": 5120, + "heads": 64, + "head_dim": 512, + "rope_dim": 64, + "q_lora_rank": 1280, + "window": 128, + "o_groups": 8, + "o_lora_rank": 1024, + "compress_ratio": 1, + "indexer": "own", + "compress_rope_theta": 160000, + "rope_scaling": { + "type": "yarn", + "factor": 16, + "original_max": 65536, + "beta_fast": 32, + "beta_slow": 1 + }, + "topk": 512, + "index_heads": 32, + "index_dim": 128, + "proj": { + "indexer.wq_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + }, + "wkv": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + }, + "wo_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + }, + "wq_a": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + }, + "wq_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + } + }, + "batch": { + "groups": [ + { + "count": 1, + "q": 32768, + "ctx": 0 + } + ] + } + }, + "sources": [ + "deepseek-ai/DeepSeek-V4.1-Flash/attn" + ] + }, + { + "type": "dsv4_attn", + "args": { + "hidden": 5120, + "heads": 64, + "head_dim": 512, + "rope_dim": 64, + "q_lora_rank": 1280, + "window": 128, + "o_groups": 8, + "o_lora_rank": 1024, + "compress_ratio": 1, + "indexer": "own", + "compress_rope_theta": 160000, + "rope_scaling": { + "type": "yarn", + "factor": 16, + "original_max": 65536, + "beta_fast": 32, + "beta_slow": 1 + }, + "topk": 512, + "index_heads": 32, + "index_dim": 128, + "proj": { + "indexer.wq_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + }, + "wkv": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + }, + "wo_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + }, + "wq_a": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + }, + "wq_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + } + }, + "batch": { + "groups": [ + { + "count": 1, + "q": 8192, + "ctx": 8192 + } + ] + } + }, + "sources": [ + "deepseek-ai/DeepSeek-V4.1-Flash/attn" + ] + }, + { + "type": "dsv4_attn", + "args": { + "hidden": 5120, + "heads": 64, + "head_dim": 512, + "rope_dim": 64, + "q_lora_rank": 1280, + "window": 128, + "o_groups": 8, + "o_lora_rank": 1024, + "compress_ratio": 1, + "indexer": "own", + "compress_rope_theta": 160000, + "rope_scaling": { + "type": "yarn", + "factor": 16, + "original_max": 65536, + "beta_fast": 32, + "beta_slow": 1 + }, + "topk": 512, + "index_heads": 32, + "index_dim": 128, + "proj": { + "indexer.wq_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + }, + "wkv": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + }, + "wo_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + }, + "wq_a": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + }, + "wq_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + } + }, + "batch": { + "groups": [ + { + "count": 1, + "q": 8192, + "ctx": 32768 + } + ] + } + }, + "sources": [ + "deepseek-ai/DeepSeek-V4.1-Flash/attn" + ] + }, + { + "type": "dsv4_attn", + "args": { + "hidden": 5120, + "heads": 64, + "head_dim": 512, + "rope_dim": 64, + "q_lora_rank": 1280, + "window": 128, + "o_groups": 8, + "o_lora_rank": 1024, + "compress_ratio": 1, + "indexer": "own", + "compress_rope_theta": 160000, + "rope_scaling": { + "type": "yarn", + "factor": 16, + "original_max": 65536, + "beta_fast": 32, + "beta_slow": 1 + }, + "topk": 512, + "index_heads": 32, + "index_dim": 128, + "proj": { + "indexer.wq_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + }, + "wkv": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + }, + "wo_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + }, + "wq_a": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + }, + "wq_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + } + }, + "batch": { + "groups": [ + { + "count": 1, + "q": 8192, + "ctx": 131072 + } + ] + } + }, + "sources": [ + "deepseek-ai/DeepSeek-V4.1-Flash/attn" + ] + }, + { + "type": "dsv4_attn", + "args": { + "hidden": 5120, + "heads": 64, + "head_dim": 512, + "rope_dim": 64, + "q_lora_rank": 1280, + "window": 128, + "o_groups": 8, + "o_lora_rank": 1024, + "compress_ratio": 1, + "indexer": "own", + "compress_rope_theta": 160000, + "rope_scaling": { + "type": "yarn", + "factor": 16, + "original_max": 65536, + "beta_fast": 32, + "beta_slow": 1 + }, + "topk": 512, + "index_heads": 32, + "index_dim": 128, + "proj": { + "indexer.wq_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + }, + "wkv": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + }, + "wo_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + }, + "wq_a": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + }, + "wq_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + } + }, + "batch": { + "groups": [ + { + "count": 1, + "q": 8192, + "ctx": 8192 + }, + { + "count": 64, + "q": 1, + "ctx": 8192 + } + ] + } + }, + "sources": [ + "deepseek-ai/DeepSeek-V4.1-Flash/attn" + ] + }, + { + "type": "dsv4_attn", + "args": { + "hidden": 5120, + "heads": 64, + "head_dim": 512, + "rope_dim": 64, + "q_lora_rank": 1280, + "window": 128, + "o_groups": 8, + "o_lora_rank": 1024, + "compress_ratio": 1, + "source": false, + "indexer": "reuse", + "compress_rope_theta": 160000, + "rope_scaling": { + "type": "yarn", + "factor": 16, + "original_max": 65536, + "beta_fast": 32, + "beta_slow": 1 + }, + "topk": 512, + "index_heads": 32, + "index_dim": 128, + "proj": { + "wkv": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + }, + "wo_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + }, + "wq_a": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + }, + "wq_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + } + }, + "batch": { + "groups": [ + { + "count": 1, + "q": 1, + "ctx": 1024 + } + ] + } + }, + "sources": [ + "deepseek-ai/DeepSeek-V4.1-Flash/attn" + ] + }, + { + "type": "dsv4_attn", + "args": { + "hidden": 5120, + "heads": 64, + "head_dim": 512, + "rope_dim": 64, + "q_lora_rank": 1280, + "window": 128, + "o_groups": 8, + "o_lora_rank": 1024, + "compress_ratio": 1, + "source": false, + "indexer": "reuse", + "compress_rope_theta": 160000, + "rope_scaling": { + "type": "yarn", + "factor": 16, + "original_max": 65536, + "beta_fast": 32, + "beta_slow": 1 + }, + "topk": 512, + "index_heads": 32, + "index_dim": 128, + "proj": { + "wkv": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + }, + "wo_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + }, + "wq_a": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + }, + "wq_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + } + }, + "batch": { + "groups": [ + { + "count": 1, + "q": 1, + "ctx": 4096 + } + ] + } + }, + "sources": [ + "deepseek-ai/DeepSeek-V4.1-Flash/attn" + ] + }, + { + "type": "dsv4_attn", + "args": { + "hidden": 5120, + "heads": 64, + "head_dim": 512, + "rope_dim": 64, + "q_lora_rank": 1280, + "window": 128, + "o_groups": 8, + "o_lora_rank": 1024, + "compress_ratio": 1, + "source": false, + "indexer": "reuse", + "compress_rope_theta": 160000, + "rope_scaling": { + "type": "yarn", + "factor": 16, + "original_max": 65536, + "beta_fast": 32, + "beta_slow": 1 + }, + "topk": 512, + "index_heads": 32, + "index_dim": 128, + "proj": { + "wkv": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + }, + "wo_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + }, + "wq_a": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + }, + "wq_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + } + }, + "batch": { + "groups": [ + { + "count": 1, + "q": 1, + "ctx": 16384 + } + ] + } + }, + "sources": [ + "deepseek-ai/DeepSeek-V4.1-Flash/attn" + ] + }, + { + "type": "dsv4_attn", + "args": { + "hidden": 5120, + "heads": 64, + "head_dim": 512, + "rope_dim": 64, + "q_lora_rank": 1280, + "window": 128, + "o_groups": 8, + "o_lora_rank": 1024, + "compress_ratio": 1, + "source": false, + "indexer": "reuse", + "compress_rope_theta": 160000, + "rope_scaling": { + "type": "yarn", + "factor": 16, + "original_max": 65536, + "beta_fast": 32, + "beta_slow": 1 + }, + "topk": 512, + "index_heads": 32, + "index_dim": 128, + "proj": { + "wkv": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + }, + "wo_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + }, + "wq_a": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + }, + "wq_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + } + }, + "batch": { + "groups": [ + { + "count": 1, + "q": 1, + "ctx": 65536 + } + ] + } + }, + "sources": [ + "deepseek-ai/DeepSeek-V4.1-Flash/attn" + ] + }, + { + "type": "dsv4_attn", + "args": { + "hidden": 5120, + "heads": 64, + "head_dim": 512, + "rope_dim": 64, + "q_lora_rank": 1280, + "window": 128, + "o_groups": 8, + "o_lora_rank": 1024, + "compress_ratio": 1, + "source": false, + "indexer": "reuse", + "compress_rope_theta": 160000, + "rope_scaling": { + "type": "yarn", + "factor": 16, + "original_max": 65536, + "beta_fast": 32, + "beta_slow": 1 + }, + "topk": 512, + "index_heads": 32, + "index_dim": 128, + "proj": { + "wkv": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + }, + "wo_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + }, + "wq_a": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + }, + "wq_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + } + }, + "batch": { + "groups": [ + { + "count": 1, + "q": 1, + "ctx": 131072 + } + ] + } + }, + "sources": [ + "deepseek-ai/DeepSeek-V4.1-Flash/attn" + ] + }, + { + "type": "dsv4_attn", + "args": { + "hidden": 5120, + "heads": 64, + "head_dim": 512, + "rope_dim": 64, + "q_lora_rank": 1280, + "window": 128, + "o_groups": 8, + "o_lora_rank": 1024, + "compress_ratio": 1, + "source": false, + "indexer": "reuse", + "compress_rope_theta": 160000, + "rope_scaling": { + "type": "yarn", + "factor": 16, + "original_max": 65536, + "beta_fast": 32, + "beta_slow": 1 + }, + "topk": 512, + "index_heads": 32, + "index_dim": 128, + "proj": { + "wkv": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + }, + "wo_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + }, + "wq_a": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + }, + "wq_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + } + }, + "batch": { + "groups": [ + { + "count": 4, + "q": 1, + "ctx": 1024 + } + ] + } + }, + "sources": [ + "deepseek-ai/DeepSeek-V4.1-Flash/attn" + ] + }, + { + "type": "dsv4_attn", + "args": { + "hidden": 5120, + "heads": 64, + "head_dim": 512, + "rope_dim": 64, + "q_lora_rank": 1280, + "window": 128, + "o_groups": 8, + "o_lora_rank": 1024, + "compress_ratio": 1, + "source": false, + "indexer": "reuse", + "compress_rope_theta": 160000, + "rope_scaling": { + "type": "yarn", + "factor": 16, + "original_max": 65536, + "beta_fast": 32, + "beta_slow": 1 + }, + "topk": 512, + "index_heads": 32, + "index_dim": 128, + "proj": { + "wkv": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + }, + "wo_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + }, + "wq_a": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + }, + "wq_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + } + }, + "batch": { + "groups": [ + { + "count": 4, + "q": 1, + "ctx": 4096 + } + ] + } + }, + "sources": [ + "deepseek-ai/DeepSeek-V4.1-Flash/attn" + ] + }, + { + "type": "dsv4_attn", + "args": { + "hidden": 5120, + "heads": 64, + "head_dim": 512, + "rope_dim": 64, + "q_lora_rank": 1280, + "window": 128, + "o_groups": 8, + "o_lora_rank": 1024, + "compress_ratio": 1, + "source": false, + "indexer": "reuse", + "compress_rope_theta": 160000, + "rope_scaling": { + "type": "yarn", + "factor": 16, + "original_max": 65536, + "beta_fast": 32, + "beta_slow": 1 + }, + "topk": 512, + "index_heads": 32, + "index_dim": 128, + "proj": { + "wkv": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + }, + "wo_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + }, + "wq_a": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + }, + "wq_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + } + }, + "batch": { + "groups": [ + { + "count": 4, + "q": 1, + "ctx": 16384 + } + ] + } + }, + "sources": [ + "deepseek-ai/DeepSeek-V4.1-Flash/attn" + ] + }, + { + "type": "dsv4_attn", + "args": { + "hidden": 5120, + "heads": 64, + "head_dim": 512, + "rope_dim": 64, + "q_lora_rank": 1280, + "window": 128, + "o_groups": 8, + "o_lora_rank": 1024, + "compress_ratio": 1, + "source": false, + "indexer": "reuse", + "compress_rope_theta": 160000, + "rope_scaling": { + "type": "yarn", + "factor": 16, + "original_max": 65536, + "beta_fast": 32, + "beta_slow": 1 + }, + "topk": 512, + "index_heads": 32, + "index_dim": 128, + "proj": { + "wkv": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + }, + "wo_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + }, + "wq_a": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + }, + "wq_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + } + }, + "batch": { + "groups": [ + { + "count": 4, + "q": 1, + "ctx": 65536 + } + ] + } + }, + "sources": [ + "deepseek-ai/DeepSeek-V4.1-Flash/attn" + ] + }, + { + "type": "dsv4_attn", + "args": { + "hidden": 5120, + "heads": 64, + "head_dim": 512, + "rope_dim": 64, + "q_lora_rank": 1280, + "window": 128, + "o_groups": 8, + "o_lora_rank": 1024, + "compress_ratio": 1, + "source": false, + "indexer": "reuse", + "compress_rope_theta": 160000, + "rope_scaling": { + "type": "yarn", + "factor": 16, + "original_max": 65536, + "beta_fast": 32, + "beta_slow": 1 + }, + "topk": 512, + "index_heads": 32, + "index_dim": 128, + "proj": { + "wkv": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + }, + "wo_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + }, + "wq_a": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + }, + "wq_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + } + }, + "batch": { + "groups": [ + { + "count": 4, + "q": 1, + "ctx": 131072 + } + ] + } + }, + "sources": [ + "deepseek-ai/DeepSeek-V4.1-Flash/attn" + ] + }, + { + "type": "dsv4_attn", + "args": { + "hidden": 5120, + "heads": 64, + "head_dim": 512, + "rope_dim": 64, + "q_lora_rank": 1280, + "window": 128, + "o_groups": 8, + "o_lora_rank": 1024, + "compress_ratio": 1, + "source": false, + "indexer": "reuse", + "compress_rope_theta": 160000, + "rope_scaling": { + "type": "yarn", + "factor": 16, + "original_max": 65536, + "beta_fast": 32, + "beta_slow": 1 + }, + "topk": 512, + "index_heads": 32, + "index_dim": 128, + "proj": { + "wkv": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + }, + "wo_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + }, + "wq_a": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + }, + "wq_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + } + }, + "batch": { + "groups": [ + { + "count": 16, + "q": 1, + "ctx": 1024 + } + ] + } + }, + "sources": [ + "deepseek-ai/DeepSeek-V4.1-Flash/attn" + ] + }, + { + "type": "dsv4_attn", + "args": { + "hidden": 5120, + "heads": 64, + "head_dim": 512, + "rope_dim": 64, + "q_lora_rank": 1280, + "window": 128, + "o_groups": 8, + "o_lora_rank": 1024, + "compress_ratio": 1, + "source": false, + "indexer": "reuse", + "compress_rope_theta": 160000, + "rope_scaling": { + "type": "yarn", + "factor": 16, + "original_max": 65536, + "beta_fast": 32, + "beta_slow": 1 + }, + "topk": 512, + "index_heads": 32, + "index_dim": 128, + "proj": { + "wkv": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + }, + "wo_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + }, + "wq_a": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + }, + "wq_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + } + }, + "batch": { + "groups": [ + { + "count": 16, + "q": 1, + "ctx": 4096 + } + ] + } + }, + "sources": [ + "deepseek-ai/DeepSeek-V4.1-Flash/attn" + ] + }, + { + "type": "dsv4_attn", + "args": { + "hidden": 5120, + "heads": 64, + "head_dim": 512, + "rope_dim": 64, + "q_lora_rank": 1280, + "window": 128, + "o_groups": 8, + "o_lora_rank": 1024, + "compress_ratio": 1, + "source": false, + "indexer": "reuse", + "compress_rope_theta": 160000, + "rope_scaling": { + "type": "yarn", + "factor": 16, + "original_max": 65536, + "beta_fast": 32, + "beta_slow": 1 + }, + "topk": 512, + "index_heads": 32, + "index_dim": 128, + "proj": { + "wkv": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + }, + "wo_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + }, + "wq_a": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + }, + "wq_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + } + }, + "batch": { + "groups": [ + { + "count": 16, + "q": 1, + "ctx": 16384 + } + ] + } + }, + "sources": [ + "deepseek-ai/DeepSeek-V4.1-Flash/attn" + ] + }, + { + "type": "dsv4_attn", + "args": { + "hidden": 5120, + "heads": 64, + "head_dim": 512, + "rope_dim": 64, + "q_lora_rank": 1280, + "window": 128, + "o_groups": 8, + "o_lora_rank": 1024, + "compress_ratio": 1, + "source": false, + "indexer": "reuse", + "compress_rope_theta": 160000, + "rope_scaling": { + "type": "yarn", + "factor": 16, + "original_max": 65536, + "beta_fast": 32, + "beta_slow": 1 + }, + "topk": 512, + "index_heads": 32, + "index_dim": 128, + "proj": { + "wkv": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + }, + "wo_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + }, + "wq_a": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + }, + "wq_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + } + }, + "batch": { + "groups": [ + { + "count": 16, + "q": 1, + "ctx": 65536 + } + ] + } + }, + "sources": [ + "deepseek-ai/DeepSeek-V4.1-Flash/attn" + ] + }, + { + "type": "dsv4_attn", + "args": { + "hidden": 5120, + "heads": 64, + "head_dim": 512, + "rope_dim": 64, + "q_lora_rank": 1280, + "window": 128, + "o_groups": 8, + "o_lora_rank": 1024, + "compress_ratio": 1, + "source": false, + "indexer": "reuse", + "compress_rope_theta": 160000, + "rope_scaling": { + "type": "yarn", + "factor": 16, + "original_max": 65536, + "beta_fast": 32, + "beta_slow": 1 + }, + "topk": 512, + "index_heads": 32, + "index_dim": 128, + "proj": { + "wkv": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + }, + "wo_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + }, + "wq_a": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + }, + "wq_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + } + }, + "batch": { + "groups": [ + { + "count": 16, + "q": 1, + "ctx": 131072 + } + ] + } + }, + "sources": [ + "deepseek-ai/DeepSeek-V4.1-Flash/attn" + ] + }, + { + "type": "dsv4_attn", + "args": { + "hidden": 5120, + "heads": 64, + "head_dim": 512, + "rope_dim": 64, + "q_lora_rank": 1280, + "window": 128, + "o_groups": 8, + "o_lora_rank": 1024, + "compress_ratio": 1, + "source": false, + "indexer": "reuse", + "compress_rope_theta": 160000, + "rope_scaling": { + "type": "yarn", + "factor": 16, + "original_max": 65536, + "beta_fast": 32, + "beta_slow": 1 + }, + "topk": 512, + "index_heads": 32, + "index_dim": 128, + "proj": { + "wkv": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + }, + "wo_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + }, + "wq_a": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + }, + "wq_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + } + }, + "batch": { + "groups": [ + { + "count": 64, + "q": 1, + "ctx": 1024 + } + ] + } + }, + "sources": [ + "deepseek-ai/DeepSeek-V4.1-Flash/attn" + ] + }, + { + "type": "dsv4_attn", + "args": { + "hidden": 5120, + "heads": 64, + "head_dim": 512, + "rope_dim": 64, + "q_lora_rank": 1280, + "window": 128, + "o_groups": 8, + "o_lora_rank": 1024, + "compress_ratio": 1, + "source": false, + "indexer": "reuse", + "compress_rope_theta": 160000, + "rope_scaling": { + "type": "yarn", + "factor": 16, + "original_max": 65536, + "beta_fast": 32, + "beta_slow": 1 + }, + "topk": 512, + "index_heads": 32, + "index_dim": 128, + "proj": { + "wkv": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + }, + "wo_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + }, + "wq_a": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + }, + "wq_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + } + }, + "batch": { + "groups": [ + { + "count": 64, + "q": 1, + "ctx": 4096 + } + ] + } + }, + "sources": [ + "deepseek-ai/DeepSeek-V4.1-Flash/attn" + ] + }, + { + "type": "dsv4_attn", + "args": { + "hidden": 5120, + "heads": 64, + "head_dim": 512, + "rope_dim": 64, + "q_lora_rank": 1280, + "window": 128, + "o_groups": 8, + "o_lora_rank": 1024, + "compress_ratio": 1, + "source": false, + "indexer": "reuse", + "compress_rope_theta": 160000, + "rope_scaling": { + "type": "yarn", + "factor": 16, + "original_max": 65536, + "beta_fast": 32, + "beta_slow": 1 + }, + "topk": 512, + "index_heads": 32, + "index_dim": 128, + "proj": { + "wkv": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + }, + "wo_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + }, + "wq_a": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + }, + "wq_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + } + }, + "batch": { + "groups": [ + { + "count": 64, + "q": 1, + "ctx": 16384 + } + ] + } + }, + "sources": [ + "deepseek-ai/DeepSeek-V4.1-Flash/attn" + ] + }, + { + "type": "dsv4_attn", + "args": { + "hidden": 5120, + "heads": 64, + "head_dim": 512, + "rope_dim": 64, + "q_lora_rank": 1280, + "window": 128, + "o_groups": 8, + "o_lora_rank": 1024, + "compress_ratio": 1, + "source": false, + "indexer": "reuse", + "compress_rope_theta": 160000, + "rope_scaling": { + "type": "yarn", + "factor": 16, + "original_max": 65536, + "beta_fast": 32, + "beta_slow": 1 + }, + "topk": 512, + "index_heads": 32, + "index_dim": 128, + "proj": { + "wkv": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + }, + "wo_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + }, + "wq_a": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + }, + "wq_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + } + }, + "batch": { + "groups": [ + { + "count": 64, + "q": 1, + "ctx": 65536 + } + ] + } + }, + "sources": [ + "deepseek-ai/DeepSeek-V4.1-Flash/attn" + ] + }, + { + "type": "dsv4_attn", + "args": { + "hidden": 5120, + "heads": 64, + "head_dim": 512, + "rope_dim": 64, + "q_lora_rank": 1280, + "window": 128, + "o_groups": 8, + "o_lora_rank": 1024, + "compress_ratio": 1, + "source": false, + "indexer": "reuse", + "compress_rope_theta": 160000, + "rope_scaling": { + "type": "yarn", + "factor": 16, + "original_max": 65536, + "beta_fast": 32, + "beta_slow": 1 + }, + "topk": 512, + "index_heads": 32, + "index_dim": 128, + "proj": { + "wkv": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + }, + "wo_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + }, + "wq_a": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + }, + "wq_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + } + }, + "batch": { + "groups": [ + { + "count": 64, + "q": 1, + "ctx": 131072 + } + ] + } + }, + "sources": [ + "deepseek-ai/DeepSeek-V4.1-Flash/attn" + ] + }, + { + "type": "dsv4_attn", + "args": { + "hidden": 5120, + "heads": 64, + "head_dim": 512, + "rope_dim": 64, + "q_lora_rank": 1280, + "window": 128, + "o_groups": 8, + "o_lora_rank": 1024, + "compress_ratio": 1, + "source": false, + "indexer": "reuse", + "compress_rope_theta": 160000, + "rope_scaling": { + "type": "yarn", + "factor": 16, + "original_max": 65536, + "beta_fast": 32, + "beta_slow": 1 + }, + "topk": 512, + "index_heads": 32, + "index_dim": 128, + "proj": { + "wkv": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + }, + "wo_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + }, + "wq_a": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + }, + "wq_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + } + }, + "batch": { + "groups": [ + { + "count": 256, + "q": 1, + "ctx": 1024 + } + ] + } + }, + "sources": [ + "deepseek-ai/DeepSeek-V4.1-Flash/attn" + ] + }, + { + "type": "dsv4_attn", + "args": { + "hidden": 5120, + "heads": 64, + "head_dim": 512, + "rope_dim": 64, + "q_lora_rank": 1280, + "window": 128, + "o_groups": 8, + "o_lora_rank": 1024, + "compress_ratio": 1, + "source": false, + "indexer": "reuse", + "compress_rope_theta": 160000, + "rope_scaling": { + "type": "yarn", + "factor": 16, + "original_max": 65536, + "beta_fast": 32, + "beta_slow": 1 + }, + "topk": 512, + "index_heads": 32, + "index_dim": 128, + "proj": { + "wkv": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + }, + "wo_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + }, + "wq_a": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + }, + "wq_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + } + }, + "batch": { + "groups": [ + { + "count": 256, + "q": 1, + "ctx": 4096 + } + ] + } + }, + "sources": [ + "deepseek-ai/DeepSeek-V4.1-Flash/attn" + ] + }, + { + "type": "dsv4_attn", + "args": { + "hidden": 5120, + "heads": 64, + "head_dim": 512, + "rope_dim": 64, + "q_lora_rank": 1280, + "window": 128, + "o_groups": 8, + "o_lora_rank": 1024, + "compress_ratio": 1, + "source": false, + "indexer": "reuse", + "compress_rope_theta": 160000, + "rope_scaling": { + "type": "yarn", + "factor": 16, + "original_max": 65536, + "beta_fast": 32, + "beta_slow": 1 + }, + "topk": 512, + "index_heads": 32, + "index_dim": 128, + "proj": { + "wkv": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + }, + "wo_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + }, + "wq_a": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + }, + "wq_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + } + }, + "batch": { + "groups": [ + { + "count": 256, + "q": 1, + "ctx": 16384 + } + ] + } + }, + "sources": [ + "deepseek-ai/DeepSeek-V4.1-Flash/attn" + ] + }, + { + "type": "dsv4_attn", + "args": { + "hidden": 5120, + "heads": 64, + "head_dim": 512, + "rope_dim": 64, + "q_lora_rank": 1280, + "window": 128, + "o_groups": 8, + "o_lora_rank": 1024, + "compress_ratio": 1, + "source": false, + "indexer": "reuse", + "compress_rope_theta": 160000, + "rope_scaling": { + "type": "yarn", + "factor": 16, + "original_max": 65536, + "beta_fast": 32, + "beta_slow": 1 + }, + "topk": 512, + "index_heads": 32, + "index_dim": 128, + "proj": { + "wkv": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + }, + "wo_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + }, + "wq_a": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + }, + "wq_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + } + }, + "batch": { + "groups": [ + { + "count": 256, + "q": 1, + "ctx": 65536 + } + ] + } + }, + "sources": [ + "deepseek-ai/DeepSeek-V4.1-Flash/attn" + ] + }, + { + "type": "dsv4_attn", + "args": { + "hidden": 5120, + "heads": 64, + "head_dim": 512, + "rope_dim": 64, + "q_lora_rank": 1280, + "window": 128, + "o_groups": 8, + "o_lora_rank": 1024, + "compress_ratio": 1, + "source": false, + "indexer": "reuse", + "compress_rope_theta": 160000, + "rope_scaling": { + "type": "yarn", + "factor": 16, + "original_max": 65536, + "beta_fast": 32, + "beta_slow": 1 + }, + "topk": 512, + "index_heads": 32, + "index_dim": 128, + "proj": { + "wkv": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + }, + "wo_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + }, + "wq_a": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + }, + "wq_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + } + }, + "batch": { + "groups": [ + { + "count": 256, + "q": 1, + "ctx": 131072 + } + ] + } + }, + "sources": [ + "deepseek-ai/DeepSeek-V4.1-Flash/attn" + ] + }, + { + "type": "dsv4_attn", + "args": { + "hidden": 5120, + "heads": 64, + "head_dim": 512, + "rope_dim": 64, + "q_lora_rank": 1280, + "window": 128, + "o_groups": 8, + "o_lora_rank": 1024, + "compress_ratio": 1, + "source": false, + "indexer": "reuse", + "compress_rope_theta": 160000, + "rope_scaling": { + "type": "yarn", + "factor": 16, + "original_max": 65536, + "beta_fast": 32, + "beta_slow": 1 + }, + "topk": 512, + "index_heads": 32, + "index_dim": 128, + "proj": { + "wkv": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + }, + "wo_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + }, + "wq_a": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + }, + "wq_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + } + }, + "batch": { + "groups": [ + { + "count": 1, + "q": 1024, + "ctx": 0 + } + ] + } + }, + "sources": [ + "deepseek-ai/DeepSeek-V4.1-Flash/attn" + ] + }, + { + "type": "dsv4_attn", + "args": { + "hidden": 5120, + "heads": 64, + "head_dim": 512, + "rope_dim": 64, + "q_lora_rank": 1280, + "window": 128, + "o_groups": 8, + "o_lora_rank": 1024, + "compress_ratio": 1, + "source": false, + "indexer": "reuse", + "compress_rope_theta": 160000, + "rope_scaling": { + "type": "yarn", + "factor": 16, + "original_max": 65536, + "beta_fast": 32, + "beta_slow": 1 + }, + "topk": 512, + "index_heads": 32, + "index_dim": 128, + "proj": { + "wkv": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + }, + "wo_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + }, + "wq_a": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + }, + "wq_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + } + }, + "batch": { + "groups": [ + { + "count": 1, + "q": 4096, + "ctx": 0 + } + ] + } + }, + "sources": [ + "deepseek-ai/DeepSeek-V4.1-Flash/attn" + ] + }, + { + "type": "dsv4_attn", + "args": { + "hidden": 5120, + "heads": 64, + "head_dim": 512, + "rope_dim": 64, + "q_lora_rank": 1280, + "window": 128, + "o_groups": 8, + "o_lora_rank": 1024, + "compress_ratio": 1, + "source": false, + "indexer": "reuse", + "compress_rope_theta": 160000, + "rope_scaling": { + "type": "yarn", + "factor": 16, + "original_max": 65536, + "beta_fast": 32, + "beta_slow": 1 + }, + "topk": 512, + "index_heads": 32, + "index_dim": 128, + "proj": { + "wkv": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + }, + "wo_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + }, + "wq_a": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + }, + "wq_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + } + }, + "batch": { + "groups": [ + { + "count": 1, + "q": 16384, + "ctx": 0 + } + ] + } + }, + "sources": [ + "deepseek-ai/DeepSeek-V4.1-Flash/attn" + ] + }, + { + "type": "dsv4_attn", + "args": { + "hidden": 5120, + "heads": 64, + "head_dim": 512, + "rope_dim": 64, + "q_lora_rank": 1280, + "window": 128, + "o_groups": 8, + "o_lora_rank": 1024, + "compress_ratio": 1, + "source": false, + "indexer": "reuse", + "compress_rope_theta": 160000, + "rope_scaling": { + "type": "yarn", + "factor": 16, + "original_max": 65536, + "beta_fast": 32, + "beta_slow": 1 + }, + "topk": 512, + "index_heads": 32, + "index_dim": 128, + "proj": { + "wkv": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + }, + "wo_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + }, + "wq_a": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + }, + "wq_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + } + }, + "batch": { + "groups": [ + { + "count": 1, + "q": 32768, + "ctx": 0 + } + ] + } + }, + "sources": [ + "deepseek-ai/DeepSeek-V4.1-Flash/attn" + ] + }, + { + "type": "dsv4_attn", + "args": { + "hidden": 5120, + "heads": 64, + "head_dim": 512, + "rope_dim": 64, + "q_lora_rank": 1280, + "window": 128, + "o_groups": 8, + "o_lora_rank": 1024, + "compress_ratio": 1, + "source": false, + "indexer": "reuse", + "compress_rope_theta": 160000, + "rope_scaling": { + "type": "yarn", + "factor": 16, + "original_max": 65536, + "beta_fast": 32, + "beta_slow": 1 + }, + "topk": 512, + "index_heads": 32, + "index_dim": 128, + "proj": { + "wkv": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + }, + "wo_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + }, + "wq_a": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + }, + "wq_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + } + }, + "batch": { + "groups": [ + { + "count": 1, + "q": 8192, + "ctx": 8192 + } + ] + } + }, + "sources": [ + "deepseek-ai/DeepSeek-V4.1-Flash/attn" + ] + }, + { + "type": "dsv4_attn", + "args": { + "hidden": 5120, + "heads": 64, + "head_dim": 512, + "rope_dim": 64, + "q_lora_rank": 1280, + "window": 128, + "o_groups": 8, + "o_lora_rank": 1024, + "compress_ratio": 1, + "source": false, + "indexer": "reuse", + "compress_rope_theta": 160000, + "rope_scaling": { + "type": "yarn", + "factor": 16, + "original_max": 65536, + "beta_fast": 32, + "beta_slow": 1 + }, + "topk": 512, + "index_heads": 32, + "index_dim": 128, + "proj": { + "wkv": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + }, + "wo_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + }, + "wq_a": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + }, + "wq_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + } + }, + "batch": { + "groups": [ + { + "count": 1, + "q": 8192, + "ctx": 32768 + } + ] + } + }, + "sources": [ + "deepseek-ai/DeepSeek-V4.1-Flash/attn" + ] + }, + { + "type": "dsv4_attn", + "args": { + "hidden": 5120, + "heads": 64, + "head_dim": 512, + "rope_dim": 64, + "q_lora_rank": 1280, + "window": 128, + "o_groups": 8, + "o_lora_rank": 1024, + "compress_ratio": 1, + "source": false, + "indexer": "reuse", + "compress_rope_theta": 160000, + "rope_scaling": { + "type": "yarn", + "factor": 16, + "original_max": 65536, + "beta_fast": 32, + "beta_slow": 1 + }, + "topk": 512, + "index_heads": 32, + "index_dim": 128, + "proj": { + "wkv": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + }, + "wo_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + }, + "wq_a": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + }, + "wq_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + } + }, + "batch": { + "groups": [ + { + "count": 1, + "q": 8192, + "ctx": 131072 + } + ] + } + }, + "sources": [ + "deepseek-ai/DeepSeek-V4.1-Flash/attn" + ] + }, + { + "type": "dsv4_attn", + "args": { + "hidden": 5120, + "heads": 64, + "head_dim": 512, + "rope_dim": 64, + "q_lora_rank": 1280, + "window": 128, + "o_groups": 8, + "o_lora_rank": 1024, + "compress_ratio": 1, + "source": false, + "indexer": "reuse", + "compress_rope_theta": 160000, + "rope_scaling": { + "type": "yarn", + "factor": 16, + "original_max": 65536, + "beta_fast": 32, + "beta_slow": 1 + }, + "topk": 512, + "index_heads": 32, + "index_dim": 128, + "proj": { + "wkv": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + }, + "wo_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + }, + "wq_a": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + }, + "wq_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + } + }, + "batch": { + "groups": [ + { + "count": 1, + "q": 8192, + "ctx": 8192 + }, + { + "count": 64, + "q": 1, + "ctx": 8192 + } + ] + } + }, + "sources": [ + "deepseek-ai/DeepSeek-V4.1-Flash/attn" + ] + }, + { + "type": "dsv4_attn", + "args": { + "hidden": 5120, + "heads": 64, + "head_dim": 512, + "rope_dim": 64, + "q_lora_rank": 1280, + "window": 128, + "o_groups": 8, + "o_lora_rank": 1024, + "compress_ratio": 1, + "source": false, + "indexer": "shared", + "compress_rope_theta": 160000, + "rope_scaling": { + "type": "yarn", + "factor": 16, + "original_max": 65536, + "beta_fast": 32, + "beta_slow": 1 + }, + "topk": 512, + "index_heads": 32, + "index_dim": 128, + "proj": { + "indexer.wq_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + }, + "wkv": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + }, + "wo_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + }, + "wq_a": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + }, + "wq_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + } + }, + "batch": { + "groups": [ + { + "count": 1, + "q": 1, + "ctx": 1024 + } + ] + } + }, + "sources": [ + "deepseek-ai/DeepSeek-V4.1-Flash/attn" + ] + }, + { + "type": "dsv4_attn", + "args": { + "hidden": 5120, + "heads": 64, + "head_dim": 512, + "rope_dim": 64, + "q_lora_rank": 1280, + "window": 128, + "o_groups": 8, + "o_lora_rank": 1024, + "compress_ratio": 1, + "source": false, + "indexer": "shared", + "compress_rope_theta": 160000, + "rope_scaling": { + "type": "yarn", + "factor": 16, + "original_max": 65536, + "beta_fast": 32, + "beta_slow": 1 + }, + "topk": 512, + "index_heads": 32, + "index_dim": 128, + "proj": { + "indexer.wq_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + }, + "wkv": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + }, + "wo_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + }, + "wq_a": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + }, + "wq_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + } + }, + "batch": { + "groups": [ + { + "count": 1, + "q": 1, + "ctx": 4096 + } + ] + } + }, + "sources": [ + "deepseek-ai/DeepSeek-V4.1-Flash/attn" + ] + }, + { + "type": "dsv4_attn", + "args": { + "hidden": 5120, + "heads": 64, + "head_dim": 512, + "rope_dim": 64, + "q_lora_rank": 1280, + "window": 128, + "o_groups": 8, + "o_lora_rank": 1024, + "compress_ratio": 1, + "source": false, + "indexer": "shared", + "compress_rope_theta": 160000, + "rope_scaling": { + "type": "yarn", + "factor": 16, + "original_max": 65536, + "beta_fast": 32, + "beta_slow": 1 + }, + "topk": 512, + "index_heads": 32, + "index_dim": 128, + "proj": { + "indexer.wq_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + }, + "wkv": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + }, + "wo_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + }, + "wq_a": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + }, + "wq_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + } + }, + "batch": { + "groups": [ + { + "count": 1, + "q": 1, + "ctx": 16384 + } + ] + } + }, + "sources": [ + "deepseek-ai/DeepSeek-V4.1-Flash/attn" + ] + }, + { + "type": "dsv4_attn", + "args": { + "hidden": 5120, + "heads": 64, + "head_dim": 512, + "rope_dim": 64, + "q_lora_rank": 1280, + "window": 128, + "o_groups": 8, + "o_lora_rank": 1024, + "compress_ratio": 1, + "source": false, + "indexer": "shared", + "compress_rope_theta": 160000, + "rope_scaling": { + "type": "yarn", + "factor": 16, + "original_max": 65536, + "beta_fast": 32, + "beta_slow": 1 + }, + "topk": 512, + "index_heads": 32, + "index_dim": 128, + "proj": { + "indexer.wq_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + }, + "wkv": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + }, + "wo_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + }, + "wq_a": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + }, + "wq_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + } + }, + "batch": { + "groups": [ + { + "count": 1, + "q": 1, + "ctx": 65536 + } + ] + } + }, + "sources": [ + "deepseek-ai/DeepSeek-V4.1-Flash/attn" + ] + }, + { + "type": "dsv4_attn", + "args": { + "hidden": 5120, + "heads": 64, + "head_dim": 512, + "rope_dim": 64, + "q_lora_rank": 1280, + "window": 128, + "o_groups": 8, + "o_lora_rank": 1024, + "compress_ratio": 1, + "source": false, + "indexer": "shared", + "compress_rope_theta": 160000, + "rope_scaling": { + "type": "yarn", + "factor": 16, + "original_max": 65536, + "beta_fast": 32, + "beta_slow": 1 + }, + "topk": 512, + "index_heads": 32, + "index_dim": 128, + "proj": { + "indexer.wq_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + }, + "wkv": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + }, + "wo_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + }, + "wq_a": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + }, + "wq_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + } + }, + "batch": { + "groups": [ + { + "count": 1, + "q": 1, + "ctx": 131072 + } + ] + } + }, + "sources": [ + "deepseek-ai/DeepSeek-V4.1-Flash/attn" + ] + }, + { + "type": "dsv4_attn", + "args": { + "hidden": 5120, + "heads": 64, + "head_dim": 512, + "rope_dim": 64, + "q_lora_rank": 1280, + "window": 128, + "o_groups": 8, + "o_lora_rank": 1024, + "compress_ratio": 1, + "source": false, + "indexer": "shared", + "compress_rope_theta": 160000, + "rope_scaling": { + "type": "yarn", + "factor": 16, + "original_max": 65536, + "beta_fast": 32, + "beta_slow": 1 + }, + "topk": 512, + "index_heads": 32, + "index_dim": 128, + "proj": { + "indexer.wq_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + }, + "wkv": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + }, + "wo_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + }, + "wq_a": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + }, + "wq_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + } + }, + "batch": { + "groups": [ + { + "count": 4, + "q": 1, + "ctx": 1024 + } + ] + } + }, + "sources": [ + "deepseek-ai/DeepSeek-V4.1-Flash/attn" + ] + }, + { + "type": "dsv4_attn", + "args": { + "hidden": 5120, + "heads": 64, + "head_dim": 512, + "rope_dim": 64, + "q_lora_rank": 1280, + "window": 128, + "o_groups": 8, + "o_lora_rank": 1024, + "compress_ratio": 1, + "source": false, + "indexer": "shared", + "compress_rope_theta": 160000, + "rope_scaling": { + "type": "yarn", + "factor": 16, + "original_max": 65536, + "beta_fast": 32, + "beta_slow": 1 + }, + "topk": 512, + "index_heads": 32, + "index_dim": 128, + "proj": { + "indexer.wq_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + }, + "wkv": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + }, + "wo_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + }, + "wq_a": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + }, + "wq_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + } + }, + "batch": { + "groups": [ + { + "count": 4, + "q": 1, + "ctx": 4096 + } + ] + } + }, + "sources": [ + "deepseek-ai/DeepSeek-V4.1-Flash/attn" + ] + }, + { + "type": "dsv4_attn", + "args": { + "hidden": 5120, + "heads": 64, + "head_dim": 512, + "rope_dim": 64, + "q_lora_rank": 1280, + "window": 128, + "o_groups": 8, + "o_lora_rank": 1024, + "compress_ratio": 1, + "source": false, + "indexer": "shared", + "compress_rope_theta": 160000, + "rope_scaling": { + "type": "yarn", + "factor": 16, + "original_max": 65536, + "beta_fast": 32, + "beta_slow": 1 + }, + "topk": 512, + "index_heads": 32, + "index_dim": 128, + "proj": { + "indexer.wq_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + }, + "wkv": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + }, + "wo_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + }, + "wq_a": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + }, + "wq_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + } + }, + "batch": { + "groups": [ + { + "count": 4, + "q": 1, + "ctx": 16384 + } + ] + } + }, + "sources": [ + "deepseek-ai/DeepSeek-V4.1-Flash/attn" + ] + }, + { + "type": "dsv4_attn", + "args": { + "hidden": 5120, + "heads": 64, + "head_dim": 512, + "rope_dim": 64, + "q_lora_rank": 1280, + "window": 128, + "o_groups": 8, + "o_lora_rank": 1024, + "compress_ratio": 1, + "source": false, + "indexer": "shared", + "compress_rope_theta": 160000, + "rope_scaling": { + "type": "yarn", + "factor": 16, + "original_max": 65536, + "beta_fast": 32, + "beta_slow": 1 + }, + "topk": 512, + "index_heads": 32, + "index_dim": 128, + "proj": { + "indexer.wq_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + }, + "wkv": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + }, + "wo_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + }, + "wq_a": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + }, + "wq_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + } + }, + "batch": { + "groups": [ + { + "count": 4, + "q": 1, + "ctx": 65536 + } + ] + } + }, + "sources": [ + "deepseek-ai/DeepSeek-V4.1-Flash/attn" + ] + }, + { + "type": "dsv4_attn", + "args": { + "hidden": 5120, + "heads": 64, + "head_dim": 512, + "rope_dim": 64, + "q_lora_rank": 1280, + "window": 128, + "o_groups": 8, + "o_lora_rank": 1024, + "compress_ratio": 1, + "source": false, + "indexer": "shared", + "compress_rope_theta": 160000, + "rope_scaling": { + "type": "yarn", + "factor": 16, + "original_max": 65536, + "beta_fast": 32, + "beta_slow": 1 + }, + "topk": 512, + "index_heads": 32, + "index_dim": 128, + "proj": { + "indexer.wq_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + }, + "wkv": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + }, + "wo_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + }, + "wq_a": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + }, + "wq_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + } + }, + "batch": { + "groups": [ + { + "count": 4, + "q": 1, + "ctx": 131072 + } + ] + } + }, + "sources": [ + "deepseek-ai/DeepSeek-V4.1-Flash/attn" + ] + }, + { + "type": "dsv4_attn", + "args": { + "hidden": 5120, + "heads": 64, + "head_dim": 512, + "rope_dim": 64, + "q_lora_rank": 1280, + "window": 128, + "o_groups": 8, + "o_lora_rank": 1024, + "compress_ratio": 1, + "source": false, + "indexer": "shared", + "compress_rope_theta": 160000, + "rope_scaling": { + "type": "yarn", + "factor": 16, + "original_max": 65536, + "beta_fast": 32, + "beta_slow": 1 + }, + "topk": 512, + "index_heads": 32, + "index_dim": 128, + "proj": { + "indexer.wq_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + }, + "wkv": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + }, + "wo_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + }, + "wq_a": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + }, + "wq_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + } + }, + "batch": { + "groups": [ + { + "count": 16, + "q": 1, + "ctx": 1024 + } + ] + } + }, + "sources": [ + "deepseek-ai/DeepSeek-V4.1-Flash/attn" + ] + }, + { + "type": "dsv4_attn", + "args": { + "hidden": 5120, + "heads": 64, + "head_dim": 512, + "rope_dim": 64, + "q_lora_rank": 1280, + "window": 128, + "o_groups": 8, + "o_lora_rank": 1024, + "compress_ratio": 1, + "source": false, + "indexer": "shared", + "compress_rope_theta": 160000, + "rope_scaling": { + "type": "yarn", + "factor": 16, + "original_max": 65536, + "beta_fast": 32, + "beta_slow": 1 + }, + "topk": 512, + "index_heads": 32, + "index_dim": 128, + "proj": { + "indexer.wq_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + }, + "wkv": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + }, + "wo_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + }, + "wq_a": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + }, + "wq_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + } + }, + "batch": { + "groups": [ + { + "count": 16, + "q": 1, + "ctx": 4096 + } + ] + } + }, + "sources": [ + "deepseek-ai/DeepSeek-V4.1-Flash/attn" + ] + }, + { + "type": "dsv4_attn", + "args": { + "hidden": 5120, + "heads": 64, + "head_dim": 512, + "rope_dim": 64, + "q_lora_rank": 1280, + "window": 128, + "o_groups": 8, + "o_lora_rank": 1024, + "compress_ratio": 1, + "source": false, + "indexer": "shared", + "compress_rope_theta": 160000, + "rope_scaling": { + "type": "yarn", + "factor": 16, + "original_max": 65536, + "beta_fast": 32, + "beta_slow": 1 + }, + "topk": 512, + "index_heads": 32, + "index_dim": 128, + "proj": { + "indexer.wq_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + }, + "wkv": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + }, + "wo_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + }, + "wq_a": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + }, + "wq_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + } + }, + "batch": { + "groups": [ + { + "count": 16, + "q": 1, + "ctx": 16384 + } + ] + } + }, + "sources": [ + "deepseek-ai/DeepSeek-V4.1-Flash/attn" + ] + }, + { + "type": "dsv4_attn", + "args": { + "hidden": 5120, + "heads": 64, + "head_dim": 512, + "rope_dim": 64, + "q_lora_rank": 1280, + "window": 128, + "o_groups": 8, + "o_lora_rank": 1024, + "compress_ratio": 1, + "source": false, + "indexer": "shared", + "compress_rope_theta": 160000, + "rope_scaling": { + "type": "yarn", + "factor": 16, + "original_max": 65536, + "beta_fast": 32, + "beta_slow": 1 + }, + "topk": 512, + "index_heads": 32, + "index_dim": 128, + "proj": { + "indexer.wq_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + }, + "wkv": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + }, + "wo_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + }, + "wq_a": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + }, + "wq_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + } + }, + "batch": { + "groups": [ + { + "count": 16, + "q": 1, + "ctx": 65536 + } + ] + } + }, + "sources": [ + "deepseek-ai/DeepSeek-V4.1-Flash/attn" + ] + }, + { + "type": "dsv4_attn", + "args": { + "hidden": 5120, + "heads": 64, + "head_dim": 512, + "rope_dim": 64, + "q_lora_rank": 1280, + "window": 128, + "o_groups": 8, + "o_lora_rank": 1024, + "compress_ratio": 1, + "source": false, + "indexer": "shared", + "compress_rope_theta": 160000, + "rope_scaling": { + "type": "yarn", + "factor": 16, + "original_max": 65536, + "beta_fast": 32, + "beta_slow": 1 + }, + "topk": 512, + "index_heads": 32, + "index_dim": 128, + "proj": { + "indexer.wq_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + }, + "wkv": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + }, + "wo_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + }, + "wq_a": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + }, + "wq_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + } + }, + "batch": { + "groups": [ + { + "count": 16, + "q": 1, + "ctx": 131072 + } + ] + } + }, + "sources": [ + "deepseek-ai/DeepSeek-V4.1-Flash/attn" + ] + }, + { + "type": "dsv4_attn", + "args": { + "hidden": 5120, + "heads": 64, + "head_dim": 512, + "rope_dim": 64, + "q_lora_rank": 1280, + "window": 128, + "o_groups": 8, + "o_lora_rank": 1024, + "compress_ratio": 1, + "source": false, + "indexer": "shared", + "compress_rope_theta": 160000, + "rope_scaling": { + "type": "yarn", + "factor": 16, + "original_max": 65536, + "beta_fast": 32, + "beta_slow": 1 + }, + "topk": 512, + "index_heads": 32, + "index_dim": 128, + "proj": { + "indexer.wq_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + }, + "wkv": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + }, + "wo_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + }, + "wq_a": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + }, + "wq_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + } + }, + "batch": { + "groups": [ + { + "count": 64, + "q": 1, + "ctx": 1024 + } + ] + } + }, + "sources": [ + "deepseek-ai/DeepSeek-V4.1-Flash/attn" + ] + }, + { + "type": "dsv4_attn", + "args": { + "hidden": 5120, + "heads": 64, + "head_dim": 512, + "rope_dim": 64, + "q_lora_rank": 1280, + "window": 128, + "o_groups": 8, + "o_lora_rank": 1024, + "compress_ratio": 1, + "source": false, + "indexer": "shared", + "compress_rope_theta": 160000, + "rope_scaling": { + "type": "yarn", + "factor": 16, + "original_max": 65536, + "beta_fast": 32, + "beta_slow": 1 + }, + "topk": 512, + "index_heads": 32, + "index_dim": 128, + "proj": { + "indexer.wq_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + }, + "wkv": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + }, + "wo_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + }, + "wq_a": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + }, + "wq_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + } + }, + "batch": { + "groups": [ + { + "count": 64, + "q": 1, + "ctx": 4096 + } + ] + } + }, + "sources": [ + "deepseek-ai/DeepSeek-V4.1-Flash/attn" + ] + }, + { + "type": "dsv4_attn", + "args": { + "hidden": 5120, + "heads": 64, + "head_dim": 512, + "rope_dim": 64, + "q_lora_rank": 1280, + "window": 128, + "o_groups": 8, + "o_lora_rank": 1024, + "compress_ratio": 1, + "source": false, + "indexer": "shared", + "compress_rope_theta": 160000, + "rope_scaling": { + "type": "yarn", + "factor": 16, + "original_max": 65536, + "beta_fast": 32, + "beta_slow": 1 + }, + "topk": 512, + "index_heads": 32, + "index_dim": 128, + "proj": { + "indexer.wq_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + }, + "wkv": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + }, + "wo_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + }, + "wq_a": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + }, + "wq_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + } + }, + "batch": { + "groups": [ + { + "count": 64, + "q": 1, + "ctx": 16384 + } + ] + } + }, + "sources": [ + "deepseek-ai/DeepSeek-V4.1-Flash/attn" + ] + }, + { + "type": "dsv4_attn", + "args": { + "hidden": 5120, + "heads": 64, + "head_dim": 512, + "rope_dim": 64, + "q_lora_rank": 1280, + "window": 128, + "o_groups": 8, + "o_lora_rank": 1024, + "compress_ratio": 1, + "source": false, + "indexer": "shared", + "compress_rope_theta": 160000, + "rope_scaling": { + "type": "yarn", + "factor": 16, + "original_max": 65536, + "beta_fast": 32, + "beta_slow": 1 + }, + "topk": 512, + "index_heads": 32, + "index_dim": 128, + "proj": { + "indexer.wq_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + }, + "wkv": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + }, + "wo_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + }, + "wq_a": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + }, + "wq_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + } + }, + "batch": { + "groups": [ + { + "count": 64, + "q": 1, + "ctx": 65536 + } + ] + } + }, + "sources": [ + "deepseek-ai/DeepSeek-V4.1-Flash/attn" + ] + }, + { + "type": "dsv4_attn", + "args": { + "hidden": 5120, + "heads": 64, + "head_dim": 512, + "rope_dim": 64, + "q_lora_rank": 1280, + "window": 128, + "o_groups": 8, + "o_lora_rank": 1024, + "compress_ratio": 1, + "source": false, + "indexer": "shared", + "compress_rope_theta": 160000, + "rope_scaling": { + "type": "yarn", + "factor": 16, + "original_max": 65536, + "beta_fast": 32, + "beta_slow": 1 + }, + "topk": 512, + "index_heads": 32, + "index_dim": 128, + "proj": { + "indexer.wq_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + }, + "wkv": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + }, + "wo_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + }, + "wq_a": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + }, + "wq_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + } + }, + "batch": { + "groups": [ + { + "count": 64, + "q": 1, + "ctx": 131072 + } + ] + } + }, + "sources": [ + "deepseek-ai/DeepSeek-V4.1-Flash/attn" + ] + }, + { + "type": "dsv4_attn", + "args": { + "hidden": 5120, + "heads": 64, + "head_dim": 512, + "rope_dim": 64, + "q_lora_rank": 1280, + "window": 128, + "o_groups": 8, + "o_lora_rank": 1024, + "compress_ratio": 1, + "source": false, + "indexer": "shared", + "compress_rope_theta": 160000, + "rope_scaling": { + "type": "yarn", + "factor": 16, + "original_max": 65536, + "beta_fast": 32, + "beta_slow": 1 + }, + "topk": 512, + "index_heads": 32, + "index_dim": 128, + "proj": { + "indexer.wq_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + }, + "wkv": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + }, + "wo_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + }, + "wq_a": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + }, + "wq_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + } + }, + "batch": { + "groups": [ + { + "count": 256, + "q": 1, + "ctx": 1024 + } + ] + } + }, + "sources": [ + "deepseek-ai/DeepSeek-V4.1-Flash/attn" + ] + }, + { + "type": "dsv4_attn", + "args": { + "hidden": 5120, + "heads": 64, + "head_dim": 512, + "rope_dim": 64, + "q_lora_rank": 1280, + "window": 128, + "o_groups": 8, + "o_lora_rank": 1024, + "compress_ratio": 1, + "source": false, + "indexer": "shared", + "compress_rope_theta": 160000, + "rope_scaling": { + "type": "yarn", + "factor": 16, + "original_max": 65536, + "beta_fast": 32, + "beta_slow": 1 + }, + "topk": 512, + "index_heads": 32, + "index_dim": 128, + "proj": { + "indexer.wq_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + }, + "wkv": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + }, + "wo_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + }, + "wq_a": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + }, + "wq_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + } + }, + "batch": { + "groups": [ + { + "count": 256, + "q": 1, + "ctx": 4096 + } + ] + } + }, + "sources": [ + "deepseek-ai/DeepSeek-V4.1-Flash/attn" + ] + }, + { + "type": "dsv4_attn", + "args": { + "hidden": 5120, + "heads": 64, + "head_dim": 512, + "rope_dim": 64, + "q_lora_rank": 1280, + "window": 128, + "o_groups": 8, + "o_lora_rank": 1024, + "compress_ratio": 1, + "source": false, + "indexer": "shared", + "compress_rope_theta": 160000, + "rope_scaling": { + "type": "yarn", + "factor": 16, + "original_max": 65536, + "beta_fast": 32, + "beta_slow": 1 + }, + "topk": 512, + "index_heads": 32, + "index_dim": 128, + "proj": { + "indexer.wq_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + }, + "wkv": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + }, + "wo_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + }, + "wq_a": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + }, + "wq_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + } + }, + "batch": { + "groups": [ + { + "count": 256, + "q": 1, + "ctx": 16384 + } + ] + } + }, + "sources": [ + "deepseek-ai/DeepSeek-V4.1-Flash/attn" + ] + }, + { + "type": "dsv4_attn", + "args": { + "hidden": 5120, + "heads": 64, + "head_dim": 512, + "rope_dim": 64, + "q_lora_rank": 1280, + "window": 128, + "o_groups": 8, + "o_lora_rank": 1024, + "compress_ratio": 1, + "source": false, + "indexer": "shared", + "compress_rope_theta": 160000, + "rope_scaling": { + "type": "yarn", + "factor": 16, + "original_max": 65536, + "beta_fast": 32, + "beta_slow": 1 + }, + "topk": 512, + "index_heads": 32, + "index_dim": 128, + "proj": { + "indexer.wq_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + }, + "wkv": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + }, + "wo_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + }, + "wq_a": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + }, + "wq_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + } + }, + "batch": { + "groups": [ + { + "count": 256, + "q": 1, + "ctx": 65536 + } + ] + } + }, + "sources": [ + "deepseek-ai/DeepSeek-V4.1-Flash/attn" + ] + }, + { + "type": "dsv4_attn", + "args": { + "hidden": 5120, + "heads": 64, + "head_dim": 512, + "rope_dim": 64, + "q_lora_rank": 1280, + "window": 128, + "o_groups": 8, + "o_lora_rank": 1024, + "compress_ratio": 1, + "source": false, + "indexer": "shared", + "compress_rope_theta": 160000, + "rope_scaling": { + "type": "yarn", + "factor": 16, + "original_max": 65536, + "beta_fast": 32, + "beta_slow": 1 + }, + "topk": 512, + "index_heads": 32, + "index_dim": 128, + "proj": { + "indexer.wq_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + }, + "wkv": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + }, + "wo_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + }, + "wq_a": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + }, + "wq_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + } + }, + "batch": { + "groups": [ + { + "count": 256, + "q": 1, + "ctx": 131072 + } + ] + } + }, + "sources": [ + "deepseek-ai/DeepSeek-V4.1-Flash/attn" + ] + }, + { + "type": "dsv4_attn", + "args": { + "hidden": 5120, + "heads": 64, + "head_dim": 512, + "rope_dim": 64, + "q_lora_rank": 1280, + "window": 128, + "o_groups": 8, + "o_lora_rank": 1024, + "compress_ratio": 1, + "source": false, + "indexer": "shared", + "compress_rope_theta": 160000, + "rope_scaling": { + "type": "yarn", + "factor": 16, + "original_max": 65536, + "beta_fast": 32, + "beta_slow": 1 + }, + "topk": 512, + "index_heads": 32, + "index_dim": 128, + "proj": { + "indexer.wq_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + }, + "wkv": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + }, + "wo_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + }, + "wq_a": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + }, + "wq_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + } + }, + "batch": { + "groups": [ + { + "count": 1, + "q": 1024, + "ctx": 0 + } + ] + } + }, + "sources": [ + "deepseek-ai/DeepSeek-V4.1-Flash/attn" + ] + }, + { + "type": "dsv4_attn", + "args": { + "hidden": 5120, + "heads": 64, + "head_dim": 512, + "rope_dim": 64, + "q_lora_rank": 1280, + "window": 128, + "o_groups": 8, + "o_lora_rank": 1024, + "compress_ratio": 1, + "source": false, + "indexer": "shared", + "compress_rope_theta": 160000, + "rope_scaling": { + "type": "yarn", + "factor": 16, + "original_max": 65536, + "beta_fast": 32, + "beta_slow": 1 + }, + "topk": 512, + "index_heads": 32, + "index_dim": 128, + "proj": { + "indexer.wq_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + }, + "wkv": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + }, + "wo_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + }, + "wq_a": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + }, + "wq_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + } + }, + "batch": { + "groups": [ + { + "count": 1, + "q": 4096, + "ctx": 0 + } + ] + } + }, + "sources": [ + "deepseek-ai/DeepSeek-V4.1-Flash/attn" + ] + }, + { + "type": "dsv4_attn", + "args": { + "hidden": 5120, + "heads": 64, + "head_dim": 512, + "rope_dim": 64, + "q_lora_rank": 1280, + "window": 128, + "o_groups": 8, + "o_lora_rank": 1024, + "compress_ratio": 1, + "source": false, + "indexer": "shared", + "compress_rope_theta": 160000, + "rope_scaling": { + "type": "yarn", + "factor": 16, + "original_max": 65536, + "beta_fast": 32, + "beta_slow": 1 + }, + "topk": 512, + "index_heads": 32, + "index_dim": 128, + "proj": { + "indexer.wq_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + }, + "wkv": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + }, + "wo_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + }, + "wq_a": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + }, + "wq_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + } + }, + "batch": { + "groups": [ + { + "count": 1, + "q": 16384, + "ctx": 0 + } + ] + } + }, + "sources": [ + "deepseek-ai/DeepSeek-V4.1-Flash/attn" + ] + }, + { + "type": "dsv4_attn", + "args": { + "hidden": 5120, + "heads": 64, + "head_dim": 512, + "rope_dim": 64, + "q_lora_rank": 1280, + "window": 128, + "o_groups": 8, + "o_lora_rank": 1024, + "compress_ratio": 1, + "source": false, + "indexer": "shared", + "compress_rope_theta": 160000, + "rope_scaling": { + "type": "yarn", + "factor": 16, + "original_max": 65536, + "beta_fast": 32, + "beta_slow": 1 + }, + "topk": 512, + "index_heads": 32, + "index_dim": 128, + "proj": { + "indexer.wq_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + }, + "wkv": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + }, + "wo_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + }, + "wq_a": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + }, + "wq_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + } + }, + "batch": { + "groups": [ + { + "count": 1, + "q": 32768, + "ctx": 0 + } + ] + } + }, + "sources": [ + "deepseek-ai/DeepSeek-V4.1-Flash/attn" + ] + }, + { + "type": "dsv4_attn", + "args": { + "hidden": 5120, + "heads": 64, + "head_dim": 512, + "rope_dim": 64, + "q_lora_rank": 1280, + "window": 128, + "o_groups": 8, + "o_lora_rank": 1024, + "compress_ratio": 1, + "source": false, + "indexer": "shared", + "compress_rope_theta": 160000, + "rope_scaling": { + "type": "yarn", + "factor": 16, + "original_max": 65536, + "beta_fast": 32, + "beta_slow": 1 + }, + "topk": 512, + "index_heads": 32, + "index_dim": 128, + "proj": { + "indexer.wq_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + }, + "wkv": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + }, + "wo_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + }, + "wq_a": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + }, + "wq_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + } + }, + "batch": { + "groups": [ + { + "count": 1, + "q": 8192, + "ctx": 8192 + } + ] + } + }, + "sources": [ + "deepseek-ai/DeepSeek-V4.1-Flash/attn" + ] + }, + { + "type": "dsv4_attn", + "args": { + "hidden": 5120, + "heads": 64, + "head_dim": 512, + "rope_dim": 64, + "q_lora_rank": 1280, + "window": 128, + "o_groups": 8, + "o_lora_rank": 1024, + "compress_ratio": 1, + "source": false, + "indexer": "shared", + "compress_rope_theta": 160000, + "rope_scaling": { + "type": "yarn", + "factor": 16, + "original_max": 65536, + "beta_fast": 32, + "beta_slow": 1 + }, + "topk": 512, + "index_heads": 32, + "index_dim": 128, + "proj": { + "indexer.wq_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + }, + "wkv": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + }, + "wo_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + }, + "wq_a": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + }, + "wq_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + } + }, + "batch": { + "groups": [ + { + "count": 1, + "q": 8192, + "ctx": 32768 + } + ] + } + }, + "sources": [ + "deepseek-ai/DeepSeek-V4.1-Flash/attn" + ] + }, + { + "type": "dsv4_attn", + "args": { + "hidden": 5120, + "heads": 64, + "head_dim": 512, + "rope_dim": 64, + "q_lora_rank": 1280, + "window": 128, + "o_groups": 8, + "o_lora_rank": 1024, + "compress_ratio": 1, + "source": false, + "indexer": "shared", + "compress_rope_theta": 160000, + "rope_scaling": { + "type": "yarn", + "factor": 16, + "original_max": 65536, + "beta_fast": 32, + "beta_slow": 1 + }, + "topk": 512, + "index_heads": 32, + "index_dim": 128, + "proj": { + "indexer.wq_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + }, + "wkv": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + }, + "wo_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + }, + "wq_a": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + }, + "wq_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + } + }, + "batch": { + "groups": [ + { + "count": 1, + "q": 8192, + "ctx": 131072 + } + ] + } + }, + "sources": [ + "deepseek-ai/DeepSeek-V4.1-Flash/attn" + ] + }, + { + "type": "dsv4_attn", + "args": { + "hidden": 5120, + "heads": 64, + "head_dim": 512, + "rope_dim": 64, + "q_lora_rank": 1280, + "window": 128, + "o_groups": 8, + "o_lora_rank": 1024, + "compress_ratio": 1, + "source": false, + "indexer": "shared", + "compress_rope_theta": 160000, + "rope_scaling": { + "type": "yarn", + "factor": 16, + "original_max": 65536, + "beta_fast": 32, + "beta_slow": 1 + }, + "topk": 512, + "index_heads": 32, + "index_dim": 128, + "proj": { + "indexer.wq_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + }, + "wkv": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + }, + "wo_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + }, + "wq_a": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + }, + "wq_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + } + }, + "batch": { + "groups": [ + { + "count": 1, + "q": 8192, + "ctx": 8192 + }, + { + "count": 64, + "q": 1, + "ctx": 8192 + } + ] + } + }, + "sources": [ + "deepseek-ai/DeepSeek-V4.1-Flash/attn" + ] + } +] \ No newline at end of file diff --git a/operatorx/testlists/attn_kimi_k3.json b/operatorx/testlists/attn_kimi_k3.json new file mode 100644 index 0000000000..23360eb0bf --- /dev/null +++ b/operatorx/testlists/attn_kimi_k3.json @@ -0,0 +1,2360 @@ +[ + { + "type": "mla", + "args": { + "hidden": 7168, + "heads": 96, + "q_lora_rank": 1536, + "kv_lora_rank": 512, + "nope": 128, + "rope_dim": 64, + "v": 128, + "rope": false, + "gate": true, + "kv_cache_dtype": "fp8", + "batch": { + "groups": [ + { + "count": 1, + "q": 1, + "ctx": 1024 + } + ] + } + }, + "sources": [ + "moonshotai/Kimi-K3/self_attn" + ] + }, + { + "type": "mla", + "args": { + "hidden": 7168, + "heads": 96, + "q_lora_rank": 1536, + "kv_lora_rank": 512, + "nope": 128, + "rope_dim": 64, + "v": 128, + "rope": false, + "gate": true, + "kv_cache_dtype": "fp8", + "batch": { + "groups": [ + { + "count": 1, + "q": 1, + "ctx": 4096 + } + ] + } + }, + "sources": [ + "moonshotai/Kimi-K3/self_attn" + ] + }, + { + "type": "mla", + "args": { + "hidden": 7168, + "heads": 96, + "q_lora_rank": 1536, + "kv_lora_rank": 512, + "nope": 128, + "rope_dim": 64, + "v": 128, + "rope": false, + "gate": true, + "kv_cache_dtype": "fp8", + "batch": { + "groups": [ + { + "count": 1, + "q": 1, + "ctx": 16384 + } + ] + } + }, + "sources": [ + "moonshotai/Kimi-K3/self_attn" + ] + }, + { + "type": "mla", + "args": { + "hidden": 7168, + "heads": 96, + "q_lora_rank": 1536, + "kv_lora_rank": 512, + "nope": 128, + "rope_dim": 64, + "v": 128, + "rope": false, + "gate": true, + "kv_cache_dtype": "fp8", + "batch": { + "groups": [ + { + "count": 1, + "q": 1, + "ctx": 65536 + } + ] + } + }, + "sources": [ + "moonshotai/Kimi-K3/self_attn" + ] + }, + { + "type": "mla", + "args": { + "hidden": 7168, + "heads": 96, + "q_lora_rank": 1536, + "kv_lora_rank": 512, + "nope": 128, + "rope_dim": 64, + "v": 128, + "rope": false, + "gate": true, + "kv_cache_dtype": "fp8", + "batch": { + "groups": [ + { + "count": 1, + "q": 1, + "ctx": 131072 + } + ] + } + }, + "sources": [ + "moonshotai/Kimi-K3/self_attn" + ] + }, + { + "type": "mla", + "args": { + "hidden": 7168, + "heads": 96, + "q_lora_rank": 1536, + "kv_lora_rank": 512, + "nope": 128, + "rope_dim": 64, + "v": 128, + "rope": false, + "gate": true, + "kv_cache_dtype": "fp8", + "batch": { + "groups": [ + { + "count": 4, + "q": 1, + "ctx": 1024 + } + ] + } + }, + "sources": [ + "moonshotai/Kimi-K3/self_attn" + ] + }, + { + "type": "mla", + "args": { + "hidden": 7168, + "heads": 96, + "q_lora_rank": 1536, + "kv_lora_rank": 512, + "nope": 128, + "rope_dim": 64, + "v": 128, + "rope": false, + "gate": true, + "kv_cache_dtype": "fp8", + "batch": { + "groups": [ + { + "count": 4, + "q": 1, + "ctx": 4096 + } + ] + } + }, + "sources": [ + "moonshotai/Kimi-K3/self_attn" + ] + }, + { + "type": "mla", + "args": { + "hidden": 7168, + "heads": 96, + "q_lora_rank": 1536, + "kv_lora_rank": 512, + "nope": 128, + "rope_dim": 64, + "v": 128, + "rope": false, + "gate": true, + "kv_cache_dtype": "fp8", + "batch": { + "groups": [ + { + "count": 4, + "q": 1, + "ctx": 16384 + } + ] + } + }, + "sources": [ + "moonshotai/Kimi-K3/self_attn" + ] + }, + { + "type": "mla", + "args": { + "hidden": 7168, + "heads": 96, + "q_lora_rank": 1536, + "kv_lora_rank": 512, + "nope": 128, + "rope_dim": 64, + "v": 128, + "rope": false, + "gate": true, + "kv_cache_dtype": "fp8", + "batch": { + "groups": [ + { + "count": 4, + "q": 1, + "ctx": 65536 + } + ] + } + }, + "sources": [ + "moonshotai/Kimi-K3/self_attn" + ] + }, + { + "type": "mla", + "args": { + "hidden": 7168, + "heads": 96, + "q_lora_rank": 1536, + "kv_lora_rank": 512, + "nope": 128, + "rope_dim": 64, + "v": 128, + "rope": false, + "gate": true, + "kv_cache_dtype": "fp8", + "batch": { + "groups": [ + { + "count": 4, + "q": 1, + "ctx": 131072 + } + ] + } + }, + "sources": [ + "moonshotai/Kimi-K3/self_attn" + ] + }, + { + "type": "mla", + "args": { + "hidden": 7168, + "heads": 96, + "q_lora_rank": 1536, + "kv_lora_rank": 512, + "nope": 128, + "rope_dim": 64, + "v": 128, + "rope": false, + "gate": true, + "kv_cache_dtype": "fp8", + "batch": { + "groups": [ + { + "count": 16, + "q": 1, + "ctx": 1024 + } + ] + } + }, + "sources": [ + "moonshotai/Kimi-K3/self_attn" + ] + }, + { + "type": "mla", + "args": { + "hidden": 7168, + "heads": 96, + "q_lora_rank": 1536, + "kv_lora_rank": 512, + "nope": 128, + "rope_dim": 64, + "v": 128, + "rope": false, + "gate": true, + "kv_cache_dtype": "fp8", + "batch": { + "groups": [ + { + "count": 16, + "q": 1, + "ctx": 4096 + } + ] + } + }, + "sources": [ + "moonshotai/Kimi-K3/self_attn" + ] + }, + { + "type": "mla", + "args": { + "hidden": 7168, + "heads": 96, + "q_lora_rank": 1536, + "kv_lora_rank": 512, + "nope": 128, + "rope_dim": 64, + "v": 128, + "rope": false, + "gate": true, + "kv_cache_dtype": "fp8", + "batch": { + "groups": [ + { + "count": 16, + "q": 1, + "ctx": 16384 + } + ] + } + }, + "sources": [ + "moonshotai/Kimi-K3/self_attn" + ] + }, + { + "type": "mla", + "args": { + "hidden": 7168, + "heads": 96, + "q_lora_rank": 1536, + "kv_lora_rank": 512, + "nope": 128, + "rope_dim": 64, + "v": 128, + "rope": false, + "gate": true, + "kv_cache_dtype": "fp8", + "batch": { + "groups": [ + { + "count": 16, + "q": 1, + "ctx": 65536 + } + ] + } + }, + "sources": [ + "moonshotai/Kimi-K3/self_attn" + ] + }, + { + "type": "mla", + "args": { + "hidden": 7168, + "heads": 96, + "q_lora_rank": 1536, + "kv_lora_rank": 512, + "nope": 128, + "rope_dim": 64, + "v": 128, + "rope": false, + "gate": true, + "kv_cache_dtype": "fp8", + "batch": { + "groups": [ + { + "count": 16, + "q": 1, + "ctx": 131072 + } + ] + } + }, + "sources": [ + "moonshotai/Kimi-K3/self_attn" + ] + }, + { + "type": "mla", + "args": { + "hidden": 7168, + "heads": 96, + "q_lora_rank": 1536, + "kv_lora_rank": 512, + "nope": 128, + "rope_dim": 64, + "v": 128, + "rope": false, + "gate": true, + "kv_cache_dtype": "fp8", + "batch": { + "groups": [ + { + "count": 64, + "q": 1, + "ctx": 1024 + } + ] + } + }, + "sources": [ + "moonshotai/Kimi-K3/self_attn" + ] + }, + { + "type": "mla", + "args": { + "hidden": 7168, + "heads": 96, + "q_lora_rank": 1536, + "kv_lora_rank": 512, + "nope": 128, + "rope_dim": 64, + "v": 128, + "rope": false, + "gate": true, + "kv_cache_dtype": "fp8", + "batch": { + "groups": [ + { + "count": 64, + "q": 1, + "ctx": 4096 + } + ] + } + }, + "sources": [ + "moonshotai/Kimi-K3/self_attn" + ] + }, + { + "type": "mla", + "args": { + "hidden": 7168, + "heads": 96, + "q_lora_rank": 1536, + "kv_lora_rank": 512, + "nope": 128, + "rope_dim": 64, + "v": 128, + "rope": false, + "gate": true, + "kv_cache_dtype": "fp8", + "batch": { + "groups": [ + { + "count": 64, + "q": 1, + "ctx": 16384 + } + ] + } + }, + "sources": [ + "moonshotai/Kimi-K3/self_attn" + ] + }, + { + "type": "mla", + "args": { + "hidden": 7168, + "heads": 96, + "q_lora_rank": 1536, + "kv_lora_rank": 512, + "nope": 128, + "rope_dim": 64, + "v": 128, + "rope": false, + "gate": true, + "kv_cache_dtype": "fp8", + "batch": { + "groups": [ + { + "count": 64, + "q": 1, + "ctx": 65536 + } + ] + } + }, + "sources": [ + "moonshotai/Kimi-K3/self_attn" + ] + }, + { + "type": "mla", + "args": { + "hidden": 7168, + "heads": 96, + "q_lora_rank": 1536, + "kv_lora_rank": 512, + "nope": 128, + "rope_dim": 64, + "v": 128, + "rope": false, + "gate": true, + "kv_cache_dtype": "fp8", + "batch": { + "groups": [ + { + "count": 64, + "q": 1, + "ctx": 131072 + } + ] + } + }, + "sources": [ + "moonshotai/Kimi-K3/self_attn" + ] + }, + { + "type": "mla", + "args": { + "hidden": 7168, + "heads": 96, + "q_lora_rank": 1536, + "kv_lora_rank": 512, + "nope": 128, + "rope_dim": 64, + "v": 128, + "rope": false, + "gate": true, + "kv_cache_dtype": "fp8", + "batch": { + "groups": [ + { + "count": 256, + "q": 1, + "ctx": 1024 + } + ] + } + }, + "sources": [ + "moonshotai/Kimi-K3/self_attn" + ] + }, + { + "type": "mla", + "args": { + "hidden": 7168, + "heads": 96, + "q_lora_rank": 1536, + "kv_lora_rank": 512, + "nope": 128, + "rope_dim": 64, + "v": 128, + "rope": false, + "gate": true, + "kv_cache_dtype": "fp8", + "batch": { + "groups": [ + { + "count": 256, + "q": 1, + "ctx": 4096 + } + ] + } + }, + "sources": [ + "moonshotai/Kimi-K3/self_attn" + ] + }, + { + "type": "mla", + "args": { + "hidden": 7168, + "heads": 96, + "q_lora_rank": 1536, + "kv_lora_rank": 512, + "nope": 128, + "rope_dim": 64, + "v": 128, + "rope": false, + "gate": true, + "kv_cache_dtype": "fp8", + "batch": { + "groups": [ + { + "count": 256, + "q": 1, + "ctx": 16384 + } + ] + } + }, + "sources": [ + "moonshotai/Kimi-K3/self_attn" + ] + }, + { + "type": "mla", + "args": { + "hidden": 7168, + "heads": 96, + "q_lora_rank": 1536, + "kv_lora_rank": 512, + "nope": 128, + "rope_dim": 64, + "v": 128, + "rope": false, + "gate": true, + "kv_cache_dtype": "fp8", + "batch": { + "groups": [ + { + "count": 256, + "q": 1, + "ctx": 65536 + } + ] + } + }, + "sources": [ + "moonshotai/Kimi-K3/self_attn" + ] + }, + { + "type": "mla", + "args": { + "hidden": 7168, + "heads": 96, + "q_lora_rank": 1536, + "kv_lora_rank": 512, + "nope": 128, + "rope_dim": 64, + "v": 128, + "rope": false, + "gate": true, + "kv_cache_dtype": "fp8", + "batch": { + "groups": [ + { + "count": 256, + "q": 1, + "ctx": 131072 + } + ] + } + }, + "sources": [ + "moonshotai/Kimi-K3/self_attn" + ] + }, + { + "type": "mla", + "args": { + "hidden": 7168, + "heads": 96, + "q_lora_rank": 1536, + "kv_lora_rank": 512, + "nope": 128, + "rope_dim": 64, + "v": 128, + "rope": false, + "gate": true, + "kv_cache_dtype": "fp8", + "batch": { + "groups": [ + { + "count": 1, + "q": 1024, + "ctx": 0 + } + ] + } + }, + "sources": [ + "moonshotai/Kimi-K3/self_attn" + ] + }, + { + "type": "mla", + "args": { + "hidden": 7168, + "heads": 96, + "q_lora_rank": 1536, + "kv_lora_rank": 512, + "nope": 128, + "rope_dim": 64, + "v": 128, + "rope": false, + "gate": true, + "kv_cache_dtype": "fp8", + "batch": { + "groups": [ + { + "count": 1, + "q": 4096, + "ctx": 0 + } + ] + } + }, + "sources": [ + "moonshotai/Kimi-K3/self_attn" + ] + }, + { + "type": "mla", + "args": { + "hidden": 7168, + "heads": 96, + "q_lora_rank": 1536, + "kv_lora_rank": 512, + "nope": 128, + "rope_dim": 64, + "v": 128, + "rope": false, + "gate": true, + "kv_cache_dtype": "fp8", + "batch": { + "groups": [ + { + "count": 1, + "q": 16384, + "ctx": 0 + } + ] + } + }, + "sources": [ + "moonshotai/Kimi-K3/self_attn" + ] + }, + { + "type": "mla", + "args": { + "hidden": 7168, + "heads": 96, + "q_lora_rank": 1536, + "kv_lora_rank": 512, + "nope": 128, + "rope_dim": 64, + "v": 128, + "rope": false, + "gate": true, + "kv_cache_dtype": "fp8", + "batch": { + "groups": [ + { + "count": 1, + "q": 32768, + "ctx": 0 + } + ] + } + }, + "sources": [ + "moonshotai/Kimi-K3/self_attn" + ] + }, + { + "type": "mla", + "args": { + "hidden": 7168, + "heads": 96, + "q_lora_rank": 1536, + "kv_lora_rank": 512, + "nope": 128, + "rope_dim": 64, + "v": 128, + "rope": false, + "gate": true, + "kv_cache_dtype": "fp8", + "batch": { + "groups": [ + { + "count": 1, + "q": 8192, + "ctx": 8192 + } + ] + } + }, + "sources": [ + "moonshotai/Kimi-K3/self_attn" + ] + }, + { + "type": "mla", + "args": { + "hidden": 7168, + "heads": 96, + "q_lora_rank": 1536, + "kv_lora_rank": 512, + "nope": 128, + "rope_dim": 64, + "v": 128, + "rope": false, + "gate": true, + "kv_cache_dtype": "fp8", + "batch": { + "groups": [ + { + "count": 1, + "q": 8192, + "ctx": 32768 + } + ] + } + }, + "sources": [ + "moonshotai/Kimi-K3/self_attn" + ] + }, + { + "type": "mla", + "args": { + "hidden": 7168, + "heads": 96, + "q_lora_rank": 1536, + "kv_lora_rank": 512, + "nope": 128, + "rope_dim": 64, + "v": 128, + "rope": false, + "gate": true, + "kv_cache_dtype": "fp8", + "batch": { + "groups": [ + { + "count": 1, + "q": 8192, + "ctx": 131072 + } + ] + } + }, + "sources": [ + "moonshotai/Kimi-K3/self_attn" + ] + }, + { + "type": "mla", + "args": { + "hidden": 7168, + "heads": 96, + "q_lora_rank": 1536, + "kv_lora_rank": 512, + "nope": 128, + "rope_dim": 64, + "v": 128, + "rope": false, + "gate": true, + "kv_cache_dtype": "fp8", + "batch": { + "groups": [ + { + "count": 1, + "q": 8192, + "ctx": 8192 + }, + { + "count": 64, + "q": 1, + "ctx": 8192 + } + ] + } + }, + "sources": [ + "moonshotai/Kimi-K3/self_attn" + ] + }, + { + "type": "kda", + "args": { + "hidden": 7168, + "heads": 96, + "head_dim": 128, + "conv_kernel": 4, + "state_dtype": "fp32", + "batch": { + "groups": [ + { + "count": 1, + "q": 1, + "ctx": 1024 + } + ] + } + }, + "sources": [ + "moonshotai/Kimi-K3/self_attn" + ] + }, + { + "type": "kda", + "args": { + "hidden": 7168, + "heads": 96, + "head_dim": 128, + "conv_kernel": 4, + "state_dtype": "fp32", + "batch": { + "groups": [ + { + "count": 1, + "q": 1, + "ctx": 4096 + } + ] + } + }, + "sources": [ + "moonshotai/Kimi-K3/self_attn" + ] + }, + { + "type": "kda", + "args": { + "hidden": 7168, + "heads": 96, + "head_dim": 128, + "conv_kernel": 4, + "state_dtype": "fp32", + "batch": { + "groups": [ + { + "count": 1, + "q": 1, + "ctx": 16384 + } + ] + } + }, + "sources": [ + "moonshotai/Kimi-K3/self_attn" + ] + }, + { + "type": "kda", + "args": { + "hidden": 7168, + "heads": 96, + "head_dim": 128, + "conv_kernel": 4, + "state_dtype": "fp32", + "batch": { + "groups": [ + { + "count": 1, + "q": 1, + "ctx": 65536 + } + ] + } + }, + "sources": [ + "moonshotai/Kimi-K3/self_attn" + ] + }, + { + "type": "kda", + "args": { + "hidden": 7168, + "heads": 96, + "head_dim": 128, + "conv_kernel": 4, + "state_dtype": "fp32", + "batch": { + "groups": [ + { + "count": 1, + "q": 1, + "ctx": 131072 + } + ] + } + }, + "sources": [ + "moonshotai/Kimi-K3/self_attn" + ] + }, + { + "type": "kda", + "args": { + "hidden": 7168, + "heads": 96, + "head_dim": 128, + "conv_kernel": 4, + "state_dtype": "fp32", + "batch": { + "groups": [ + { + "count": 4, + "q": 1, + "ctx": 1024 + } + ] + } + }, + "sources": [ + "moonshotai/Kimi-K3/self_attn" + ] + }, + { + "type": "kda", + "args": { + "hidden": 7168, + "heads": 96, + "head_dim": 128, + "conv_kernel": 4, + "state_dtype": "fp32", + "batch": { + "groups": [ + { + "count": 4, + "q": 1, + "ctx": 4096 + } + ] + } + }, + "sources": [ + "moonshotai/Kimi-K3/self_attn" + ] + }, + { + "type": "kda", + "args": { + "hidden": 7168, + "heads": 96, + "head_dim": 128, + "conv_kernel": 4, + "state_dtype": "fp32", + "batch": { + "groups": [ + { + "count": 4, + "q": 1, + "ctx": 16384 + } + ] + } + }, + "sources": [ + "moonshotai/Kimi-K3/self_attn" + ] + }, + { + "type": "kda", + "args": { + "hidden": 7168, + "heads": 96, + "head_dim": 128, + "conv_kernel": 4, + "state_dtype": "fp32", + "batch": { + "groups": [ + { + "count": 4, + "q": 1, + "ctx": 65536 + } + ] + } + }, + "sources": [ + "moonshotai/Kimi-K3/self_attn" + ] + }, + { + "type": "kda", + "args": { + "hidden": 7168, + "heads": 96, + "head_dim": 128, + "conv_kernel": 4, + "state_dtype": "fp32", + "batch": { + "groups": [ + { + "count": 4, + "q": 1, + "ctx": 131072 + } + ] + } + }, + "sources": [ + "moonshotai/Kimi-K3/self_attn" + ] + }, + { + "type": "kda", + "args": { + "hidden": 7168, + "heads": 96, + "head_dim": 128, + "conv_kernel": 4, + "state_dtype": "fp32", + "batch": { + "groups": [ + { + "count": 16, + "q": 1, + "ctx": 1024 + } + ] + } + }, + "sources": [ + "moonshotai/Kimi-K3/self_attn" + ] + }, + { + "type": "kda", + "args": { + "hidden": 7168, + "heads": 96, + "head_dim": 128, + "conv_kernel": 4, + "state_dtype": "fp32", + "batch": { + "groups": [ + { + "count": 16, + "q": 1, + "ctx": 4096 + } + ] + } + }, + "sources": [ + "moonshotai/Kimi-K3/self_attn" + ] + }, + { + "type": "kda", + "args": { + "hidden": 7168, + "heads": 96, + "head_dim": 128, + "conv_kernel": 4, + "state_dtype": "fp32", + "batch": { + "groups": [ + { + "count": 16, + "q": 1, + "ctx": 16384 + } + ] + } + }, + "sources": [ + "moonshotai/Kimi-K3/self_attn" + ] + }, + { + "type": "kda", + "args": { + "hidden": 7168, + "heads": 96, + "head_dim": 128, + "conv_kernel": 4, + "state_dtype": "fp32", + "batch": { + "groups": [ + { + "count": 16, + "q": 1, + "ctx": 65536 + } + ] + } + }, + "sources": [ + "moonshotai/Kimi-K3/self_attn" + ] + }, + { + "type": "kda", + "args": { + "hidden": 7168, + "heads": 96, + "head_dim": 128, + "conv_kernel": 4, + "state_dtype": "fp32", + "batch": { + "groups": [ + { + "count": 16, + "q": 1, + "ctx": 131072 + } + ] + } + }, + "sources": [ + "moonshotai/Kimi-K3/self_attn" + ] + }, + { + "type": "kda", + "args": { + "hidden": 7168, + "heads": 96, + "head_dim": 128, + "conv_kernel": 4, + "state_dtype": "fp32", + "batch": { + "groups": [ + { + "count": 64, + "q": 1, + "ctx": 1024 + } + ] + } + }, + "sources": [ + "moonshotai/Kimi-K3/self_attn" + ] + }, + { + "type": "kda", + "args": { + "hidden": 7168, + "heads": 96, + "head_dim": 128, + "conv_kernel": 4, + "state_dtype": "fp32", + "batch": { + "groups": [ + { + "count": 64, + "q": 1, + "ctx": 4096 + } + ] + } + }, + "sources": [ + "moonshotai/Kimi-K3/self_attn" + ] + }, + { + "type": "kda", + "args": { + "hidden": 7168, + "heads": 96, + "head_dim": 128, + "conv_kernel": 4, + "state_dtype": "fp32", + "batch": { + "groups": [ + { + "count": 64, + "q": 1, + "ctx": 16384 + } + ] + } + }, + "sources": [ + "moonshotai/Kimi-K3/self_attn" + ] + }, + { + "type": "kda", + "args": { + "hidden": 7168, + "heads": 96, + "head_dim": 128, + "conv_kernel": 4, + "state_dtype": "fp32", + "batch": { + "groups": [ + { + "count": 64, + "q": 1, + "ctx": 65536 + } + ] + } + }, + "sources": [ + "moonshotai/Kimi-K3/self_attn" + ] + }, + { + "type": "kda", + "args": { + "hidden": 7168, + "heads": 96, + "head_dim": 128, + "conv_kernel": 4, + "state_dtype": "fp32", + "batch": { + "groups": [ + { + "count": 64, + "q": 1, + "ctx": 131072 + } + ] + } + }, + "sources": [ + "moonshotai/Kimi-K3/self_attn" + ] + }, + { + "type": "kda", + "args": { + "hidden": 7168, + "heads": 96, + "head_dim": 128, + "conv_kernel": 4, + "state_dtype": "fp32", + "batch": { + "groups": [ + { + "count": 256, + "q": 1, + "ctx": 1024 + } + ] + } + }, + "sources": [ + "moonshotai/Kimi-K3/self_attn" + ] + }, + { + "type": "kda", + "args": { + "hidden": 7168, + "heads": 96, + "head_dim": 128, + "conv_kernel": 4, + "state_dtype": "fp32", + "batch": { + "groups": [ + { + "count": 256, + "q": 1, + "ctx": 4096 + } + ] + } + }, + "sources": [ + "moonshotai/Kimi-K3/self_attn" + ] + }, + { + "type": "kda", + "args": { + "hidden": 7168, + "heads": 96, + "head_dim": 128, + "conv_kernel": 4, + "state_dtype": "fp32", + "batch": { + "groups": [ + { + "count": 256, + "q": 1, + "ctx": 16384 + } + ] + } + }, + "sources": [ + "moonshotai/Kimi-K3/self_attn" + ] + }, + { + "type": "kda", + "args": { + "hidden": 7168, + "heads": 96, + "head_dim": 128, + "conv_kernel": 4, + "state_dtype": "fp32", + "batch": { + "groups": [ + { + "count": 256, + "q": 1, + "ctx": 65536 + } + ] + } + }, + "sources": [ + "moonshotai/Kimi-K3/self_attn" + ] + }, + { + "type": "kda", + "args": { + "hidden": 7168, + "heads": 96, + "head_dim": 128, + "conv_kernel": 4, + "state_dtype": "fp32", + "batch": { + "groups": [ + { + "count": 256, + "q": 1, + "ctx": 131072 + } + ] + } + }, + "sources": [ + "moonshotai/Kimi-K3/self_attn" + ] + }, + { + "type": "kda", + "args": { + "hidden": 7168, + "heads": 96, + "head_dim": 128, + "conv_kernel": 4, + "state_dtype": "fp32", + "batch": { + "groups": [ + { + "count": 1, + "q": 1024, + "ctx": 0 + } + ] + } + }, + "sources": [ + "moonshotai/Kimi-K3/self_attn" + ] + }, + { + "type": "kda", + "args": { + "hidden": 7168, + "heads": 96, + "head_dim": 128, + "conv_kernel": 4, + "state_dtype": "fp32", + "batch": { + "groups": [ + { + "count": 1, + "q": 4096, + "ctx": 0 + } + ] + } + }, + "sources": [ + "moonshotai/Kimi-K3/self_attn" + ] + }, + { + "type": "kda", + "args": { + "hidden": 7168, + "heads": 96, + "head_dim": 128, + "conv_kernel": 4, + "state_dtype": "fp32", + "batch": { + "groups": [ + { + "count": 1, + "q": 16384, + "ctx": 0 + } + ] + } + }, + "sources": [ + "moonshotai/Kimi-K3/self_attn" + ] + }, + { + "type": "kda", + "args": { + "hidden": 7168, + "heads": 96, + "head_dim": 128, + "conv_kernel": 4, + "state_dtype": "fp32", + "batch": { + "groups": [ + { + "count": 1, + "q": 32768, + "ctx": 0 + } + ] + } + }, + "sources": [ + "moonshotai/Kimi-K3/self_attn" + ] + }, + { + "type": "kda", + "args": { + "hidden": 7168, + "heads": 96, + "head_dim": 128, + "conv_kernel": 4, + "state_dtype": "fp32", + "batch": { + "groups": [ + { + "count": 1, + "q": 8192, + "ctx": 8192 + } + ] + } + }, + "sources": [ + "moonshotai/Kimi-K3/self_attn" + ] + }, + { + "type": "kda", + "args": { + "hidden": 7168, + "heads": 96, + "head_dim": 128, + "conv_kernel": 4, + "state_dtype": "fp32", + "batch": { + "groups": [ + { + "count": 1, + "q": 8192, + "ctx": 32768 + } + ] + } + }, + "sources": [ + "moonshotai/Kimi-K3/self_attn" + ] + }, + { + "type": "kda", + "args": { + "hidden": 7168, + "heads": 96, + "head_dim": 128, + "conv_kernel": 4, + "state_dtype": "fp32", + "batch": { + "groups": [ + { + "count": 1, + "q": 8192, + "ctx": 131072 + } + ] + } + }, + "sources": [ + "moonshotai/Kimi-K3/self_attn" + ] + }, + { + "type": "kda", + "args": { + "hidden": 7168, + "heads": 96, + "head_dim": 128, + "conv_kernel": 4, + "state_dtype": "fp32", + "batch": { + "groups": [ + { + "count": 1, + "q": 8192, + "ctx": 8192 + }, + { + "count": 64, + "q": 1, + "ctx": 8192 + } + ] + } + }, + "sources": [ + "moonshotai/Kimi-K3/self_attn" + ] + }, + { + "type": "kda", + "args": { + "hidden": 7168, + "heads": 96, + "head_dim": 128, + "conv_kernel": 4, + "state_dtype": "bf16", + "batch": { + "groups": [ + { + "count": 1, + "q": 1, + "ctx": 1024 + } + ] + } + }, + "sources": [ + "moonshotai/Kimi-K3/self_attn" + ] + }, + { + "type": "kda", + "args": { + "hidden": 7168, + "heads": 96, + "head_dim": 128, + "conv_kernel": 4, + "state_dtype": "bf16", + "batch": { + "groups": [ + { + "count": 1, + "q": 1, + "ctx": 4096 + } + ] + } + }, + "sources": [ + "moonshotai/Kimi-K3/self_attn" + ] + }, + { + "type": "kda", + "args": { + "hidden": 7168, + "heads": 96, + "head_dim": 128, + "conv_kernel": 4, + "state_dtype": "bf16", + "batch": { + "groups": [ + { + "count": 1, + "q": 1, + "ctx": 16384 + } + ] + } + }, + "sources": [ + "moonshotai/Kimi-K3/self_attn" + ] + }, + { + "type": "kda", + "args": { + "hidden": 7168, + "heads": 96, + "head_dim": 128, + "conv_kernel": 4, + "state_dtype": "bf16", + "batch": { + "groups": [ + { + "count": 1, + "q": 1, + "ctx": 65536 + } + ] + } + }, + "sources": [ + "moonshotai/Kimi-K3/self_attn" + ] + }, + { + "type": "kda", + "args": { + "hidden": 7168, + "heads": 96, + "head_dim": 128, + "conv_kernel": 4, + "state_dtype": "bf16", + "batch": { + "groups": [ + { + "count": 1, + "q": 1, + "ctx": 131072 + } + ] + } + }, + "sources": [ + "moonshotai/Kimi-K3/self_attn" + ] + }, + { + "type": "kda", + "args": { + "hidden": 7168, + "heads": 96, + "head_dim": 128, + "conv_kernel": 4, + "state_dtype": "bf16", + "batch": { + "groups": [ + { + "count": 4, + "q": 1, + "ctx": 1024 + } + ] + } + }, + "sources": [ + "moonshotai/Kimi-K3/self_attn" + ] + }, + { + "type": "kda", + "args": { + "hidden": 7168, + "heads": 96, + "head_dim": 128, + "conv_kernel": 4, + "state_dtype": "bf16", + "batch": { + "groups": [ + { + "count": 4, + "q": 1, + "ctx": 4096 + } + ] + } + }, + "sources": [ + "moonshotai/Kimi-K3/self_attn" + ] + }, + { + "type": "kda", + "args": { + "hidden": 7168, + "heads": 96, + "head_dim": 128, + "conv_kernel": 4, + "state_dtype": "bf16", + "batch": { + "groups": [ + { + "count": 4, + "q": 1, + "ctx": 16384 + } + ] + } + }, + "sources": [ + "moonshotai/Kimi-K3/self_attn" + ] + }, + { + "type": "kda", + "args": { + "hidden": 7168, + "heads": 96, + "head_dim": 128, + "conv_kernel": 4, + "state_dtype": "bf16", + "batch": { + "groups": [ + { + "count": 4, + "q": 1, + "ctx": 65536 + } + ] + } + }, + "sources": [ + "moonshotai/Kimi-K3/self_attn" + ] + }, + { + "type": "kda", + "args": { + "hidden": 7168, + "heads": 96, + "head_dim": 128, + "conv_kernel": 4, + "state_dtype": "bf16", + "batch": { + "groups": [ + { + "count": 4, + "q": 1, + "ctx": 131072 + } + ] + } + }, + "sources": [ + "moonshotai/Kimi-K3/self_attn" + ] + }, + { + "type": "kda", + "args": { + "hidden": 7168, + "heads": 96, + "head_dim": 128, + "conv_kernel": 4, + "state_dtype": "bf16", + "batch": { + "groups": [ + { + "count": 16, + "q": 1, + "ctx": 1024 + } + ] + } + }, + "sources": [ + "moonshotai/Kimi-K3/self_attn" + ] + }, + { + "type": "kda", + "args": { + "hidden": 7168, + "heads": 96, + "head_dim": 128, + "conv_kernel": 4, + "state_dtype": "bf16", + "batch": { + "groups": [ + { + "count": 16, + "q": 1, + "ctx": 4096 + } + ] + } + }, + "sources": [ + "moonshotai/Kimi-K3/self_attn" + ] + }, + { + "type": "kda", + "args": { + "hidden": 7168, + "heads": 96, + "head_dim": 128, + "conv_kernel": 4, + "state_dtype": "bf16", + "batch": { + "groups": [ + { + "count": 16, + "q": 1, + "ctx": 16384 + } + ] + } + }, + "sources": [ + "moonshotai/Kimi-K3/self_attn" + ] + }, + { + "type": "kda", + "args": { + "hidden": 7168, + "heads": 96, + "head_dim": 128, + "conv_kernel": 4, + "state_dtype": "bf16", + "batch": { + "groups": [ + { + "count": 16, + "q": 1, + "ctx": 65536 + } + ] + } + }, + "sources": [ + "moonshotai/Kimi-K3/self_attn" + ] + }, + { + "type": "kda", + "args": { + "hidden": 7168, + "heads": 96, + "head_dim": 128, + "conv_kernel": 4, + "state_dtype": "bf16", + "batch": { + "groups": [ + { + "count": 16, + "q": 1, + "ctx": 131072 + } + ] + } + }, + "sources": [ + "moonshotai/Kimi-K3/self_attn" + ] + }, + { + "type": "kda", + "args": { + "hidden": 7168, + "heads": 96, + "head_dim": 128, + "conv_kernel": 4, + "state_dtype": "bf16", + "batch": { + "groups": [ + { + "count": 64, + "q": 1, + "ctx": 1024 + } + ] + } + }, + "sources": [ + "moonshotai/Kimi-K3/self_attn" + ] + }, + { + "type": "kda", + "args": { + "hidden": 7168, + "heads": 96, + "head_dim": 128, + "conv_kernel": 4, + "state_dtype": "bf16", + "batch": { + "groups": [ + { + "count": 64, + "q": 1, + "ctx": 4096 + } + ] + } + }, + "sources": [ + "moonshotai/Kimi-K3/self_attn" + ] + }, + { + "type": "kda", + "args": { + "hidden": 7168, + "heads": 96, + "head_dim": 128, + "conv_kernel": 4, + "state_dtype": "bf16", + "batch": { + "groups": [ + { + "count": 64, + "q": 1, + "ctx": 16384 + } + ] + } + }, + "sources": [ + "moonshotai/Kimi-K3/self_attn" + ] + }, + { + "type": "kda", + "args": { + "hidden": 7168, + "heads": 96, + "head_dim": 128, + "conv_kernel": 4, + "state_dtype": "bf16", + "batch": { + "groups": [ + { + "count": 64, + "q": 1, + "ctx": 65536 + } + ] + } + }, + "sources": [ + "moonshotai/Kimi-K3/self_attn" + ] + }, + { + "type": "kda", + "args": { + "hidden": 7168, + "heads": 96, + "head_dim": 128, + "conv_kernel": 4, + "state_dtype": "bf16", + "batch": { + "groups": [ + { + "count": 64, + "q": 1, + "ctx": 131072 + } + ] + } + }, + "sources": [ + "moonshotai/Kimi-K3/self_attn" + ] + }, + { + "type": "kda", + "args": { + "hidden": 7168, + "heads": 96, + "head_dim": 128, + "conv_kernel": 4, + "state_dtype": "bf16", + "batch": { + "groups": [ + { + "count": 256, + "q": 1, + "ctx": 1024 + } + ] + } + }, + "sources": [ + "moonshotai/Kimi-K3/self_attn" + ] + }, + { + "type": "kda", + "args": { + "hidden": 7168, + "heads": 96, + "head_dim": 128, + "conv_kernel": 4, + "state_dtype": "bf16", + "batch": { + "groups": [ + { + "count": 256, + "q": 1, + "ctx": 4096 + } + ] + } + }, + "sources": [ + "moonshotai/Kimi-K3/self_attn" + ] + }, + { + "type": "kda", + "args": { + "hidden": 7168, + "heads": 96, + "head_dim": 128, + "conv_kernel": 4, + "state_dtype": "bf16", + "batch": { + "groups": [ + { + "count": 256, + "q": 1, + "ctx": 16384 + } + ] + } + }, + "sources": [ + "moonshotai/Kimi-K3/self_attn" + ] + }, + { + "type": "kda", + "args": { + "hidden": 7168, + "heads": 96, + "head_dim": 128, + "conv_kernel": 4, + "state_dtype": "bf16", + "batch": { + "groups": [ + { + "count": 256, + "q": 1, + "ctx": 65536 + } + ] + } + }, + "sources": [ + "moonshotai/Kimi-K3/self_attn" + ] + }, + { + "type": "kda", + "args": { + "hidden": 7168, + "heads": 96, + "head_dim": 128, + "conv_kernel": 4, + "state_dtype": "bf16", + "batch": { + "groups": [ + { + "count": 256, + "q": 1, + "ctx": 131072 + } + ] + } + }, + "sources": [ + "moonshotai/Kimi-K3/self_attn" + ] + }, + { + "type": "kda", + "args": { + "hidden": 7168, + "heads": 96, + "head_dim": 128, + "conv_kernel": 4, + "state_dtype": "bf16", + "batch": { + "groups": [ + { + "count": 1, + "q": 1024, + "ctx": 0 + } + ] + } + }, + "sources": [ + "moonshotai/Kimi-K3/self_attn" + ] + }, + { + "type": "kda", + "args": { + "hidden": 7168, + "heads": 96, + "head_dim": 128, + "conv_kernel": 4, + "state_dtype": "bf16", + "batch": { + "groups": [ + { + "count": 1, + "q": 4096, + "ctx": 0 + } + ] + } + }, + "sources": [ + "moonshotai/Kimi-K3/self_attn" + ] + }, + { + "type": "kda", + "args": { + "hidden": 7168, + "heads": 96, + "head_dim": 128, + "conv_kernel": 4, + "state_dtype": "bf16", + "batch": { + "groups": [ + { + "count": 1, + "q": 16384, + "ctx": 0 + } + ] + } + }, + "sources": [ + "moonshotai/Kimi-K3/self_attn" + ] + }, + { + "type": "kda", + "args": { + "hidden": 7168, + "heads": 96, + "head_dim": 128, + "conv_kernel": 4, + "state_dtype": "bf16", + "batch": { + "groups": [ + { + "count": 1, + "q": 32768, + "ctx": 0 + } + ] + } + }, + "sources": [ + "moonshotai/Kimi-K3/self_attn" + ] + }, + { + "type": "kda", + "args": { + "hidden": 7168, + "heads": 96, + "head_dim": 128, + "conv_kernel": 4, + "state_dtype": "bf16", + "batch": { + "groups": [ + { + "count": 1, + "q": 8192, + "ctx": 8192 + } + ] + } + }, + "sources": [ + "moonshotai/Kimi-K3/self_attn" + ] + }, + { + "type": "kda", + "args": { + "hidden": 7168, + "heads": 96, + "head_dim": 128, + "conv_kernel": 4, + "state_dtype": "bf16", + "batch": { + "groups": [ + { + "count": 1, + "q": 8192, + "ctx": 32768 + } + ] + } + }, + "sources": [ + "moonshotai/Kimi-K3/self_attn" + ] + }, + { + "type": "kda", + "args": { + "hidden": 7168, + "heads": 96, + "head_dim": 128, + "conv_kernel": 4, + "state_dtype": "bf16", + "batch": { + "groups": [ + { + "count": 1, + "q": 8192, + "ctx": 131072 + } + ] + } + }, + "sources": [ + "moonshotai/Kimi-K3/self_attn" + ] + }, + { + "type": "kda", + "args": { + "hidden": 7168, + "heads": 96, + "head_dim": 128, + "conv_kernel": 4, + "state_dtype": "bf16", + "batch": { + "groups": [ + { + "count": 1, + "q": 8192, + "ctx": 8192 + }, + { + "count": 64, + "q": 1, + "ctx": 8192 + } + ] + } + }, + "sources": [ + "moonshotai/Kimi-K3/self_attn" + ] + } +] \ No newline at end of file diff --git a/operatorx/testlists/attn_minimax_m3.json b/operatorx/testlists/attn_minimax_m3.json new file mode 100644 index 0000000000..cc6d67d014 --- /dev/null +++ b/operatorx/testlists/attn_minimax_m3.json @@ -0,0 +1,4099 @@ +[ + { + "type": "gqa", + "args": { + "hidden": 6144, + "q_heads": 64, + "kv_heads": 4, + "head_dim": 128, + "rope_dim": 64, + "rope_theta": 5000000, + "qk_norm": true, + "kv_cache_dtype": "fp8", + "proj": { + "k_proj": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + } + }, + "o_proj": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + } + }, + "q_proj": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + } + }, + "v_proj": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + } + } + }, + "batch": { + "groups": [ + { + "count": 1, + "q": 1, + "ctx": 1024 + } + ] + } + }, + "sources": [ + "MiniMaxAI/MiniMax-M3-MXFP8/self_attn", + "nvidia/MiniMax-M3-NVFP4/self_attn" + ] + }, + { + "type": "gqa", + "args": { + "hidden": 6144, + "q_heads": 64, + "kv_heads": 4, + "head_dim": 128, + "rope_dim": 64, + "rope_theta": 5000000, + "qk_norm": true, + "kv_cache_dtype": "fp8", + "proj": { + "k_proj": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + } + }, + "o_proj": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + } + }, + "q_proj": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + } + }, + "v_proj": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + } + } + }, + "batch": { + "groups": [ + { + "count": 1, + "q": 1, + "ctx": 4096 + } + ] + } + }, + "sources": [ + "MiniMaxAI/MiniMax-M3-MXFP8/self_attn", + "nvidia/MiniMax-M3-NVFP4/self_attn" + ] + }, + { + "type": "gqa", + "args": { + "hidden": 6144, + "q_heads": 64, + "kv_heads": 4, + "head_dim": 128, + "rope_dim": 64, + "rope_theta": 5000000, + "qk_norm": true, + "kv_cache_dtype": "fp8", + "proj": { + "k_proj": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + } + }, + "o_proj": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + } + }, + "q_proj": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + } + }, + "v_proj": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + } + } + }, + "batch": { + "groups": [ + { + "count": 1, + "q": 1, + "ctx": 16384 + } + ] + } + }, + "sources": [ + "MiniMaxAI/MiniMax-M3-MXFP8/self_attn", + "nvidia/MiniMax-M3-NVFP4/self_attn" + ] + }, + { + "type": "gqa", + "args": { + "hidden": 6144, + "q_heads": 64, + "kv_heads": 4, + "head_dim": 128, + "rope_dim": 64, + "rope_theta": 5000000, + "qk_norm": true, + "kv_cache_dtype": "fp8", + "proj": { + "k_proj": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + } + }, + "o_proj": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + } + }, + "q_proj": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + } + }, + "v_proj": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + } + } + }, + "batch": { + "groups": [ + { + "count": 1, + "q": 1, + "ctx": 65536 + } + ] + } + }, + "sources": [ + "MiniMaxAI/MiniMax-M3-MXFP8/self_attn", + "nvidia/MiniMax-M3-NVFP4/self_attn" + ] + }, + { + "type": "gqa", + "args": { + "hidden": 6144, + "q_heads": 64, + "kv_heads": 4, + "head_dim": 128, + "rope_dim": 64, + "rope_theta": 5000000, + "qk_norm": true, + "kv_cache_dtype": "fp8", + "proj": { + "k_proj": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + } + }, + "o_proj": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + } + }, + "q_proj": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + } + }, + "v_proj": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + } + } + }, + "batch": { + "groups": [ + { + "count": 1, + "q": 1, + "ctx": 131072 + } + ] + } + }, + "sources": [ + "MiniMaxAI/MiniMax-M3-MXFP8/self_attn", + "nvidia/MiniMax-M3-NVFP4/self_attn" + ] + }, + { + "type": "gqa", + "args": { + "hidden": 6144, + "q_heads": 64, + "kv_heads": 4, + "head_dim": 128, + "rope_dim": 64, + "rope_theta": 5000000, + "qk_norm": true, + "kv_cache_dtype": "fp8", + "proj": { + "k_proj": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + } + }, + "o_proj": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + } + }, + "q_proj": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + } + }, + "v_proj": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + } + } + }, + "batch": { + "groups": [ + { + "count": 4, + "q": 1, + "ctx": 1024 + } + ] + } + }, + "sources": [ + "MiniMaxAI/MiniMax-M3-MXFP8/self_attn", + "nvidia/MiniMax-M3-NVFP4/self_attn" + ] + }, + { + "type": "gqa", + "args": { + "hidden": 6144, + "q_heads": 64, + "kv_heads": 4, + "head_dim": 128, + "rope_dim": 64, + "rope_theta": 5000000, + "qk_norm": true, + "kv_cache_dtype": "fp8", + "proj": { + "k_proj": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + } + }, + "o_proj": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + } + }, + "q_proj": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + } + }, + "v_proj": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + } + } + }, + "batch": { + "groups": [ + { + "count": 4, + "q": 1, + "ctx": 4096 + } + ] + } + }, + "sources": [ + "MiniMaxAI/MiniMax-M3-MXFP8/self_attn", + "nvidia/MiniMax-M3-NVFP4/self_attn" + ] + }, + { + "type": "gqa", + "args": { + "hidden": 6144, + "q_heads": 64, + "kv_heads": 4, + "head_dim": 128, + "rope_dim": 64, + "rope_theta": 5000000, + "qk_norm": true, + "kv_cache_dtype": "fp8", + "proj": { + "k_proj": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + } + }, + "o_proj": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + } + }, + "q_proj": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + } + }, + "v_proj": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + } + } + }, + "batch": { + "groups": [ + { + "count": 4, + "q": 1, + "ctx": 16384 + } + ] + } + }, + "sources": [ + "MiniMaxAI/MiniMax-M3-MXFP8/self_attn", + "nvidia/MiniMax-M3-NVFP4/self_attn" + ] + }, + { + "type": "gqa", + "args": { + "hidden": 6144, + "q_heads": 64, + "kv_heads": 4, + "head_dim": 128, + "rope_dim": 64, + "rope_theta": 5000000, + "qk_norm": true, + "kv_cache_dtype": "fp8", + "proj": { + "k_proj": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + } + }, + "o_proj": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + } + }, + "q_proj": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + } + }, + "v_proj": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + } + } + }, + "batch": { + "groups": [ + { + "count": 4, + "q": 1, + "ctx": 65536 + } + ] + } + }, + "sources": [ + "MiniMaxAI/MiniMax-M3-MXFP8/self_attn", + "nvidia/MiniMax-M3-NVFP4/self_attn" + ] + }, + { + "type": "gqa", + "args": { + "hidden": 6144, + "q_heads": 64, + "kv_heads": 4, + "head_dim": 128, + "rope_dim": 64, + "rope_theta": 5000000, + "qk_norm": true, + "kv_cache_dtype": "fp8", + "proj": { + "k_proj": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + } + }, + "o_proj": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + } + }, + "q_proj": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + } + }, + "v_proj": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + } + } + }, + "batch": { + "groups": [ + { + "count": 4, + "q": 1, + "ctx": 131072 + } + ] + } + }, + "sources": [ + "MiniMaxAI/MiniMax-M3-MXFP8/self_attn", + "nvidia/MiniMax-M3-NVFP4/self_attn" + ] + }, + { + "type": "gqa", + "args": { + "hidden": 6144, + "q_heads": 64, + "kv_heads": 4, + "head_dim": 128, + "rope_dim": 64, + "rope_theta": 5000000, + "qk_norm": true, + "kv_cache_dtype": "fp8", + "proj": { + "k_proj": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + } + }, + "o_proj": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + } + }, + "q_proj": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + } + }, + "v_proj": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + } + } + }, + "batch": { + "groups": [ + { + "count": 16, + "q": 1, + "ctx": 1024 + } + ] + } + }, + "sources": [ + "MiniMaxAI/MiniMax-M3-MXFP8/self_attn", + "nvidia/MiniMax-M3-NVFP4/self_attn" + ] + }, + { + "type": "gqa", + "args": { + "hidden": 6144, + "q_heads": 64, + "kv_heads": 4, + "head_dim": 128, + "rope_dim": 64, + "rope_theta": 5000000, + "qk_norm": true, + "kv_cache_dtype": "fp8", + "proj": { + "k_proj": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + } + }, + "o_proj": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + } + }, + "q_proj": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + } + }, + "v_proj": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + } + } + }, + "batch": { + "groups": [ + { + "count": 16, + "q": 1, + "ctx": 4096 + } + ] + } + }, + "sources": [ + "MiniMaxAI/MiniMax-M3-MXFP8/self_attn", + "nvidia/MiniMax-M3-NVFP4/self_attn" + ] + }, + { + "type": "gqa", + "args": { + "hidden": 6144, + "q_heads": 64, + "kv_heads": 4, + "head_dim": 128, + "rope_dim": 64, + "rope_theta": 5000000, + "qk_norm": true, + "kv_cache_dtype": "fp8", + "proj": { + "k_proj": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + } + }, + "o_proj": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + } + }, + "q_proj": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + } + }, + "v_proj": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + } + } + }, + "batch": { + "groups": [ + { + "count": 16, + "q": 1, + "ctx": 16384 + } + ] + } + }, + "sources": [ + "MiniMaxAI/MiniMax-M3-MXFP8/self_attn", + "nvidia/MiniMax-M3-NVFP4/self_attn" + ] + }, + { + "type": "gqa", + "args": { + "hidden": 6144, + "q_heads": 64, + "kv_heads": 4, + "head_dim": 128, + "rope_dim": 64, + "rope_theta": 5000000, + "qk_norm": true, + "kv_cache_dtype": "fp8", + "proj": { + "k_proj": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + } + }, + "o_proj": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + } + }, + "q_proj": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + } + }, + "v_proj": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + } + } + }, + "batch": { + "groups": [ + { + "count": 16, + "q": 1, + "ctx": 65536 + } + ] + } + }, + "sources": [ + "MiniMaxAI/MiniMax-M3-MXFP8/self_attn", + "nvidia/MiniMax-M3-NVFP4/self_attn" + ] + }, + { + "type": "gqa", + "args": { + "hidden": 6144, + "q_heads": 64, + "kv_heads": 4, + "head_dim": 128, + "rope_dim": 64, + "rope_theta": 5000000, + "qk_norm": true, + "kv_cache_dtype": "fp8", + "proj": { + "k_proj": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + } + }, + "o_proj": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + } + }, + "q_proj": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + } + }, + "v_proj": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + } + } + }, + "batch": { + "groups": [ + { + "count": 16, + "q": 1, + "ctx": 131072 + } + ] + } + }, + "sources": [ + "MiniMaxAI/MiniMax-M3-MXFP8/self_attn", + "nvidia/MiniMax-M3-NVFP4/self_attn" + ] + }, + { + "type": "gqa", + "args": { + "hidden": 6144, + "q_heads": 64, + "kv_heads": 4, + "head_dim": 128, + "rope_dim": 64, + "rope_theta": 5000000, + "qk_norm": true, + "kv_cache_dtype": "fp8", + "proj": { + "k_proj": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + } + }, + "o_proj": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + } + }, + "q_proj": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + } + }, + "v_proj": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + } + } + }, + "batch": { + "groups": [ + { + "count": 64, + "q": 1, + "ctx": 1024 + } + ] + } + }, + "sources": [ + "MiniMaxAI/MiniMax-M3-MXFP8/self_attn", + "nvidia/MiniMax-M3-NVFP4/self_attn" + ] + }, + { + "type": "gqa", + "args": { + "hidden": 6144, + "q_heads": 64, + "kv_heads": 4, + "head_dim": 128, + "rope_dim": 64, + "rope_theta": 5000000, + "qk_norm": true, + "kv_cache_dtype": "fp8", + "proj": { + "k_proj": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + } + }, + "o_proj": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + } + }, + "q_proj": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + } + }, + "v_proj": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + } + } + }, + "batch": { + "groups": [ + { + "count": 64, + "q": 1, + "ctx": 4096 + } + ] + } + }, + "sources": [ + "MiniMaxAI/MiniMax-M3-MXFP8/self_attn", + "nvidia/MiniMax-M3-NVFP4/self_attn" + ] + }, + { + "type": "gqa", + "args": { + "hidden": 6144, + "q_heads": 64, + "kv_heads": 4, + "head_dim": 128, + "rope_dim": 64, + "rope_theta": 5000000, + "qk_norm": true, + "kv_cache_dtype": "fp8", + "proj": { + "k_proj": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + } + }, + "o_proj": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + } + }, + "q_proj": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + } + }, + "v_proj": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + } + } + }, + "batch": { + "groups": [ + { + "count": 64, + "q": 1, + "ctx": 16384 + } + ] + } + }, + "sources": [ + "MiniMaxAI/MiniMax-M3-MXFP8/self_attn", + "nvidia/MiniMax-M3-NVFP4/self_attn" + ] + }, + { + "type": "gqa", + "args": { + "hidden": 6144, + "q_heads": 64, + "kv_heads": 4, + "head_dim": 128, + "rope_dim": 64, + "rope_theta": 5000000, + "qk_norm": true, + "kv_cache_dtype": "fp8", + "proj": { + "k_proj": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + } + }, + "o_proj": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + } + }, + "q_proj": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + } + }, + "v_proj": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + } + } + }, + "batch": { + "groups": [ + { + "count": 64, + "q": 1, + "ctx": 65536 + } + ] + } + }, + "sources": [ + "MiniMaxAI/MiniMax-M3-MXFP8/self_attn", + "nvidia/MiniMax-M3-NVFP4/self_attn" + ] + }, + { + "type": "gqa", + "args": { + "hidden": 6144, + "q_heads": 64, + "kv_heads": 4, + "head_dim": 128, + "rope_dim": 64, + "rope_theta": 5000000, + "qk_norm": true, + "kv_cache_dtype": "fp8", + "proj": { + "k_proj": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + } + }, + "o_proj": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + } + }, + "q_proj": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + } + }, + "v_proj": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + } + } + }, + "batch": { + "groups": [ + { + "count": 64, + "q": 1, + "ctx": 131072 + } + ] + } + }, + "sources": [ + "MiniMaxAI/MiniMax-M3-MXFP8/self_attn", + "nvidia/MiniMax-M3-NVFP4/self_attn" + ] + }, + { + "type": "gqa", + "args": { + "hidden": 6144, + "q_heads": 64, + "kv_heads": 4, + "head_dim": 128, + "rope_dim": 64, + "rope_theta": 5000000, + "qk_norm": true, + "kv_cache_dtype": "fp8", + "proj": { + "k_proj": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + } + }, + "o_proj": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + } + }, + "q_proj": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + } + }, + "v_proj": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + } + } + }, + "batch": { + "groups": [ + { + "count": 256, + "q": 1, + "ctx": 1024 + } + ] + } + }, + "sources": [ + "MiniMaxAI/MiniMax-M3-MXFP8/self_attn", + "nvidia/MiniMax-M3-NVFP4/self_attn" + ] + }, + { + "type": "gqa", + "args": { + "hidden": 6144, + "q_heads": 64, + "kv_heads": 4, + "head_dim": 128, + "rope_dim": 64, + "rope_theta": 5000000, + "qk_norm": true, + "kv_cache_dtype": "fp8", + "proj": { + "k_proj": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + } + }, + "o_proj": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + } + }, + "q_proj": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + } + }, + "v_proj": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + } + } + }, + "batch": { + "groups": [ + { + "count": 256, + "q": 1, + "ctx": 4096 + } + ] + } + }, + "sources": [ + "MiniMaxAI/MiniMax-M3-MXFP8/self_attn", + "nvidia/MiniMax-M3-NVFP4/self_attn" + ] + }, + { + "type": "gqa", + "args": { + "hidden": 6144, + "q_heads": 64, + "kv_heads": 4, + "head_dim": 128, + "rope_dim": 64, + "rope_theta": 5000000, + "qk_norm": true, + "kv_cache_dtype": "fp8", + "proj": { + "k_proj": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + } + }, + "o_proj": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + } + }, + "q_proj": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + } + }, + "v_proj": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + } + } + }, + "batch": { + "groups": [ + { + "count": 256, + "q": 1, + "ctx": 16384 + } + ] + } + }, + "sources": [ + "MiniMaxAI/MiniMax-M3-MXFP8/self_attn", + "nvidia/MiniMax-M3-NVFP4/self_attn" + ] + }, + { + "type": "gqa", + "args": { + "hidden": 6144, + "q_heads": 64, + "kv_heads": 4, + "head_dim": 128, + "rope_dim": 64, + "rope_theta": 5000000, + "qk_norm": true, + "kv_cache_dtype": "fp8", + "proj": { + "k_proj": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + } + }, + "o_proj": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + } + }, + "q_proj": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + } + }, + "v_proj": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + } + } + }, + "batch": { + "groups": [ + { + "count": 256, + "q": 1, + "ctx": 65536 + } + ] + } + }, + "sources": [ + "MiniMaxAI/MiniMax-M3-MXFP8/self_attn", + "nvidia/MiniMax-M3-NVFP4/self_attn" + ] + }, + { + "type": "gqa", + "args": { + "hidden": 6144, + "q_heads": 64, + "kv_heads": 4, + "head_dim": 128, + "rope_dim": 64, + "rope_theta": 5000000, + "qk_norm": true, + "kv_cache_dtype": "fp8", + "proj": { + "k_proj": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + } + }, + "o_proj": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + } + }, + "q_proj": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + } + }, + "v_proj": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + } + } + }, + "batch": { + "groups": [ + { + "count": 256, + "q": 1, + "ctx": 131072 + } + ] + } + }, + "sources": [ + "MiniMaxAI/MiniMax-M3-MXFP8/self_attn", + "nvidia/MiniMax-M3-NVFP4/self_attn" + ] + }, + { + "type": "gqa", + "args": { + "hidden": 6144, + "q_heads": 64, + "kv_heads": 4, + "head_dim": 128, + "rope_dim": 64, + "rope_theta": 5000000, + "qk_norm": true, + "kv_cache_dtype": "fp8", + "proj": { + "k_proj": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + } + }, + "o_proj": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + } + }, + "q_proj": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + } + }, + "v_proj": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + } + } + }, + "batch": { + "groups": [ + { + "count": 1, + "q": 1024, + "ctx": 0 + } + ] + } + }, + "sources": [ + "MiniMaxAI/MiniMax-M3-MXFP8/self_attn", + "nvidia/MiniMax-M3-NVFP4/self_attn" + ] + }, + { + "type": "gqa", + "args": { + "hidden": 6144, + "q_heads": 64, + "kv_heads": 4, + "head_dim": 128, + "rope_dim": 64, + "rope_theta": 5000000, + "qk_norm": true, + "kv_cache_dtype": "fp8", + "proj": { + "k_proj": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + } + }, + "o_proj": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + } + }, + "q_proj": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + } + }, + "v_proj": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + } + } + }, + "batch": { + "groups": [ + { + "count": 1, + "q": 4096, + "ctx": 0 + } + ] + } + }, + "sources": [ + "MiniMaxAI/MiniMax-M3-MXFP8/self_attn", + "nvidia/MiniMax-M3-NVFP4/self_attn" + ] + }, + { + "type": "gqa", + "args": { + "hidden": 6144, + "q_heads": 64, + "kv_heads": 4, + "head_dim": 128, + "rope_dim": 64, + "rope_theta": 5000000, + "qk_norm": true, + "kv_cache_dtype": "fp8", + "proj": { + "k_proj": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + } + }, + "o_proj": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + } + }, + "q_proj": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + } + }, + "v_proj": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + } + } + }, + "batch": { + "groups": [ + { + "count": 1, + "q": 16384, + "ctx": 0 + } + ] + } + }, + "sources": [ + "MiniMaxAI/MiniMax-M3-MXFP8/self_attn", + "nvidia/MiniMax-M3-NVFP4/self_attn" + ] + }, + { + "type": "gqa", + "args": { + "hidden": 6144, + "q_heads": 64, + "kv_heads": 4, + "head_dim": 128, + "rope_dim": 64, + "rope_theta": 5000000, + "qk_norm": true, + "kv_cache_dtype": "fp8", + "proj": { + "k_proj": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + } + }, + "o_proj": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + } + }, + "q_proj": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + } + }, + "v_proj": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + } + } + }, + "batch": { + "groups": [ + { + "count": 1, + "q": 32768, + "ctx": 0 + } + ] + } + }, + "sources": [ + "MiniMaxAI/MiniMax-M3-MXFP8/self_attn", + "nvidia/MiniMax-M3-NVFP4/self_attn" + ] + }, + { + "type": "gqa", + "args": { + "hidden": 6144, + "q_heads": 64, + "kv_heads": 4, + "head_dim": 128, + "rope_dim": 64, + "rope_theta": 5000000, + "qk_norm": true, + "kv_cache_dtype": "fp8", + "proj": { + "k_proj": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + } + }, + "o_proj": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + } + }, + "q_proj": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + } + }, + "v_proj": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + } + } + }, + "batch": { + "groups": [ + { + "count": 1, + "q": 8192, + "ctx": 8192 + } + ] + } + }, + "sources": [ + "MiniMaxAI/MiniMax-M3-MXFP8/self_attn", + "nvidia/MiniMax-M3-NVFP4/self_attn" + ] + }, + { + "type": "gqa", + "args": { + "hidden": 6144, + "q_heads": 64, + "kv_heads": 4, + "head_dim": 128, + "rope_dim": 64, + "rope_theta": 5000000, + "qk_norm": true, + "kv_cache_dtype": "fp8", + "proj": { + "k_proj": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + } + }, + "o_proj": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + } + }, + "q_proj": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + } + }, + "v_proj": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + } + } + }, + "batch": { + "groups": [ + { + "count": 1, + "q": 8192, + "ctx": 32768 + } + ] + } + }, + "sources": [ + "MiniMaxAI/MiniMax-M3-MXFP8/self_attn", + "nvidia/MiniMax-M3-NVFP4/self_attn" + ] + }, + { + "type": "gqa", + "args": { + "hidden": 6144, + "q_heads": 64, + "kv_heads": 4, + "head_dim": 128, + "rope_dim": 64, + "rope_theta": 5000000, + "qk_norm": true, + "kv_cache_dtype": "fp8", + "proj": { + "k_proj": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + } + }, + "o_proj": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + } + }, + "q_proj": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + } + }, + "v_proj": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + } + } + }, + "batch": { + "groups": [ + { + "count": 1, + "q": 8192, + "ctx": 131072 + } + ] + } + }, + "sources": [ + "MiniMaxAI/MiniMax-M3-MXFP8/self_attn", + "nvidia/MiniMax-M3-NVFP4/self_attn" + ] + }, + { + "type": "gqa", + "args": { + "hidden": 6144, + "q_heads": 64, + "kv_heads": 4, + "head_dim": 128, + "rope_dim": 64, + "rope_theta": 5000000, + "qk_norm": true, + "kv_cache_dtype": "fp8", + "proj": { + "k_proj": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + } + }, + "o_proj": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + } + }, + "q_proj": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + } + }, + "v_proj": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + } + } + }, + "batch": { + "groups": [ + { + "count": 1, + "q": 8192, + "ctx": 8192 + }, + { + "count": 64, + "q": 1, + "ctx": 8192 + } + ] + } + }, + "sources": [ + "MiniMaxAI/MiniMax-M3-MXFP8/self_attn", + "nvidia/MiniMax-M3-NVFP4/self_attn" + ] + } +] \ No newline at end of file diff --git a/operatorx/testlists/attn_parallel.json b/operatorx/testlists/attn_parallel.json new file mode 100644 index 0000000000..03edbd7c57 --- /dev/null +++ b/operatorx/testlists/attn_parallel.json @@ -0,0 +1,11884 @@ +[ + { + "type": "dsv4_attn", + "args": { + "hidden": 7168, + "heads": 128, + "head_dim": 512, + "rope_dim": 64, + "q_lora_rank": 1536, + "window": 128, + "o_groups": 16, + "o_lora_rank": 1024, + "compress_rope_theta": 160000, + "rope_scaling": { + "type": "yarn", + "factor": 16, + "original_max": 65536, + "beta_fast": 32, + "beta_slow": 1 + }, + "kv_cache_dtype": "fp8", + "compress_ratio": 4, + "indexer": "own", + "topk": 1024, + "index_heads": 64, + "index_dim": 128, + "index_cache_dtype": "fp8", + "proj": { + "indexer.wq_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 128 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + } + }, + "wkv": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 128 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + } + }, + "wo_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 128 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + } + }, + "wq_a": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 128 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + } + }, + "wq_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 128 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + } + } + }, + "batch": { + "groups": [ + { + "count": 16, + "q": 1, + "ctx": 16384 + } + ] + }, + "parallel": { + "tp": 4 + } + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro/attn" + ] + }, + { + "type": "dsv4_attn", + "args": { + "hidden": 7168, + "heads": 128, + "head_dim": 512, + "rope_dim": 64, + "q_lora_rank": 1536, + "window": 128, + "o_groups": 16, + "o_lora_rank": 1024, + "compress_rope_theta": 160000, + "rope_scaling": { + "type": "yarn", + "factor": 16, + "original_max": 65536, + "beta_fast": 32, + "beta_slow": 1 + }, + "kv_cache_dtype": "fp8", + "compress_ratio": 4, + "indexer": "own", + "topk": 1024, + "index_heads": 64, + "index_dim": 128, + "index_cache_dtype": "fp8", + "proj": { + "indexer.wq_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 128 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + } + }, + "wkv": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 128 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + } + }, + "wo_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 128 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + } + }, + "wq_a": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 128 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + } + }, + "wq_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 128 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + } + } + }, + "batch": { + "groups": [ + { + "count": 16, + "q": 1, + "ctx": 16384 + } + ] + }, + "parallel": { + "tp": 8 + } + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro/attn" + ] + }, + { + "type": "dsv4_attn", + "args": { + "hidden": 7168, + "heads": 128, + "head_dim": 512, + "rope_dim": 64, + "q_lora_rank": 1536, + "window": 128, + "o_groups": 16, + "o_lora_rank": 1024, + "compress_rope_theta": 160000, + "rope_scaling": { + "type": "yarn", + "factor": 16, + "original_max": 65536, + "beta_fast": 32, + "beta_slow": 1 + }, + "kv_cache_dtype": "fp8", + "compress_ratio": 4, + "indexer": "own", + "topk": 1024, + "index_heads": 64, + "index_dim": 128, + "index_cache_dtype": "fp8", + "proj": { + "indexer.wq_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 128 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + } + }, + "wkv": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 128 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + } + }, + "wo_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 128 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + } + }, + "wq_a": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 128 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + } + }, + "wq_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 128 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + } + } + }, + "batch": { + "groups": [ + { + "count": 16, + "q": 1, + "ctx": 16384 + } + ] + }, + "parallel": { + "dp": 4 + } + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro/attn" + ] + }, + { + "type": "dsv4_attn", + "args": { + "hidden": 7168, + "heads": 128, + "head_dim": 512, + "rope_dim": 64, + "q_lora_rank": 1536, + "window": 128, + "o_groups": 16, + "o_lora_rank": 1024, + "compress_rope_theta": 160000, + "rope_scaling": { + "type": "yarn", + "factor": 16, + "original_max": 65536, + "beta_fast": 32, + "beta_slow": 1 + }, + "kv_cache_dtype": "fp8", + "compress_ratio": 4, + "indexer": "own", + "topk": 1024, + "index_heads": 64, + "index_dim": 128, + "index_cache_dtype": "fp8", + "proj": { + "indexer.wq_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 128 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + } + }, + "wkv": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 128 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + } + }, + "wo_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 128 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + } + }, + "wq_a": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 128 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + } + }, + "wq_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 128 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + } + } + }, + "batch": { + "groups": [ + { + "count": 16, + "q": 1, + "ctx": 16384 + } + ] + }, + "parallel": { + "dp": 8 + } + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro/attn" + ] + }, + { + "type": "dsv4_attn", + "args": { + "hidden": 7168, + "heads": 128, + "head_dim": 512, + "rope_dim": 64, + "q_lora_rank": 1536, + "window": 128, + "o_groups": 16, + "o_lora_rank": 1024, + "compress_rope_theta": 160000, + "rope_scaling": { + "type": "yarn", + "factor": 16, + "original_max": 65536, + "beta_fast": 32, + "beta_slow": 1 + }, + "kv_cache_dtype": "fp8", + "compress_ratio": 4, + "indexer": "own", + "topk": 1024, + "index_heads": 64, + "index_dim": 128, + "index_cache_dtype": "fp8", + "proj": { + "indexer.wq_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 128 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + } + }, + "wkv": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 128 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + } + }, + "wo_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 128 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + } + }, + "wq_a": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 128 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + } + }, + "wq_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 128 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + } + } + }, + "batch": { + "groups": [ + { + "count": 1, + "q": 4096, + "ctx": 0 + } + ] + }, + "parallel": { + "tp": 4 + } + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro/attn" + ] + }, + { + "type": "dsv4_attn", + "args": { + "hidden": 7168, + "heads": 128, + "head_dim": 512, + "rope_dim": 64, + "q_lora_rank": 1536, + "window": 128, + "o_groups": 16, + "o_lora_rank": 1024, + "compress_rope_theta": 160000, + "rope_scaling": { + "type": "yarn", + "factor": 16, + "original_max": 65536, + "beta_fast": 32, + "beta_slow": 1 + }, + "kv_cache_dtype": "fp8", + "compress_ratio": 4, + "indexer": "own", + "topk": 1024, + "index_heads": 64, + "index_dim": 128, + "index_cache_dtype": "fp8", + "proj": { + "indexer.wq_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 128 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + } + }, + "wkv": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 128 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + } + }, + "wo_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 128 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + } + }, + "wq_a": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 128 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + } + }, + "wq_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 128 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + } + } + }, + "batch": { + "groups": [ + { + "count": 1, + "q": 4096, + "ctx": 0 + } + ] + }, + "parallel": { + "tp": 8 + } + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro/attn" + ] + }, + { + "type": "dsv4_attn", + "args": { + "hidden": 7168, + "heads": 128, + "head_dim": 512, + "rope_dim": 64, + "q_lora_rank": 1536, + "window": 128, + "o_groups": 16, + "o_lora_rank": 1024, + "compress_rope_theta": 160000, + "rope_scaling": { + "type": "yarn", + "factor": 16, + "original_max": 65536, + "beta_fast": 32, + "beta_slow": 1 + }, + "kv_cache_dtype": "fp8", + "compress_ratio": 4, + "indexer": "own", + "topk": 1024, + "index_heads": 64, + "index_dim": 128, + "index_cache_dtype": "fp8", + "proj": { + "indexer.wq_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 128 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + } + }, + "wkv": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 128 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + } + }, + "wo_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 128 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + } + }, + "wq_a": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 128 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + } + }, + "wq_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 128 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + } + } + }, + "batch": { + "groups": [ + { + "count": 1, + "q": 4096, + "ctx": 0 + } + ] + }, + "parallel": { + "dp": 4 + } + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro/attn" + ] + }, + { + "type": "dsv4_attn", + "args": { + "hidden": 7168, + "heads": 128, + "head_dim": 512, + "rope_dim": 64, + "q_lora_rank": 1536, + "window": 128, + "o_groups": 16, + "o_lora_rank": 1024, + "compress_rope_theta": 160000, + "rope_scaling": { + "type": "yarn", + "factor": 16, + "original_max": 65536, + "beta_fast": 32, + "beta_slow": 1 + }, + "kv_cache_dtype": "fp8", + "compress_ratio": 4, + "indexer": "own", + "topk": 1024, + "index_heads": 64, + "index_dim": 128, + "index_cache_dtype": "fp8", + "proj": { + "indexer.wq_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 128 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + } + }, + "wkv": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 128 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + } + }, + "wo_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 128 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + } + }, + "wq_a": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 128 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + } + }, + "wq_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 128 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + } + } + }, + "batch": { + "groups": [ + { + "count": 1, + "q": 4096, + "ctx": 0 + } + ] + }, + "parallel": { + "dp": 8 + } + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro/attn" + ] + }, + { + "type": "dsv4_attn", + "args": { + "hidden": 7168, + "heads": 128, + "head_dim": 512, + "rope_dim": 64, + "q_lora_rank": 1536, + "window": 128, + "o_groups": 16, + "o_lora_rank": 1024, + "compress_rope_theta": 160000, + "rope_scaling": { + "type": "yarn", + "factor": 16, + "original_max": 65536, + "beta_fast": 32, + "beta_slow": 1 + }, + "kv_cache_dtype": "fp8", + "compress_ratio": 4, + "indexer": "own", + "topk": 1024, + "index_heads": 64, + "index_dim": 128, + "index_cache_dtype": "mxfp4", + "proj": { + "indexer.wq_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 128 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + } + }, + "wkv": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 128 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + } + }, + "wo_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 128 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + } + }, + "wq_a": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 128 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + } + }, + "wq_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 128 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + } + } + }, + "batch": { + "groups": [ + { + "count": 16, + "q": 1, + "ctx": 16384 + } + ] + }, + "parallel": { + "tp": 4 + } + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro/attn" + ] + }, + { + "type": "dsv4_attn", + "args": { + "hidden": 7168, + "heads": 128, + "head_dim": 512, + "rope_dim": 64, + "q_lora_rank": 1536, + "window": 128, + "o_groups": 16, + "o_lora_rank": 1024, + "compress_rope_theta": 160000, + "rope_scaling": { + "type": "yarn", + "factor": 16, + "original_max": 65536, + "beta_fast": 32, + "beta_slow": 1 + }, + "kv_cache_dtype": "fp8", + "compress_ratio": 4, + "indexer": "own", + "topk": 1024, + "index_heads": 64, + "index_dim": 128, + "index_cache_dtype": "mxfp4", + "proj": { + "indexer.wq_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 128 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + } + }, + "wkv": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 128 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + } + }, + "wo_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 128 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + } + }, + "wq_a": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 128 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + } + }, + "wq_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 128 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + } + } + }, + "batch": { + "groups": [ + { + "count": 16, + "q": 1, + "ctx": 16384 + } + ] + }, + "parallel": { + "tp": 8 + } + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro/attn" + ] + }, + { + "type": "dsv4_attn", + "args": { + "hidden": 7168, + "heads": 128, + "head_dim": 512, + "rope_dim": 64, + "q_lora_rank": 1536, + "window": 128, + "o_groups": 16, + "o_lora_rank": 1024, + "compress_rope_theta": 160000, + "rope_scaling": { + "type": "yarn", + "factor": 16, + "original_max": 65536, + "beta_fast": 32, + "beta_slow": 1 + }, + "kv_cache_dtype": "fp8", + "compress_ratio": 4, + "indexer": "own", + "topk": 1024, + "index_heads": 64, + "index_dim": 128, + "index_cache_dtype": "mxfp4", + "proj": { + "indexer.wq_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 128 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + } + }, + "wkv": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 128 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + } + }, + "wo_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 128 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + } + }, + "wq_a": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 128 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + } + }, + "wq_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 128 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + } + } + }, + "batch": { + "groups": [ + { + "count": 16, + "q": 1, + "ctx": 16384 + } + ] + }, + "parallel": { + "dp": 4 + } + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro/attn" + ] + }, + { + "type": "dsv4_attn", + "args": { + "hidden": 7168, + "heads": 128, + "head_dim": 512, + "rope_dim": 64, + "q_lora_rank": 1536, + "window": 128, + "o_groups": 16, + "o_lora_rank": 1024, + "compress_rope_theta": 160000, + "rope_scaling": { + "type": "yarn", + "factor": 16, + "original_max": 65536, + "beta_fast": 32, + "beta_slow": 1 + }, + "kv_cache_dtype": "fp8", + "compress_ratio": 4, + "indexer": "own", + "topk": 1024, + "index_heads": 64, + "index_dim": 128, + "index_cache_dtype": "mxfp4", + "proj": { + "indexer.wq_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 128 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + } + }, + "wkv": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 128 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + } + }, + "wo_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 128 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + } + }, + "wq_a": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 128 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + } + }, + "wq_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 128 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + } + } + }, + "batch": { + "groups": [ + { + "count": 16, + "q": 1, + "ctx": 16384 + } + ] + }, + "parallel": { + "dp": 8 + } + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro/attn" + ] + }, + { + "type": "dsv4_attn", + "args": { + "hidden": 7168, + "heads": 128, + "head_dim": 512, + "rope_dim": 64, + "q_lora_rank": 1536, + "window": 128, + "o_groups": 16, + "o_lora_rank": 1024, + "compress_rope_theta": 160000, + "rope_scaling": { + "type": "yarn", + "factor": 16, + "original_max": 65536, + "beta_fast": 32, + "beta_slow": 1 + }, + "kv_cache_dtype": "fp8", + "compress_ratio": 4, + "indexer": "own", + "topk": 1024, + "index_heads": 64, + "index_dim": 128, + "index_cache_dtype": "mxfp4", + "proj": { + "indexer.wq_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 128 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + } + }, + "wkv": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 128 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + } + }, + "wo_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 128 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + } + }, + "wq_a": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 128 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + } + }, + "wq_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 128 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + } + } + }, + "batch": { + "groups": [ + { + "count": 1, + "q": 4096, + "ctx": 0 + } + ] + }, + "parallel": { + "tp": 4 + } + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro/attn" + ] + }, + { + "type": "dsv4_attn", + "args": { + "hidden": 7168, + "heads": 128, + "head_dim": 512, + "rope_dim": 64, + "q_lora_rank": 1536, + "window": 128, + "o_groups": 16, + "o_lora_rank": 1024, + "compress_rope_theta": 160000, + "rope_scaling": { + "type": "yarn", + "factor": 16, + "original_max": 65536, + "beta_fast": 32, + "beta_slow": 1 + }, + "kv_cache_dtype": "fp8", + "compress_ratio": 4, + "indexer": "own", + "topk": 1024, + "index_heads": 64, + "index_dim": 128, + "index_cache_dtype": "mxfp4", + "proj": { + "indexer.wq_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 128 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + } + }, + "wkv": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 128 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + } + }, + "wo_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 128 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + } + }, + "wq_a": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 128 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + } + }, + "wq_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 128 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + } + } + }, + "batch": { + "groups": [ + { + "count": 1, + "q": 4096, + "ctx": 0 + } + ] + }, + "parallel": { + "tp": 8 + } + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro/attn" + ] + }, + { + "type": "dsv4_attn", + "args": { + "hidden": 7168, + "heads": 128, + "head_dim": 512, + "rope_dim": 64, + "q_lora_rank": 1536, + "window": 128, + "o_groups": 16, + "o_lora_rank": 1024, + "compress_rope_theta": 160000, + "rope_scaling": { + "type": "yarn", + "factor": 16, + "original_max": 65536, + "beta_fast": 32, + "beta_slow": 1 + }, + "kv_cache_dtype": "fp8", + "compress_ratio": 4, + "indexer": "own", + "topk": 1024, + "index_heads": 64, + "index_dim": 128, + "index_cache_dtype": "mxfp4", + "proj": { + "indexer.wq_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 128 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + } + }, + "wkv": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 128 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + } + }, + "wo_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 128 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + } + }, + "wq_a": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 128 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + } + }, + "wq_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 128 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + } + } + }, + "batch": { + "groups": [ + { + "count": 1, + "q": 4096, + "ctx": 0 + } + ] + }, + "parallel": { + "dp": 4 + } + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro/attn" + ] + }, + { + "type": "dsv4_attn", + "args": { + "hidden": 7168, + "heads": 128, + "head_dim": 512, + "rope_dim": 64, + "q_lora_rank": 1536, + "window": 128, + "o_groups": 16, + "o_lora_rank": 1024, + "compress_rope_theta": 160000, + "rope_scaling": { + "type": "yarn", + "factor": 16, + "original_max": 65536, + "beta_fast": 32, + "beta_slow": 1 + }, + "kv_cache_dtype": "fp8", + "compress_ratio": 4, + "indexer": "own", + "topk": 1024, + "index_heads": 64, + "index_dim": 128, + "index_cache_dtype": "mxfp4", + "proj": { + "indexer.wq_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 128 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + } + }, + "wkv": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 128 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + } + }, + "wo_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 128 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + } + }, + "wq_a": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 128 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + } + }, + "wq_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 128 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + } + } + }, + "batch": { + "groups": [ + { + "count": 1, + "q": 4096, + "ctx": 0 + } + ] + }, + "parallel": { + "dp": 8 + } + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro/attn" + ] + }, + { + "type": "dsv4_attn", + "args": { + "hidden": 7168, + "heads": 128, + "head_dim": 512, + "rope_dim": 64, + "q_lora_rank": 1536, + "window": 128, + "o_groups": 16, + "o_lora_rank": 1024, + "compress_rope_theta": 160000, + "rope_scaling": { + "type": "yarn", + "factor": 16, + "original_max": 65536, + "beta_fast": 32, + "beta_slow": 1 + }, + "kv_cache_dtype": "fp8", + "compress_ratio": 128, + "proj": { + "wkv": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 128 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + } + }, + "wo_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 128 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + } + }, + "wq_a": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 128 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + } + }, + "wq_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 128 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + } + } + }, + "batch": { + "groups": [ + { + "count": 16, + "q": 1, + "ctx": 16384 + } + ] + }, + "parallel": { + "tp": 4 + } + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro/attn" + ] + }, + { + "type": "dsv4_attn", + "args": { + "hidden": 7168, + "heads": 128, + "head_dim": 512, + "rope_dim": 64, + "q_lora_rank": 1536, + "window": 128, + "o_groups": 16, + "o_lora_rank": 1024, + "compress_rope_theta": 160000, + "rope_scaling": { + "type": "yarn", + "factor": 16, + "original_max": 65536, + "beta_fast": 32, + "beta_slow": 1 + }, + "kv_cache_dtype": "fp8", + "compress_ratio": 128, + "proj": { + "wkv": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 128 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + } + }, + "wo_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 128 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + } + }, + "wq_a": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 128 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + } + }, + "wq_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 128 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + } + } + }, + "batch": { + "groups": [ + { + "count": 16, + "q": 1, + "ctx": 16384 + } + ] + }, + "parallel": { + "tp": 8 + } + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro/attn" + ] + }, + { + "type": "dsv4_attn", + "args": { + "hidden": 7168, + "heads": 128, + "head_dim": 512, + "rope_dim": 64, + "q_lora_rank": 1536, + "window": 128, + "o_groups": 16, + "o_lora_rank": 1024, + "compress_rope_theta": 160000, + "rope_scaling": { + "type": "yarn", + "factor": 16, + "original_max": 65536, + "beta_fast": 32, + "beta_slow": 1 + }, + "kv_cache_dtype": "fp8", + "compress_ratio": 128, + "proj": { + "wkv": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 128 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + } + }, + "wo_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 128 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + } + }, + "wq_a": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 128 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + } + }, + "wq_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 128 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + } + } + }, + "batch": { + "groups": [ + { + "count": 16, + "q": 1, + "ctx": 16384 + } + ] + }, + "parallel": { + "dp": 4 + } + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro/attn" + ] + }, + { + "type": "dsv4_attn", + "args": { + "hidden": 7168, + "heads": 128, + "head_dim": 512, + "rope_dim": 64, + "q_lora_rank": 1536, + "window": 128, + "o_groups": 16, + "o_lora_rank": 1024, + "compress_rope_theta": 160000, + "rope_scaling": { + "type": "yarn", + "factor": 16, + "original_max": 65536, + "beta_fast": 32, + "beta_slow": 1 + }, + "kv_cache_dtype": "fp8", + "compress_ratio": 128, + "proj": { + "wkv": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 128 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + } + }, + "wo_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 128 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + } + }, + "wq_a": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 128 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + } + }, + "wq_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 128 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + } + } + }, + "batch": { + "groups": [ + { + "count": 16, + "q": 1, + "ctx": 16384 + } + ] + }, + "parallel": { + "dp": 8 + } + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro/attn" + ] + }, + { + "type": "dsv4_attn", + "args": { + "hidden": 7168, + "heads": 128, + "head_dim": 512, + "rope_dim": 64, + "q_lora_rank": 1536, + "window": 128, + "o_groups": 16, + "o_lora_rank": 1024, + "compress_rope_theta": 160000, + "rope_scaling": { + "type": "yarn", + "factor": 16, + "original_max": 65536, + "beta_fast": 32, + "beta_slow": 1 + }, + "kv_cache_dtype": "fp8", + "compress_ratio": 128, + "proj": { + "wkv": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 128 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + } + }, + "wo_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 128 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + } + }, + "wq_a": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 128 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + } + }, + "wq_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 128 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + } + } + }, + "batch": { + "groups": [ + { + "count": 1, + "q": 4096, + "ctx": 0 + } + ] + }, + "parallel": { + "tp": 4 + } + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro/attn" + ] + }, + { + "type": "dsv4_attn", + "args": { + "hidden": 7168, + "heads": 128, + "head_dim": 512, + "rope_dim": 64, + "q_lora_rank": 1536, + "window": 128, + "o_groups": 16, + "o_lora_rank": 1024, + "compress_rope_theta": 160000, + "rope_scaling": { + "type": "yarn", + "factor": 16, + "original_max": 65536, + "beta_fast": 32, + "beta_slow": 1 + }, + "kv_cache_dtype": "fp8", + "compress_ratio": 128, + "proj": { + "wkv": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 128 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + } + }, + "wo_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 128 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + } + }, + "wq_a": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 128 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + } + }, + "wq_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 128 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + } + } + }, + "batch": { + "groups": [ + { + "count": 1, + "q": 4096, + "ctx": 0 + } + ] + }, + "parallel": { + "tp": 8 + } + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro/attn" + ] + }, + { + "type": "dsv4_attn", + "args": { + "hidden": 7168, + "heads": 128, + "head_dim": 512, + "rope_dim": 64, + "q_lora_rank": 1536, + "window": 128, + "o_groups": 16, + "o_lora_rank": 1024, + "compress_rope_theta": 160000, + "rope_scaling": { + "type": "yarn", + "factor": 16, + "original_max": 65536, + "beta_fast": 32, + "beta_slow": 1 + }, + "kv_cache_dtype": "fp8", + "compress_ratio": 128, + "proj": { + "wkv": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 128 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + } + }, + "wo_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 128 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + } + }, + "wq_a": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 128 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + } + }, + "wq_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 128 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + } + } + }, + "batch": { + "groups": [ + { + "count": 1, + "q": 4096, + "ctx": 0 + } + ] + }, + "parallel": { + "dp": 4 + } + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro/attn" + ] + }, + { + "type": "dsv4_attn", + "args": { + "hidden": 7168, + "heads": 128, + "head_dim": 512, + "rope_dim": 64, + "q_lora_rank": 1536, + "window": 128, + "o_groups": 16, + "o_lora_rank": 1024, + "compress_rope_theta": 160000, + "rope_scaling": { + "type": "yarn", + "factor": 16, + "original_max": 65536, + "beta_fast": 32, + "beta_slow": 1 + }, + "kv_cache_dtype": "fp8", + "compress_ratio": 128, + "proj": { + "wkv": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 128 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + } + }, + "wo_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 128 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + } + }, + "wq_a": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 128 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + } + }, + "wq_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 128 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + } + } + }, + "batch": { + "groups": [ + { + "count": 1, + "q": 4096, + "ctx": 0 + } + ] + }, + "parallel": { + "dp": 8 + } + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro/attn" + ] + }, + { + "type": "dsv4_attn", + "args": { + "hidden": 5120, + "heads": 64, + "head_dim": 512, + "rope_dim": 64, + "q_lora_rank": 1280, + "window": 128, + "o_groups": 8, + "o_lora_rank": 1024, + "compress_ratio": 0, + "proj": { + "wkv": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + }, + "wo_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + }, + "wq_a": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + }, + "wq_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + } + }, + "batch": { + "groups": [ + { + "count": 16, + "q": 1, + "ctx": 16384 + } + ] + }, + "parallel": { + "tp": 2 + } + }, + "sources": [ + "deepseek-ai/DeepSeek-V4.1-Flash/attn" + ] + }, + { + "type": "dsv4_attn", + "args": { + "hidden": 5120, + "heads": 64, + "head_dim": 512, + "rope_dim": 64, + "q_lora_rank": 1280, + "window": 128, + "o_groups": 8, + "o_lora_rank": 1024, + "compress_ratio": 0, + "proj": { + "wkv": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + }, + "wo_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + }, + "wq_a": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + }, + "wq_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + } + }, + "batch": { + "groups": [ + { + "count": 16, + "q": 1, + "ctx": 16384 + } + ] + }, + "parallel": { + "tp": 4 + } + }, + "sources": [ + "deepseek-ai/DeepSeek-V4.1-Flash/attn" + ] + }, + { + "type": "dsv4_attn", + "args": { + "hidden": 5120, + "heads": 64, + "head_dim": 512, + "rope_dim": 64, + "q_lora_rank": 1280, + "window": 128, + "o_groups": 8, + "o_lora_rank": 1024, + "compress_ratio": 0, + "proj": { + "wkv": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + }, + "wo_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + }, + "wq_a": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + }, + "wq_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + } + }, + "batch": { + "groups": [ + { + "count": 16, + "q": 1, + "ctx": 16384 + } + ] + }, + "parallel": { + "tp": 8 + } + }, + "sources": [ + "deepseek-ai/DeepSeek-V4.1-Flash/attn" + ] + }, + { + "type": "dsv4_attn", + "args": { + "hidden": 5120, + "heads": 64, + "head_dim": 512, + "rope_dim": 64, + "q_lora_rank": 1280, + "window": 128, + "o_groups": 8, + "o_lora_rank": 1024, + "compress_ratio": 0, + "proj": { + "wkv": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + }, + "wo_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + }, + "wq_a": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + }, + "wq_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + } + }, + "batch": { + "groups": [ + { + "count": 16, + "q": 1, + "ctx": 16384 + } + ] + }, + "parallel": { + "dp": 8 + } + }, + "sources": [ + "deepseek-ai/DeepSeek-V4.1-Flash/attn" + ] + }, + { + "type": "dsv4_attn", + "args": { + "hidden": 5120, + "heads": 64, + "head_dim": 512, + "rope_dim": 64, + "q_lora_rank": 1280, + "window": 128, + "o_groups": 8, + "o_lora_rank": 1024, + "compress_ratio": 0, + "proj": { + "wkv": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + }, + "wo_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + }, + "wq_a": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + }, + "wq_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + } + }, + "batch": { + "groups": [ + { + "count": 1, + "q": 4096, + "ctx": 0 + } + ] + }, + "parallel": { + "tp": 2 + } + }, + "sources": [ + "deepseek-ai/DeepSeek-V4.1-Flash/attn" + ] + }, + { + "type": "dsv4_attn", + "args": { + "hidden": 5120, + "heads": 64, + "head_dim": 512, + "rope_dim": 64, + "q_lora_rank": 1280, + "window": 128, + "o_groups": 8, + "o_lora_rank": 1024, + "compress_ratio": 0, + "proj": { + "wkv": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + }, + "wo_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + }, + "wq_a": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + }, + "wq_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + } + }, + "batch": { + "groups": [ + { + "count": 1, + "q": 4096, + "ctx": 0 + } + ] + }, + "parallel": { + "tp": 4 + } + }, + "sources": [ + "deepseek-ai/DeepSeek-V4.1-Flash/attn" + ] + }, + { + "type": "dsv4_attn", + "args": { + "hidden": 5120, + "heads": 64, + "head_dim": 512, + "rope_dim": 64, + "q_lora_rank": 1280, + "window": 128, + "o_groups": 8, + "o_lora_rank": 1024, + "compress_ratio": 0, + "proj": { + "wkv": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + }, + "wo_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + }, + "wq_a": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + }, + "wq_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + } + }, + "batch": { + "groups": [ + { + "count": 1, + "q": 4096, + "ctx": 0 + } + ] + }, + "parallel": { + "tp": 8 + } + }, + "sources": [ + "deepseek-ai/DeepSeek-V4.1-Flash/attn" + ] + }, + { + "type": "dsv4_attn", + "args": { + "hidden": 5120, + "heads": 64, + "head_dim": 512, + "rope_dim": 64, + "q_lora_rank": 1280, + "window": 128, + "o_groups": 8, + "o_lora_rank": 1024, + "compress_ratio": 0, + "proj": { + "wkv": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + }, + "wo_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + }, + "wq_a": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + }, + "wq_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + } + }, + "batch": { + "groups": [ + { + "count": 1, + "q": 4096, + "ctx": 0 + } + ] + }, + "parallel": { + "dp": 8 + } + }, + "sources": [ + "deepseek-ai/DeepSeek-V4.1-Flash/attn" + ] + }, + { + "type": "dsv4_attn", + "args": { + "hidden": 5120, + "heads": 64, + "head_dim": 512, + "rope_dim": 64, + "q_lora_rank": 1280, + "window": 128, + "o_groups": 8, + "o_lora_rank": 1024, + "compress_ratio": 2, + "indexer": "own", + "compress_rope_theta": 160000, + "rope_scaling": { + "type": "yarn", + "factor": 16, + "original_max": 65536, + "beta_fast": 32, + "beta_slow": 1 + }, + "topk": 512, + "index_heads": 32, + "index_dim": 128, + "proj": { + "indexer.wq_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + }, + "wkv": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + }, + "wo_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + }, + "wq_a": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + }, + "wq_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + } + }, + "batch": { + "groups": [ + { + "count": 16, + "q": 1, + "ctx": 16384 + } + ] + }, + "parallel": { + "tp": 2 + } + }, + "sources": [ + "deepseek-ai/DeepSeek-V4.1-Flash/attn" + ] + }, + { + "type": "dsv4_attn", + "args": { + "hidden": 5120, + "heads": 64, + "head_dim": 512, + "rope_dim": 64, + "q_lora_rank": 1280, + "window": 128, + "o_groups": 8, + "o_lora_rank": 1024, + "compress_ratio": 2, + "indexer": "own", + "compress_rope_theta": 160000, + "rope_scaling": { + "type": "yarn", + "factor": 16, + "original_max": 65536, + "beta_fast": 32, + "beta_slow": 1 + }, + "topk": 512, + "index_heads": 32, + "index_dim": 128, + "proj": { + "indexer.wq_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + }, + "wkv": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + }, + "wo_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + }, + "wq_a": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + }, + "wq_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + } + }, + "batch": { + "groups": [ + { + "count": 16, + "q": 1, + "ctx": 16384 + } + ] + }, + "parallel": { + "tp": 4 + } + }, + "sources": [ + "deepseek-ai/DeepSeek-V4.1-Flash/attn" + ] + }, + { + "type": "dsv4_attn", + "args": { + "hidden": 5120, + "heads": 64, + "head_dim": 512, + "rope_dim": 64, + "q_lora_rank": 1280, + "window": 128, + "o_groups": 8, + "o_lora_rank": 1024, + "compress_ratio": 2, + "indexer": "own", + "compress_rope_theta": 160000, + "rope_scaling": { + "type": "yarn", + "factor": 16, + "original_max": 65536, + "beta_fast": 32, + "beta_slow": 1 + }, + "topk": 512, + "index_heads": 32, + "index_dim": 128, + "proj": { + "indexer.wq_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + }, + "wkv": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + }, + "wo_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + }, + "wq_a": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + }, + "wq_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + } + }, + "batch": { + "groups": [ + { + "count": 16, + "q": 1, + "ctx": 16384 + } + ] + }, + "parallel": { + "tp": 8 + } + }, + "sources": [ + "deepseek-ai/DeepSeek-V4.1-Flash/attn" + ] + }, + { + "type": "dsv4_attn", + "args": { + "hidden": 5120, + "heads": 64, + "head_dim": 512, + "rope_dim": 64, + "q_lora_rank": 1280, + "window": 128, + "o_groups": 8, + "o_lora_rank": 1024, + "compress_ratio": 2, + "indexer": "own", + "compress_rope_theta": 160000, + "rope_scaling": { + "type": "yarn", + "factor": 16, + "original_max": 65536, + "beta_fast": 32, + "beta_slow": 1 + }, + "topk": 512, + "index_heads": 32, + "index_dim": 128, + "proj": { + "indexer.wq_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + }, + "wkv": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + }, + "wo_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + }, + "wq_a": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + }, + "wq_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + } + }, + "batch": { + "groups": [ + { + "count": 16, + "q": 1, + "ctx": 16384 + } + ] + }, + "parallel": { + "dp": 8 + } + }, + "sources": [ + "deepseek-ai/DeepSeek-V4.1-Flash/attn" + ] + }, + { + "type": "dsv4_attn", + "args": { + "hidden": 5120, + "heads": 64, + "head_dim": 512, + "rope_dim": 64, + "q_lora_rank": 1280, + "window": 128, + "o_groups": 8, + "o_lora_rank": 1024, + "compress_ratio": 2, + "indexer": "own", + "compress_rope_theta": 160000, + "rope_scaling": { + "type": "yarn", + "factor": 16, + "original_max": 65536, + "beta_fast": 32, + "beta_slow": 1 + }, + "topk": 512, + "index_heads": 32, + "index_dim": 128, + "proj": { + "indexer.wq_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + }, + "wkv": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + }, + "wo_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + }, + "wq_a": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + }, + "wq_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + } + }, + "batch": { + "groups": [ + { + "count": 1, + "q": 4096, + "ctx": 0 + } + ] + }, + "parallel": { + "tp": 2 + } + }, + "sources": [ + "deepseek-ai/DeepSeek-V4.1-Flash/attn" + ] + }, + { + "type": "dsv4_attn", + "args": { + "hidden": 5120, + "heads": 64, + "head_dim": 512, + "rope_dim": 64, + "q_lora_rank": 1280, + "window": 128, + "o_groups": 8, + "o_lora_rank": 1024, + "compress_ratio": 2, + "indexer": "own", + "compress_rope_theta": 160000, + "rope_scaling": { + "type": "yarn", + "factor": 16, + "original_max": 65536, + "beta_fast": 32, + "beta_slow": 1 + }, + "topk": 512, + "index_heads": 32, + "index_dim": 128, + "proj": { + "indexer.wq_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + }, + "wkv": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + }, + "wo_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + }, + "wq_a": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + }, + "wq_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + } + }, + "batch": { + "groups": [ + { + "count": 1, + "q": 4096, + "ctx": 0 + } + ] + }, + "parallel": { + "tp": 4 + } + }, + "sources": [ + "deepseek-ai/DeepSeek-V4.1-Flash/attn" + ] + }, + { + "type": "dsv4_attn", + "args": { + "hidden": 5120, + "heads": 64, + "head_dim": 512, + "rope_dim": 64, + "q_lora_rank": 1280, + "window": 128, + "o_groups": 8, + "o_lora_rank": 1024, + "compress_ratio": 2, + "indexer": "own", + "compress_rope_theta": 160000, + "rope_scaling": { + "type": "yarn", + "factor": 16, + "original_max": 65536, + "beta_fast": 32, + "beta_slow": 1 + }, + "topk": 512, + "index_heads": 32, + "index_dim": 128, + "proj": { + "indexer.wq_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + }, + "wkv": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + }, + "wo_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + }, + "wq_a": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + }, + "wq_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + } + }, + "batch": { + "groups": [ + { + "count": 1, + "q": 4096, + "ctx": 0 + } + ] + }, + "parallel": { + "tp": 8 + } + }, + "sources": [ + "deepseek-ai/DeepSeek-V4.1-Flash/attn" + ] + }, + { + "type": "dsv4_attn", + "args": { + "hidden": 5120, + "heads": 64, + "head_dim": 512, + "rope_dim": 64, + "q_lora_rank": 1280, + "window": 128, + "o_groups": 8, + "o_lora_rank": 1024, + "compress_ratio": 2, + "indexer": "own", + "compress_rope_theta": 160000, + "rope_scaling": { + "type": "yarn", + "factor": 16, + "original_max": 65536, + "beta_fast": 32, + "beta_slow": 1 + }, + "topk": 512, + "index_heads": 32, + "index_dim": 128, + "proj": { + "indexer.wq_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + }, + "wkv": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + }, + "wo_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + }, + "wq_a": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + }, + "wq_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + } + }, + "batch": { + "groups": [ + { + "count": 1, + "q": 4096, + "ctx": 0 + } + ] + }, + "parallel": { + "dp": 8 + } + }, + "sources": [ + "deepseek-ai/DeepSeek-V4.1-Flash/attn" + ] + }, + { + "type": "dsv4_attn", + "args": { + "hidden": 5120, + "heads": 64, + "head_dim": 512, + "rope_dim": 64, + "q_lora_rank": 1280, + "window": 128, + "o_groups": 8, + "o_lora_rank": 1024, + "compress_ratio": 2, + "source": false, + "indexer": "reuse", + "compress_rope_theta": 160000, + "rope_scaling": { + "type": "yarn", + "factor": 16, + "original_max": 65536, + "beta_fast": 32, + "beta_slow": 1 + }, + "topk": 512, + "index_heads": 32, + "index_dim": 128, + "proj": { + "wkv": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + }, + "wo_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + }, + "wq_a": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + }, + "wq_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + } + }, + "batch": { + "groups": [ + { + "count": 16, + "q": 1, + "ctx": 16384 + } + ] + }, + "parallel": { + "tp": 2 + } + }, + "sources": [ + "deepseek-ai/DeepSeek-V4.1-Flash/attn" + ] + }, + { + "type": "dsv4_attn", + "args": { + "hidden": 5120, + "heads": 64, + "head_dim": 512, + "rope_dim": 64, + "q_lora_rank": 1280, + "window": 128, + "o_groups": 8, + "o_lora_rank": 1024, + "compress_ratio": 2, + "source": false, + "indexer": "reuse", + "compress_rope_theta": 160000, + "rope_scaling": { + "type": "yarn", + "factor": 16, + "original_max": 65536, + "beta_fast": 32, + "beta_slow": 1 + }, + "topk": 512, + "index_heads": 32, + "index_dim": 128, + "proj": { + "wkv": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + }, + "wo_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + }, + "wq_a": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + }, + "wq_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + } + }, + "batch": { + "groups": [ + { + "count": 16, + "q": 1, + "ctx": 16384 + } + ] + }, + "parallel": { + "tp": 4 + } + }, + "sources": [ + "deepseek-ai/DeepSeek-V4.1-Flash/attn" + ] + }, + { + "type": "dsv4_attn", + "args": { + "hidden": 5120, + "heads": 64, + "head_dim": 512, + "rope_dim": 64, + "q_lora_rank": 1280, + "window": 128, + "o_groups": 8, + "o_lora_rank": 1024, + "compress_ratio": 2, + "source": false, + "indexer": "reuse", + "compress_rope_theta": 160000, + "rope_scaling": { + "type": "yarn", + "factor": 16, + "original_max": 65536, + "beta_fast": 32, + "beta_slow": 1 + }, + "topk": 512, + "index_heads": 32, + "index_dim": 128, + "proj": { + "wkv": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + }, + "wo_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + }, + "wq_a": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + }, + "wq_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + } + }, + "batch": { + "groups": [ + { + "count": 16, + "q": 1, + "ctx": 16384 + } + ] + }, + "parallel": { + "tp": 8 + } + }, + "sources": [ + "deepseek-ai/DeepSeek-V4.1-Flash/attn" + ] + }, + { + "type": "dsv4_attn", + "args": { + "hidden": 5120, + "heads": 64, + "head_dim": 512, + "rope_dim": 64, + "q_lora_rank": 1280, + "window": 128, + "o_groups": 8, + "o_lora_rank": 1024, + "compress_ratio": 2, + "source": false, + "indexer": "reuse", + "compress_rope_theta": 160000, + "rope_scaling": { + "type": "yarn", + "factor": 16, + "original_max": 65536, + "beta_fast": 32, + "beta_slow": 1 + }, + "topk": 512, + "index_heads": 32, + "index_dim": 128, + "proj": { + "wkv": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + }, + "wo_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + }, + "wq_a": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + }, + "wq_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + } + }, + "batch": { + "groups": [ + { + "count": 16, + "q": 1, + "ctx": 16384 + } + ] + }, + "parallel": { + "dp": 8 + } + }, + "sources": [ + "deepseek-ai/DeepSeek-V4.1-Flash/attn" + ] + }, + { + "type": "dsv4_attn", + "args": { + "hidden": 5120, + "heads": 64, + "head_dim": 512, + "rope_dim": 64, + "q_lora_rank": 1280, + "window": 128, + "o_groups": 8, + "o_lora_rank": 1024, + "compress_ratio": 2, + "source": false, + "indexer": "reuse", + "compress_rope_theta": 160000, + "rope_scaling": { + "type": "yarn", + "factor": 16, + "original_max": 65536, + "beta_fast": 32, + "beta_slow": 1 + }, + "topk": 512, + "index_heads": 32, + "index_dim": 128, + "proj": { + "wkv": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + }, + "wo_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + }, + "wq_a": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + }, + "wq_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + } + }, + "batch": { + "groups": [ + { + "count": 1, + "q": 4096, + "ctx": 0 + } + ] + }, + "parallel": { + "tp": 2 + } + }, + "sources": [ + "deepseek-ai/DeepSeek-V4.1-Flash/attn" + ] + }, + { + "type": "dsv4_attn", + "args": { + "hidden": 5120, + "heads": 64, + "head_dim": 512, + "rope_dim": 64, + "q_lora_rank": 1280, + "window": 128, + "o_groups": 8, + "o_lora_rank": 1024, + "compress_ratio": 2, + "source": false, + "indexer": "reuse", + "compress_rope_theta": 160000, + "rope_scaling": { + "type": "yarn", + "factor": 16, + "original_max": 65536, + "beta_fast": 32, + "beta_slow": 1 + }, + "topk": 512, + "index_heads": 32, + "index_dim": 128, + "proj": { + "wkv": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + }, + "wo_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + }, + "wq_a": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + }, + "wq_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + } + }, + "batch": { + "groups": [ + { + "count": 1, + "q": 4096, + "ctx": 0 + } + ] + }, + "parallel": { + "tp": 4 + } + }, + "sources": [ + "deepseek-ai/DeepSeek-V4.1-Flash/attn" + ] + }, + { + "type": "dsv4_attn", + "args": { + "hidden": 5120, + "heads": 64, + "head_dim": 512, + "rope_dim": 64, + "q_lora_rank": 1280, + "window": 128, + "o_groups": 8, + "o_lora_rank": 1024, + "compress_ratio": 2, + "source": false, + "indexer": "reuse", + "compress_rope_theta": 160000, + "rope_scaling": { + "type": "yarn", + "factor": 16, + "original_max": 65536, + "beta_fast": 32, + "beta_slow": 1 + }, + "topk": 512, + "index_heads": 32, + "index_dim": 128, + "proj": { + "wkv": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + }, + "wo_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + }, + "wq_a": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + }, + "wq_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + } + }, + "batch": { + "groups": [ + { + "count": 1, + "q": 4096, + "ctx": 0 + } + ] + }, + "parallel": { + "tp": 8 + } + }, + "sources": [ + "deepseek-ai/DeepSeek-V4.1-Flash/attn" + ] + }, + { + "type": "dsv4_attn", + "args": { + "hidden": 5120, + "heads": 64, + "head_dim": 512, + "rope_dim": 64, + "q_lora_rank": 1280, + "window": 128, + "o_groups": 8, + "o_lora_rank": 1024, + "compress_ratio": 2, + "source": false, + "indexer": "reuse", + "compress_rope_theta": 160000, + "rope_scaling": { + "type": "yarn", + "factor": 16, + "original_max": 65536, + "beta_fast": 32, + "beta_slow": 1 + }, + "topk": 512, + "index_heads": 32, + "index_dim": 128, + "proj": { + "wkv": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + }, + "wo_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + }, + "wq_a": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + }, + "wq_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + } + }, + "batch": { + "groups": [ + { + "count": 1, + "q": 4096, + "ctx": 0 + } + ] + }, + "parallel": { + "dp": 8 + } + }, + "sources": [ + "deepseek-ai/DeepSeek-V4.1-Flash/attn" + ] + }, + { + "type": "dsv4_attn", + "args": { + "hidden": 5120, + "heads": 64, + "head_dim": 512, + "rope_dim": 64, + "q_lora_rank": 1280, + "window": 128, + "o_groups": 8, + "o_lora_rank": 1024, + "compress_ratio": 1, + "indexer": "own", + "compress_rope_theta": 160000, + "rope_scaling": { + "type": "yarn", + "factor": 16, + "original_max": 65536, + "beta_fast": 32, + "beta_slow": 1 + }, + "topk": 512, + "index_heads": 32, + "index_dim": 128, + "proj": { + "indexer.wq_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + }, + "wkv": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + }, + "wo_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + }, + "wq_a": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + }, + "wq_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + } + }, + "batch": { + "groups": [ + { + "count": 16, + "q": 1, + "ctx": 16384 + } + ] + }, + "parallel": { + "tp": 2 + } + }, + "sources": [ + "deepseek-ai/DeepSeek-V4.1-Flash/attn" + ] + }, + { + "type": "dsv4_attn", + "args": { + "hidden": 5120, + "heads": 64, + "head_dim": 512, + "rope_dim": 64, + "q_lora_rank": 1280, + "window": 128, + "o_groups": 8, + "o_lora_rank": 1024, + "compress_ratio": 1, + "indexer": "own", + "compress_rope_theta": 160000, + "rope_scaling": { + "type": "yarn", + "factor": 16, + "original_max": 65536, + "beta_fast": 32, + "beta_slow": 1 + }, + "topk": 512, + "index_heads": 32, + "index_dim": 128, + "proj": { + "indexer.wq_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + }, + "wkv": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + }, + "wo_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + }, + "wq_a": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + }, + "wq_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + } + }, + "batch": { + "groups": [ + { + "count": 16, + "q": 1, + "ctx": 16384 + } + ] + }, + "parallel": { + "tp": 4 + } + }, + "sources": [ + "deepseek-ai/DeepSeek-V4.1-Flash/attn" + ] + }, + { + "type": "dsv4_attn", + "args": { + "hidden": 5120, + "heads": 64, + "head_dim": 512, + "rope_dim": 64, + "q_lora_rank": 1280, + "window": 128, + "o_groups": 8, + "o_lora_rank": 1024, + "compress_ratio": 1, + "indexer": "own", + "compress_rope_theta": 160000, + "rope_scaling": { + "type": "yarn", + "factor": 16, + "original_max": 65536, + "beta_fast": 32, + "beta_slow": 1 + }, + "topk": 512, + "index_heads": 32, + "index_dim": 128, + "proj": { + "indexer.wq_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + }, + "wkv": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + }, + "wo_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + }, + "wq_a": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + }, + "wq_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + } + }, + "batch": { + "groups": [ + { + "count": 16, + "q": 1, + "ctx": 16384 + } + ] + }, + "parallel": { + "tp": 8 + } + }, + "sources": [ + "deepseek-ai/DeepSeek-V4.1-Flash/attn" + ] + }, + { + "type": "dsv4_attn", + "args": { + "hidden": 5120, + "heads": 64, + "head_dim": 512, + "rope_dim": 64, + "q_lora_rank": 1280, + "window": 128, + "o_groups": 8, + "o_lora_rank": 1024, + "compress_ratio": 1, + "indexer": "own", + "compress_rope_theta": 160000, + "rope_scaling": { + "type": "yarn", + "factor": 16, + "original_max": 65536, + "beta_fast": 32, + "beta_slow": 1 + }, + "topk": 512, + "index_heads": 32, + "index_dim": 128, + "proj": { + "indexer.wq_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + }, + "wkv": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + }, + "wo_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + }, + "wq_a": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + }, + "wq_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + } + }, + "batch": { + "groups": [ + { + "count": 16, + "q": 1, + "ctx": 16384 + } + ] + }, + "parallel": { + "dp": 8 + } + }, + "sources": [ + "deepseek-ai/DeepSeek-V4.1-Flash/attn" + ] + }, + { + "type": "dsv4_attn", + "args": { + "hidden": 5120, + "heads": 64, + "head_dim": 512, + "rope_dim": 64, + "q_lora_rank": 1280, + "window": 128, + "o_groups": 8, + "o_lora_rank": 1024, + "compress_ratio": 1, + "indexer": "own", + "compress_rope_theta": 160000, + "rope_scaling": { + "type": "yarn", + "factor": 16, + "original_max": 65536, + "beta_fast": 32, + "beta_slow": 1 + }, + "topk": 512, + "index_heads": 32, + "index_dim": 128, + "proj": { + "indexer.wq_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + }, + "wkv": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + }, + "wo_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + }, + "wq_a": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + }, + "wq_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + } + }, + "batch": { + "groups": [ + { + "count": 1, + "q": 4096, + "ctx": 0 + } + ] + }, + "parallel": { + "tp": 2 + } + }, + "sources": [ + "deepseek-ai/DeepSeek-V4.1-Flash/attn" + ] + }, + { + "type": "dsv4_attn", + "args": { + "hidden": 5120, + "heads": 64, + "head_dim": 512, + "rope_dim": 64, + "q_lora_rank": 1280, + "window": 128, + "o_groups": 8, + "o_lora_rank": 1024, + "compress_ratio": 1, + "indexer": "own", + "compress_rope_theta": 160000, + "rope_scaling": { + "type": "yarn", + "factor": 16, + "original_max": 65536, + "beta_fast": 32, + "beta_slow": 1 + }, + "topk": 512, + "index_heads": 32, + "index_dim": 128, + "proj": { + "indexer.wq_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + }, + "wkv": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + }, + "wo_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + }, + "wq_a": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + }, + "wq_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + } + }, + "batch": { + "groups": [ + { + "count": 1, + "q": 4096, + "ctx": 0 + } + ] + }, + "parallel": { + "tp": 4 + } + }, + "sources": [ + "deepseek-ai/DeepSeek-V4.1-Flash/attn" + ] + }, + { + "type": "dsv4_attn", + "args": { + "hidden": 5120, + "heads": 64, + "head_dim": 512, + "rope_dim": 64, + "q_lora_rank": 1280, + "window": 128, + "o_groups": 8, + "o_lora_rank": 1024, + "compress_ratio": 1, + "indexer": "own", + "compress_rope_theta": 160000, + "rope_scaling": { + "type": "yarn", + "factor": 16, + "original_max": 65536, + "beta_fast": 32, + "beta_slow": 1 + }, + "topk": 512, + "index_heads": 32, + "index_dim": 128, + "proj": { + "indexer.wq_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + }, + "wkv": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + }, + "wo_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + }, + "wq_a": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + }, + "wq_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + } + }, + "batch": { + "groups": [ + { + "count": 1, + "q": 4096, + "ctx": 0 + } + ] + }, + "parallel": { + "tp": 8 + } + }, + "sources": [ + "deepseek-ai/DeepSeek-V4.1-Flash/attn" + ] + }, + { + "type": "dsv4_attn", + "args": { + "hidden": 5120, + "heads": 64, + "head_dim": 512, + "rope_dim": 64, + "q_lora_rank": 1280, + "window": 128, + "o_groups": 8, + "o_lora_rank": 1024, + "compress_ratio": 1, + "indexer": "own", + "compress_rope_theta": 160000, + "rope_scaling": { + "type": "yarn", + "factor": 16, + "original_max": 65536, + "beta_fast": 32, + "beta_slow": 1 + }, + "topk": 512, + "index_heads": 32, + "index_dim": 128, + "proj": { + "indexer.wq_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + }, + "wkv": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + }, + "wo_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + }, + "wq_a": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + }, + "wq_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + } + }, + "batch": { + "groups": [ + { + "count": 1, + "q": 4096, + "ctx": 0 + } + ] + }, + "parallel": { + "dp": 8 + } + }, + "sources": [ + "deepseek-ai/DeepSeek-V4.1-Flash/attn" + ] + }, + { + "type": "dsv4_attn", + "args": { + "hidden": 5120, + "heads": 64, + "head_dim": 512, + "rope_dim": 64, + "q_lora_rank": 1280, + "window": 128, + "o_groups": 8, + "o_lora_rank": 1024, + "compress_ratio": 1, + "source": false, + "indexer": "reuse", + "compress_rope_theta": 160000, + "rope_scaling": { + "type": "yarn", + "factor": 16, + "original_max": 65536, + "beta_fast": 32, + "beta_slow": 1 + }, + "topk": 512, + "index_heads": 32, + "index_dim": 128, + "proj": { + "wkv": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + }, + "wo_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + }, + "wq_a": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + }, + "wq_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + } + }, + "batch": { + "groups": [ + { + "count": 16, + "q": 1, + "ctx": 16384 + } + ] + }, + "parallel": { + "tp": 2 + } + }, + "sources": [ + "deepseek-ai/DeepSeek-V4.1-Flash/attn" + ] + }, + { + "type": "dsv4_attn", + "args": { + "hidden": 5120, + "heads": 64, + "head_dim": 512, + "rope_dim": 64, + "q_lora_rank": 1280, + "window": 128, + "o_groups": 8, + "o_lora_rank": 1024, + "compress_ratio": 1, + "source": false, + "indexer": "reuse", + "compress_rope_theta": 160000, + "rope_scaling": { + "type": "yarn", + "factor": 16, + "original_max": 65536, + "beta_fast": 32, + "beta_slow": 1 + }, + "topk": 512, + "index_heads": 32, + "index_dim": 128, + "proj": { + "wkv": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + }, + "wo_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + }, + "wq_a": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + }, + "wq_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + } + }, + "batch": { + "groups": [ + { + "count": 16, + "q": 1, + "ctx": 16384 + } + ] + }, + "parallel": { + "tp": 4 + } + }, + "sources": [ + "deepseek-ai/DeepSeek-V4.1-Flash/attn" + ] + }, + { + "type": "dsv4_attn", + "args": { + "hidden": 5120, + "heads": 64, + "head_dim": 512, + "rope_dim": 64, + "q_lora_rank": 1280, + "window": 128, + "o_groups": 8, + "o_lora_rank": 1024, + "compress_ratio": 1, + "source": false, + "indexer": "reuse", + "compress_rope_theta": 160000, + "rope_scaling": { + "type": "yarn", + "factor": 16, + "original_max": 65536, + "beta_fast": 32, + "beta_slow": 1 + }, + "topk": 512, + "index_heads": 32, + "index_dim": 128, + "proj": { + "wkv": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + }, + "wo_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + }, + "wq_a": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + }, + "wq_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + } + }, + "batch": { + "groups": [ + { + "count": 16, + "q": 1, + "ctx": 16384 + } + ] + }, + "parallel": { + "tp": 8 + } + }, + "sources": [ + "deepseek-ai/DeepSeek-V4.1-Flash/attn" + ] + }, + { + "type": "dsv4_attn", + "args": { + "hidden": 5120, + "heads": 64, + "head_dim": 512, + "rope_dim": 64, + "q_lora_rank": 1280, + "window": 128, + "o_groups": 8, + "o_lora_rank": 1024, + "compress_ratio": 1, + "source": false, + "indexer": "reuse", + "compress_rope_theta": 160000, + "rope_scaling": { + "type": "yarn", + "factor": 16, + "original_max": 65536, + "beta_fast": 32, + "beta_slow": 1 + }, + "topk": 512, + "index_heads": 32, + "index_dim": 128, + "proj": { + "wkv": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + }, + "wo_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + }, + "wq_a": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + }, + "wq_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + } + }, + "batch": { + "groups": [ + { + "count": 16, + "q": 1, + "ctx": 16384 + } + ] + }, + "parallel": { + "dp": 8 + } + }, + "sources": [ + "deepseek-ai/DeepSeek-V4.1-Flash/attn" + ] + }, + { + "type": "dsv4_attn", + "args": { + "hidden": 5120, + "heads": 64, + "head_dim": 512, + "rope_dim": 64, + "q_lora_rank": 1280, + "window": 128, + "o_groups": 8, + "o_lora_rank": 1024, + "compress_ratio": 1, + "source": false, + "indexer": "reuse", + "compress_rope_theta": 160000, + "rope_scaling": { + "type": "yarn", + "factor": 16, + "original_max": 65536, + "beta_fast": 32, + "beta_slow": 1 + }, + "topk": 512, + "index_heads": 32, + "index_dim": 128, + "proj": { + "wkv": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + }, + "wo_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + }, + "wq_a": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + }, + "wq_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + } + }, + "batch": { + "groups": [ + { + "count": 1, + "q": 4096, + "ctx": 0 + } + ] + }, + "parallel": { + "tp": 2 + } + }, + "sources": [ + "deepseek-ai/DeepSeek-V4.1-Flash/attn" + ] + }, + { + "type": "dsv4_attn", + "args": { + "hidden": 5120, + "heads": 64, + "head_dim": 512, + "rope_dim": 64, + "q_lora_rank": 1280, + "window": 128, + "o_groups": 8, + "o_lora_rank": 1024, + "compress_ratio": 1, + "source": false, + "indexer": "reuse", + "compress_rope_theta": 160000, + "rope_scaling": { + "type": "yarn", + "factor": 16, + "original_max": 65536, + "beta_fast": 32, + "beta_slow": 1 + }, + "topk": 512, + "index_heads": 32, + "index_dim": 128, + "proj": { + "wkv": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + }, + "wo_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + }, + "wq_a": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + }, + "wq_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + } + }, + "batch": { + "groups": [ + { + "count": 1, + "q": 4096, + "ctx": 0 + } + ] + }, + "parallel": { + "tp": 4 + } + }, + "sources": [ + "deepseek-ai/DeepSeek-V4.1-Flash/attn" + ] + }, + { + "type": "dsv4_attn", + "args": { + "hidden": 5120, + "heads": 64, + "head_dim": 512, + "rope_dim": 64, + "q_lora_rank": 1280, + "window": 128, + "o_groups": 8, + "o_lora_rank": 1024, + "compress_ratio": 1, + "source": false, + "indexer": "reuse", + "compress_rope_theta": 160000, + "rope_scaling": { + "type": "yarn", + "factor": 16, + "original_max": 65536, + "beta_fast": 32, + "beta_slow": 1 + }, + "topk": 512, + "index_heads": 32, + "index_dim": 128, + "proj": { + "wkv": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + }, + "wo_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + }, + "wq_a": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + }, + "wq_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + } + }, + "batch": { + "groups": [ + { + "count": 1, + "q": 4096, + "ctx": 0 + } + ] + }, + "parallel": { + "tp": 8 + } + }, + "sources": [ + "deepseek-ai/DeepSeek-V4.1-Flash/attn" + ] + }, + { + "type": "dsv4_attn", + "args": { + "hidden": 5120, + "heads": 64, + "head_dim": 512, + "rope_dim": 64, + "q_lora_rank": 1280, + "window": 128, + "o_groups": 8, + "o_lora_rank": 1024, + "compress_ratio": 1, + "source": false, + "indexer": "reuse", + "compress_rope_theta": 160000, + "rope_scaling": { + "type": "yarn", + "factor": 16, + "original_max": 65536, + "beta_fast": 32, + "beta_slow": 1 + }, + "topk": 512, + "index_heads": 32, + "index_dim": 128, + "proj": { + "wkv": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + }, + "wo_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + }, + "wq_a": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + }, + "wq_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + } + }, + "batch": { + "groups": [ + { + "count": 1, + "q": 4096, + "ctx": 0 + } + ] + }, + "parallel": { + "dp": 8 + } + }, + "sources": [ + "deepseek-ai/DeepSeek-V4.1-Flash/attn" + ] + }, + { + "type": "dsv4_attn", + "args": { + "hidden": 5120, + "heads": 64, + "head_dim": 512, + "rope_dim": 64, + "q_lora_rank": 1280, + "window": 128, + "o_groups": 8, + "o_lora_rank": 1024, + "compress_ratio": 1, + "source": false, + "indexer": "shared", + "compress_rope_theta": 160000, + "rope_scaling": { + "type": "yarn", + "factor": 16, + "original_max": 65536, + "beta_fast": 32, + "beta_slow": 1 + }, + "topk": 512, + "index_heads": 32, + "index_dim": 128, + "proj": { + "indexer.wq_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + }, + "wkv": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + }, + "wo_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + }, + "wq_a": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + }, + "wq_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + } + }, + "batch": { + "groups": [ + { + "count": 16, + "q": 1, + "ctx": 16384 + } + ] + }, + "parallel": { + "tp": 2 + } + }, + "sources": [ + "deepseek-ai/DeepSeek-V4.1-Flash/attn" + ] + }, + { + "type": "dsv4_attn", + "args": { + "hidden": 5120, + "heads": 64, + "head_dim": 512, + "rope_dim": 64, + "q_lora_rank": 1280, + "window": 128, + "o_groups": 8, + "o_lora_rank": 1024, + "compress_ratio": 1, + "source": false, + "indexer": "shared", + "compress_rope_theta": 160000, + "rope_scaling": { + "type": "yarn", + "factor": 16, + "original_max": 65536, + "beta_fast": 32, + "beta_slow": 1 + }, + "topk": 512, + "index_heads": 32, + "index_dim": 128, + "proj": { + "indexer.wq_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + }, + "wkv": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + }, + "wo_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + }, + "wq_a": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + }, + "wq_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + } + }, + "batch": { + "groups": [ + { + "count": 16, + "q": 1, + "ctx": 16384 + } + ] + }, + "parallel": { + "tp": 4 + } + }, + "sources": [ + "deepseek-ai/DeepSeek-V4.1-Flash/attn" + ] + }, + { + "type": "dsv4_attn", + "args": { + "hidden": 5120, + "heads": 64, + "head_dim": 512, + "rope_dim": 64, + "q_lora_rank": 1280, + "window": 128, + "o_groups": 8, + "o_lora_rank": 1024, + "compress_ratio": 1, + "source": false, + "indexer": "shared", + "compress_rope_theta": 160000, + "rope_scaling": { + "type": "yarn", + "factor": 16, + "original_max": 65536, + "beta_fast": 32, + "beta_slow": 1 + }, + "topk": 512, + "index_heads": 32, + "index_dim": 128, + "proj": { + "indexer.wq_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + }, + "wkv": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + }, + "wo_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + }, + "wq_a": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + }, + "wq_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + } + }, + "batch": { + "groups": [ + { + "count": 16, + "q": 1, + "ctx": 16384 + } + ] + }, + "parallel": { + "tp": 8 + } + }, + "sources": [ + "deepseek-ai/DeepSeek-V4.1-Flash/attn" + ] + }, + { + "type": "dsv4_attn", + "args": { + "hidden": 5120, + "heads": 64, + "head_dim": 512, + "rope_dim": 64, + "q_lora_rank": 1280, + "window": 128, + "o_groups": 8, + "o_lora_rank": 1024, + "compress_ratio": 1, + "source": false, + "indexer": "shared", + "compress_rope_theta": 160000, + "rope_scaling": { + "type": "yarn", + "factor": 16, + "original_max": 65536, + "beta_fast": 32, + "beta_slow": 1 + }, + "topk": 512, + "index_heads": 32, + "index_dim": 128, + "proj": { + "indexer.wq_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + }, + "wkv": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + }, + "wo_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + }, + "wq_a": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + }, + "wq_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + } + }, + "batch": { + "groups": [ + { + "count": 16, + "q": 1, + "ctx": 16384 + } + ] + }, + "parallel": { + "dp": 8 + } + }, + "sources": [ + "deepseek-ai/DeepSeek-V4.1-Flash/attn" + ] + }, + { + "type": "dsv4_attn", + "args": { + "hidden": 5120, + "heads": 64, + "head_dim": 512, + "rope_dim": 64, + "q_lora_rank": 1280, + "window": 128, + "o_groups": 8, + "o_lora_rank": 1024, + "compress_ratio": 1, + "source": false, + "indexer": "shared", + "compress_rope_theta": 160000, + "rope_scaling": { + "type": "yarn", + "factor": 16, + "original_max": 65536, + "beta_fast": 32, + "beta_slow": 1 + }, + "topk": 512, + "index_heads": 32, + "index_dim": 128, + "proj": { + "indexer.wq_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + }, + "wkv": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + }, + "wo_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + }, + "wq_a": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + }, + "wq_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + } + }, + "batch": { + "groups": [ + { + "count": 1, + "q": 4096, + "ctx": 0 + } + ] + }, + "parallel": { + "tp": 2 + } + }, + "sources": [ + "deepseek-ai/DeepSeek-V4.1-Flash/attn" + ] + }, + { + "type": "dsv4_attn", + "args": { + "hidden": 5120, + "heads": 64, + "head_dim": 512, + "rope_dim": 64, + "q_lora_rank": 1280, + "window": 128, + "o_groups": 8, + "o_lora_rank": 1024, + "compress_ratio": 1, + "source": false, + "indexer": "shared", + "compress_rope_theta": 160000, + "rope_scaling": { + "type": "yarn", + "factor": 16, + "original_max": 65536, + "beta_fast": 32, + "beta_slow": 1 + }, + "topk": 512, + "index_heads": 32, + "index_dim": 128, + "proj": { + "indexer.wq_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + }, + "wkv": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + }, + "wo_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + }, + "wq_a": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + }, + "wq_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + } + }, + "batch": { + "groups": [ + { + "count": 1, + "q": 4096, + "ctx": 0 + } + ] + }, + "parallel": { + "tp": 4 + } + }, + "sources": [ + "deepseek-ai/DeepSeek-V4.1-Flash/attn" + ] + }, + { + "type": "dsv4_attn", + "args": { + "hidden": 5120, + "heads": 64, + "head_dim": 512, + "rope_dim": 64, + "q_lora_rank": 1280, + "window": 128, + "o_groups": 8, + "o_lora_rank": 1024, + "compress_ratio": 1, + "source": false, + "indexer": "shared", + "compress_rope_theta": 160000, + "rope_scaling": { + "type": "yarn", + "factor": 16, + "original_max": 65536, + "beta_fast": 32, + "beta_slow": 1 + }, + "topk": 512, + "index_heads": 32, + "index_dim": 128, + "proj": { + "indexer.wq_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + }, + "wkv": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + }, + "wo_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + }, + "wq_a": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + }, + "wq_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + } + }, + "batch": { + "groups": [ + { + "count": 1, + "q": 4096, + "ctx": 0 + } + ] + }, + "parallel": { + "tp": 8 + } + }, + "sources": [ + "deepseek-ai/DeepSeek-V4.1-Flash/attn" + ] + }, + { + "type": "dsv4_attn", + "args": { + "hidden": 5120, + "heads": 64, + "head_dim": 512, + "rope_dim": 64, + "q_lora_rank": 1280, + "window": 128, + "o_groups": 8, + "o_lora_rank": 1024, + "compress_ratio": 1, + "source": false, + "indexer": "shared", + "compress_rope_theta": 160000, + "rope_scaling": { + "type": "yarn", + "factor": 16, + "original_max": 65536, + "beta_fast": 32, + "beta_slow": 1 + }, + "topk": 512, + "index_heads": 32, + "index_dim": 128, + "proj": { + "indexer.wq_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + }, + "wkv": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + }, + "wo_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + }, + "wq_a": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + }, + "wq_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + } + }, + "batch": { + "groups": [ + { + "count": 1, + "q": 4096, + "ctx": 0 + } + ] + }, + "parallel": { + "dp": 8 + } + }, + "sources": [ + "deepseek-ai/DeepSeek-V4.1-Flash/attn" + ] + }, + { + "type": "mla", + "args": { + "hidden": 7168, + "heads": 96, + "q_lora_rank": 1536, + "kv_lora_rank": 512, + "nope": 128, + "rope_dim": 64, + "v": 128, + "rope": false, + "gate": true, + "kv_cache_dtype": "fp8", + "batch": { + "groups": [ + { + "count": 16, + "q": 1, + "ctx": 16384 + } + ] + }, + "parallel": { + "tp": 8 + } + }, + "sources": [ + "moonshotai/Kimi-K3/self_attn" + ] + }, + { + "type": "mla", + "args": { + "hidden": 7168, + "heads": 96, + "q_lora_rank": 1536, + "kv_lora_rank": 512, + "nope": 128, + "rope_dim": 64, + "v": 128, + "rope": false, + "gate": true, + "kv_cache_dtype": "fp8", + "batch": { + "groups": [ + { + "count": 16, + "q": 1, + "ctx": 16384 + } + ] + }, + "parallel": { + "tp": 8, + "dcp": 8 + } + }, + "sources": [ + "moonshotai/Kimi-K3/self_attn" + ] + }, + { + "type": "mla", + "args": { + "hidden": 7168, + "heads": 96, + "q_lora_rank": 1536, + "kv_lora_rank": 512, + "nope": 128, + "rope_dim": 64, + "v": 128, + "rope": false, + "gate": true, + "kv_cache_dtype": "fp8", + "batch": { + "groups": [ + { + "count": 1, + "q": 4096, + "ctx": 0 + } + ] + }, + "parallel": { + "tp": 8 + } + }, + "sources": [ + "moonshotai/Kimi-K3/self_attn" + ] + }, + { + "type": "mla", + "args": { + "hidden": 7168, + "heads": 96, + "q_lora_rank": 1536, + "kv_lora_rank": 512, + "nope": 128, + "rope_dim": 64, + "v": 128, + "rope": false, + "gate": true, + "kv_cache_dtype": "fp8", + "batch": { + "groups": [ + { + "count": 1, + "q": 4096, + "ctx": 0 + } + ] + }, + "parallel": { + "tp": 8, + "dcp": 8 + } + }, + "sources": [ + "moonshotai/Kimi-K3/self_attn" + ] + }, + { + "type": "kda", + "args": { + "hidden": 7168, + "heads": 96, + "head_dim": 128, + "conv_kernel": 4, + "state_dtype": "fp32", + "batch": { + "groups": [ + { + "count": 16, + "q": 1, + "ctx": 16384 + } + ] + }, + "parallel": { + "tp": 8 + } + }, + "sources": [ + "moonshotai/Kimi-K3/self_attn" + ] + }, + { + "type": "kda", + "args": { + "hidden": 7168, + "heads": 96, + "head_dim": 128, + "conv_kernel": 4, + "state_dtype": "fp32", + "batch": { + "groups": [ + { + "count": 1, + "q": 4096, + "ctx": 0 + } + ] + }, + "parallel": { + "tp": 8 + } + }, + "sources": [ + "moonshotai/Kimi-K3/self_attn" + ] + }, + { + "type": "kda", + "args": { + "hidden": 7168, + "heads": 96, + "head_dim": 128, + "conv_kernel": 4, + "state_dtype": "bf16", + "batch": { + "groups": [ + { + "count": 16, + "q": 1, + "ctx": 16384 + } + ] + }, + "parallel": { + "tp": 8 + } + }, + "sources": [ + "moonshotai/Kimi-K3/self_attn" + ] + }, + { + "type": "kda", + "args": { + "hidden": 7168, + "heads": 96, + "head_dim": 128, + "conv_kernel": 4, + "state_dtype": "bf16", + "batch": { + "groups": [ + { + "count": 1, + "q": 4096, + "ctx": 0 + } + ] + }, + "parallel": { + "tp": 8 + } + }, + "sources": [ + "moonshotai/Kimi-K3/self_attn" + ] + }, + { + "type": "gqa", + "args": { + "hidden": 6144, + "q_heads": 64, + "kv_heads": 4, + "head_dim": 128, + "rope_dim": 64, + "rope_theta": 5000000, + "qk_norm": true, + "kv_cache_dtype": "fp8", + "proj": { + "k_proj": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + } + }, + "o_proj": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + } + }, + "q_proj": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + } + }, + "v_proj": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + } + } + }, + "batch": { + "groups": [ + { + "count": 16, + "q": 1, + "ctx": 16384 + } + ] + }, + "parallel": { + "tp": 8 + } + }, + "sources": [ + "MiniMaxAI/MiniMax-M3-MXFP8/self_attn" + ] + }, + { + "type": "gqa", + "args": { + "hidden": 6144, + "q_heads": 64, + "kv_heads": 4, + "head_dim": 128, + "rope_dim": 64, + "rope_theta": 5000000, + "qk_norm": true, + "kv_cache_dtype": "fp8", + "proj": { + "k_proj": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + } + }, + "o_proj": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + } + }, + "q_proj": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + } + }, + "v_proj": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + } + } + }, + "batch": { + "groups": [ + { + "count": 16, + "q": 1, + "ctx": 16384 + } + ] + }, + "parallel": { + "tp": 2 + } + }, + "sources": [ + "nvidia/MiniMax-M3-NVFP4/self_attn" + ] + }, + { + "type": "gqa", + "args": { + "hidden": 6144, + "q_heads": 64, + "kv_heads": 4, + "head_dim": 128, + "rope_dim": 64, + "rope_theta": 5000000, + "qk_norm": true, + "kv_cache_dtype": "fp8", + "proj": { + "k_proj": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + } + }, + "o_proj": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + } + }, + "q_proj": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + } + }, + "v_proj": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + } + } + }, + "batch": { + "groups": [ + { + "count": 16, + "q": 1, + "ctx": 16384 + } + ] + }, + "parallel": { + "tp": 4 + } + }, + "sources": [ + "nvidia/MiniMax-M3-NVFP4/self_attn" + ] + }, + { + "type": "gqa", + "args": { + "hidden": 6144, + "q_heads": 64, + "kv_heads": 4, + "head_dim": 128, + "rope_dim": 64, + "rope_theta": 5000000, + "qk_norm": true, + "kv_cache_dtype": "fp8", + "proj": { + "k_proj": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + } + }, + "o_proj": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + } + }, + "q_proj": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + } + }, + "v_proj": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + } + } + }, + "batch": { + "groups": [ + { + "count": 1, + "q": 4096, + "ctx": 0 + } + ] + }, + "parallel": { + "tp": 8 + } + }, + "sources": [ + "MiniMaxAI/MiniMax-M3-MXFP8/self_attn" + ] + }, + { + "type": "gqa", + "args": { + "hidden": 6144, + "q_heads": 64, + "kv_heads": 4, + "head_dim": 128, + "rope_dim": 64, + "rope_theta": 5000000, + "qk_norm": true, + "kv_cache_dtype": "fp8", + "proj": { + "k_proj": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + } + }, + "o_proj": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + } + }, + "q_proj": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + } + }, + "v_proj": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + } + } + }, + "batch": { + "groups": [ + { + "count": 1, + "q": 4096, + "ctx": 0 + } + ] + }, + "parallel": { + "tp": 2 + } + }, + "sources": [ + "nvidia/MiniMax-M3-NVFP4/self_attn" + ] + }, + { + "type": "gqa", + "args": { + "hidden": 6144, + "q_heads": 64, + "kv_heads": 4, + "head_dim": 128, + "rope_dim": 64, + "rope_theta": 5000000, + "qk_norm": true, + "kv_cache_dtype": "fp8", + "proj": { + "k_proj": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + } + }, + "o_proj": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + } + }, + "q_proj": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + } + }, + "v_proj": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + } + } + }, + "batch": { + "groups": [ + { + "count": 1, + "q": 4096, + "ctx": 0 + } + ] + }, + "parallel": { + "tp": 4 + } + }, + "sources": [ + "nvidia/MiniMax-M3-NVFP4/self_attn" + ] + } +] \ No newline at end of file diff --git a/operatorx/testlists/collectives.json b/operatorx/testlists/collectives.json deleted file mode 100644 index 8051f87aec..0000000000 --- a/operatorx/testlists/collectives.json +++ /dev/null @@ -1,36490 +0,0 @@ -[ - { - "type": "allreduce", - "args": { - "num_elements": 262144, - "dtype": "bf16", - "world_size": 2 - } - }, - { - "type": "allreduce", - "args": { - "num_elements": 4194304, - "dtype": "bf16", - "world_size": 2 - } - }, - { - "type": "allreduce", - "args": { - "num_elements": 67108864, - "dtype": "bf16", - "world_size": 2 - } - }, - { - "type": "allreduce", - "args": { - "num_elements": 262144, - "dtype": "bf16", - "world_size": 4 - } - }, - { - "type": "allreduce", - "args": { - "num_elements": 4194304, - "dtype": "bf16", - "world_size": 4 - } - }, - { - "type": "allreduce", - "args": { - "num_elements": 67108864, - "dtype": "bf16", - "world_size": 4 - } - }, - { - "type": "allreduce", - "args": { - "num_elements": 262144, - "dtype": "bf16", - "world_size": 8 - } - }, - { - "type": "allreduce", - "args": { - "num_elements": 4194304, - "dtype": "bf16", - "world_size": 8 - } - }, - { - "type": "allreduce", - "args": { - "num_elements": 67108864, - "dtype": "bf16", - "world_size": 8 - } - }, - { - "type": "allreduce", - "args": { - "num_elements": 262144, - "dtype": "bf16", - "world_size": 16 - } - }, - { - "type": "allreduce", - "args": { - "num_elements": 4194304, - "dtype": "bf16", - "world_size": 16 - } - }, - { - "type": "allreduce", - "args": { - "num_elements": 67108864, - "dtype": "bf16", - "world_size": 16 - } - }, - { - "type": "allreduce", - "args": { - "num_elements": 262144, - "dtype": "bf16", - "world_size": 32 - } - }, - { - "type": "allreduce", - "args": { - "num_elements": 4194304, - "dtype": "bf16", - "world_size": 32 - } - }, - { - "type": "allreduce", - "args": { - "num_elements": 67108864, - "dtype": "bf16", - "world_size": 32 - } - }, - { - "type": "allgather", - "args": { - "num_elements_per_rank": 262144, - "dtype": "bf16", - "world_size": 2 - } - }, - { - "type": "allgather", - "args": { - "num_elements_per_rank": 4194304, - "dtype": "bf16", - "world_size": 2 - } - }, - { - "type": "allgather", - "args": { - "num_elements_per_rank": 67108864, - "dtype": "bf16", - "world_size": 2 - } - }, - { - "type": "allgather", - "args": { - "num_elements_per_rank": 262144, - "dtype": "bf16", - "world_size": 4 - } - }, - { - "type": "allgather", - "args": { - "num_elements_per_rank": 4194304, - "dtype": "bf16", - "world_size": 4 - } - }, - { - "type": "allgather", - "args": { - "num_elements_per_rank": 67108864, - "dtype": "bf16", - "world_size": 4 - } - }, - { - "type": "allgather", - "args": { - "num_elements_per_rank": 262144, - "dtype": "bf16", - "world_size": 8 - } - }, - { - "type": "allgather", - "args": { - "num_elements_per_rank": 4194304, - "dtype": "bf16", - "world_size": 8 - } - }, - { - "type": "allgather", - "args": { - "num_elements_per_rank": 67108864, - "dtype": "bf16", - "world_size": 8 - } - }, - { - "type": "allgather", - "args": { - "num_elements_per_rank": 262144, - "dtype": "bf16", - "world_size": 16 - } - }, - { - "type": "allgather", - "args": { - "num_elements_per_rank": 4194304, - "dtype": "bf16", - "world_size": 16 - } - }, - { - "type": "allgather", - "args": { - "num_elements_per_rank": 67108864, - "dtype": "bf16", - "world_size": 16 - } - }, - { - "type": "allgather", - "args": { - "num_elements_per_rank": 262144, - "dtype": "bf16", - "world_size": 32 - } - }, - { - "type": "allgather", - "args": { - "num_elements_per_rank": 4194304, - "dtype": "bf16", - "world_size": 32 - } - }, - { - "type": "allgather", - "args": { - "num_elements_per_rank": 67108864, - "dtype": "bf16", - "world_size": 32 - } - }, - { - "type": "reduce_scatter", - "args": { - "num_elements": 262144, - "dtype": "bf16", - "world_size": 2 - } - }, - { - "type": "reduce_scatter", - "args": { - "num_elements": 4194304, - "dtype": "bf16", - "world_size": 2 - } - }, - { - "type": "reduce_scatter", - "args": { - "num_elements": 67108864, - "dtype": "bf16", - "world_size": 2 - } - }, - { - "type": "reduce_scatter", - "args": { - "num_elements": 262144, - "dtype": "bf16", - "world_size": 4 - } - }, - { - "type": "reduce_scatter", - "args": { - "num_elements": 4194304, - "dtype": "bf16", - "world_size": 4 - } - }, - { - "type": "reduce_scatter", - "args": { - "num_elements": 67108864, - "dtype": "bf16", - "world_size": 4 - } - }, - { - "type": "reduce_scatter", - "args": { - "num_elements": 262144, - "dtype": "bf16", - "world_size": 8 - } - }, - { - "type": "reduce_scatter", - "args": { - "num_elements": 4194304, - "dtype": "bf16", - "world_size": 8 - } - }, - { - "type": "reduce_scatter", - "args": { - "num_elements": 67108864, - "dtype": "bf16", - "world_size": 8 - } - }, - { - "type": "reduce_scatter", - "args": { - "num_elements": 262144, - "dtype": "bf16", - "world_size": 16 - } - }, - { - "type": "reduce_scatter", - "args": { - "num_elements": 4194304, - "dtype": "bf16", - "world_size": 16 - } - }, - { - "type": "reduce_scatter", - "args": { - "num_elements": 67108864, - "dtype": "bf16", - "world_size": 16 - } - }, - { - "type": "reduce_scatter", - "args": { - "num_elements": 262144, - "dtype": "bf16", - "world_size": 32 - } - }, - { - "type": "reduce_scatter", - "args": { - "num_elements": 4194304, - "dtype": "bf16", - "world_size": 32 - } - }, - { - "type": "reduce_scatter", - "args": { - "num_elements": 67108864, - "dtype": "bf16", - "world_size": 32 - } - }, - { - "type": "alltoall", - "args": { - "num_elements_per_rank": 262144, - "dtype": "bf16", - "world_size": 2 - } - }, - { - "type": "alltoall", - "args": { - "num_elements_per_rank": 4194304, - "dtype": "bf16", - "world_size": 2 - } - }, - { - "type": "alltoall", - "args": { - "num_elements_per_rank": 67108864, - "dtype": "bf16", - "world_size": 2 - } - }, - { - "type": "alltoall", - "args": { - "num_elements_per_rank": 262144, - "dtype": "bf16", - "world_size": 4 - } - }, - { - "type": "alltoall", - "args": { - "num_elements_per_rank": 4194304, - "dtype": "bf16", - "world_size": 4 - } - }, - { - "type": "alltoall", - "args": { - "num_elements_per_rank": 67108864, - "dtype": "bf16", - "world_size": 4 - } - }, - { - "type": "alltoall", - "args": { - "num_elements_per_rank": 262144, - "dtype": "bf16", - "world_size": 8 - } - }, - { - "type": "alltoall", - "args": { - "num_elements_per_rank": 4194304, - "dtype": "bf16", - "world_size": 8 - } - }, - { - "type": "alltoall", - "args": { - "num_elements_per_rank": 67108864, - "dtype": "bf16", - "world_size": 8 - } - }, - { - "type": "alltoall", - "args": { - "num_elements_per_rank": 262144, - "dtype": "bf16", - "world_size": 16 - } - }, - { - "type": "alltoall", - "args": { - "num_elements_per_rank": 4194304, - "dtype": "bf16", - "world_size": 16 - } - }, - { - "type": "alltoall", - "args": { - "num_elements_per_rank": 67108864, - "dtype": "bf16", - "world_size": 16 - } - }, - { - "type": "alltoall", - "args": { - "num_elements_per_rank": 262144, - "dtype": "bf16", - "world_size": 32 - } - }, - { - "type": "alltoall", - "args": { - "num_elements_per_rank": 4194304, - "dtype": "bf16", - "world_size": 32 - } - }, - { - "type": "alltoall", - "args": { - "num_elements_per_rank": 67108864, - "dtype": "bf16", - "world_size": 32 - } - }, - { - "type": "allreduce", - "args": { - "num_elements": 4194304, - "dtype": "fp16", - "world_size": 8 - } - }, - { - "type": "allreduce", - "args": { - "num_elements": 4194304, - "dtype": "fp8", - "world_size": 8 - } - }, - { - "type": "allgather", - "args": { - "num_elements_per_rank": 4194304, - "dtype": "fp16", - "world_size": 8 - } - }, - { - "type": "allgather", - "args": { - "num_elements_per_rank": 4194304, - "dtype": "fp8", - "world_size": 8 - } - }, - { - "type": "reduce_scatter", - "args": { - "num_elements": 4194304, - "dtype": "fp16", - "world_size": 8 - } - }, - { - "type": "reduce_scatter", - "args": { - "num_elements": 4194304, - "dtype": "fp8", - "world_size": 8 - } - }, - { - "type": "alltoall", - "args": { - "num_elements_per_rank": 4194304, - "dtype": "fp16", - "world_size": 8 - } - }, - { - "type": "alltoall", - "args": { - "num_elements_per_rank": 4194304, - "dtype": "fp8", - "world_size": 8 - } - }, - { - "type": "allreduce", - "args": { - "num_elements": 4194304, - "dtype": "fp16", - "world_size": 16 - } - }, - { - "type": "allreduce", - "args": { - "num_elements": 4194304, - "dtype": "fp8", - "world_size": 16 - } - }, - { - "type": "allgather", - "args": { - "num_elements_per_rank": 4194304, - "dtype": "fp16", - "world_size": 16 - } - }, - { - "type": "allgather", - "args": { - "num_elements_per_rank": 4194304, - "dtype": "fp8", - "world_size": 16 - } - }, - { - "type": "reduce_scatter", - "args": { - "num_elements": 4194304, - "dtype": "fp16", - "world_size": 16 - } - }, - { - "type": "reduce_scatter", - "args": { - "num_elements": 4194304, - "dtype": "fp8", - "world_size": 16 - } - }, - { - "type": "alltoall", - "args": { - "num_elements_per_rank": 4194304, - "dtype": "fp16", - "world_size": 16 - } - }, - { - "type": "alltoall", - "args": { - "num_elements_per_rank": 4194304, - "dtype": "fp8", - "world_size": 16 - } - }, - { - "type": "allreduce", - "args": { - "num_elements": 4194304, - "dtype": "fp16", - "world_size": 32 - } - }, - { - "type": "allreduce", - "args": { - "num_elements": 4194304, - "dtype": "fp8", - "world_size": 32 - } - }, - { - "type": "allgather", - "args": { - "num_elements_per_rank": 4194304, - "dtype": "fp16", - "world_size": 32 - } - }, - { - "type": "allgather", - "args": { - "num_elements_per_rank": 4194304, - "dtype": "fp8", - "world_size": 32 - } - }, - { - "type": "reduce_scatter", - "args": { - "num_elements": 4194304, - "dtype": "fp16", - "world_size": 32 - } - }, - { - "type": "reduce_scatter", - "args": { - "num_elements": 4194304, - "dtype": "fp8", - "world_size": 32 - } - }, - { - "type": "alltoall", - "args": { - "num_elements_per_rank": 4194304, - "dtype": "fp16", - "world_size": 32 - } - }, - { - "type": "alltoall", - "args": { - "num_elements_per_rank": 4194304, - "dtype": "fp8", - "world_size": 32 - } - }, - { - "type": "allgather", - "args": { - "num_elements_per_rank": 2227699712, - "dtype": "bf16", - "world_size": 4 - } - }, - { - "type": "reduce_scatter", - "args": { - "num_elements": 8910798848, - "dtype": "bf16", - "world_size": 4 - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 310784, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 4, - "mode": "normal" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 310784, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 4, - "mode": "normal" - } - }, - { - "type": "allgather", - "args": { - "num_elements_per_rank": 4350976, - "dtype": "bf16", - "world_size": 4 - } - }, - { - "type": "reduce_scatter", - "args": { - "num_elements": 17403904, - "dtype": "bf16", - "world_size": 4 - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 607, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 4, - "mode": "low_latency" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 607, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 4, - "mode": "low_latency" - } - }, - { - "type": "allgather", - "args": { - "num_elements_per_rank": 1113849856, - "dtype": "bf16", - "world_size": 8 - } - }, - { - "type": "reduce_scatter", - "args": { - "num_elements": 8910798848, - "dtype": "bf16", - "world_size": 8 - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 155392, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 8, - "mode": "normal" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 155392, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 8, - "mode": "normal" - } - }, - { - "type": "allgather", - "args": { - "num_elements_per_rank": 2171904, - "dtype": "bf16", - "world_size": 8 - } - }, - { - "type": "reduce_scatter", - "args": { - "num_elements": 17375232, - "dtype": "bf16", - "world_size": 8 - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 303, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 8, - "mode": "low_latency" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 303, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 8, - "mode": "low_latency" - } - }, - { - "type": "allgather", - "args": { - "num_elements_per_rank": 1605632000, - "dtype": "bf16", - "world_size": 4 - } - }, - { - "type": "reduce_scatter", - "args": { - "num_elements": 6422528000, - "dtype": "bf16", - "world_size": 4 - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 224000, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 4, - "mode": "normal" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 224000, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 4, - "mode": "normal" - } - }, - { - "type": "allgather", - "args": { - "num_elements_per_rank": 3132416, - "dtype": "bf16", - "world_size": 4 - } - }, - { - "type": "reduce_scatter", - "args": { - "num_elements": 12529664, - "dtype": "bf16", - "world_size": 4 - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 437, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 4, - "mode": "low_latency" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 437, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 4, - "mode": "low_latency" - } - }, - { - "type": "allgather", - "args": { - "num_elements_per_rank": 802816000, - "dtype": "bf16", - "world_size": 8 - } - }, - { - "type": "reduce_scatter", - "args": { - "num_elements": 6422528000, - "dtype": "bf16", - "world_size": 8 - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 112000, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 8, - "mode": "normal" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 112000, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 8, - "mode": "normal" - } - }, - { - "type": "allgather", - "args": { - "num_elements_per_rank": 1562624, - "dtype": "bf16", - "world_size": 8 - } - }, - { - "type": "reduce_scatter", - "args": { - "num_elements": 12500992, - "dtype": "bf16", - "world_size": 8 - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 218, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 8, - "mode": "low_latency" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 218, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 8, - "mode": "low_latency" - } - }, - { - "type": "allgather", - "args": { - "num_elements_per_rank": 11010048, - "dtype": "bf16", - "world_size": 4 - } - }, - { - "type": "reduce_scatter", - "args": { - "num_elements": 44040192, - "dtype": "bf16", - "world_size": 4 - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 1536, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 4, - "mode": "normal" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 1536, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 4, - "mode": "normal" - } - }, - { - "type": "allgather", - "args": { - "num_elements_per_rank": 21504, - "dtype": "bf16", - "world_size": 4 - } - }, - { - "type": "reduce_scatter", - "args": { - "num_elements": 86016, - "dtype": "bf16", - "world_size": 4 - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 3, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 4, - "mode": "low_latency" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 3, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 4, - "mode": "low_latency" - } - }, - { - "type": "allreduce", - "args": { - "num_elements": 44040192, - "dtype": "bf16", - "world_size": 8 - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 6144, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 8, - "mode": "normal" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 6144, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 8, - "mode": "normal" - } - }, - { - "type": "allreduce", - "args": { - "num_elements": 86016, - "dtype": "bf16", - "world_size": 8 - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 12, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 8, - "mode": "low_latency" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 12, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 8, - "mode": "low_latency" - } - }, - { - "type": "allgather", - "args": { - "num_elements_per_rank": 18350080, - "dtype": "bf16", - "world_size": 4 - } - }, - { - "type": "reduce_scatter", - "args": { - "num_elements": 73400320, - "dtype": "bf16", - "world_size": 4 - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 2560, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 4, - "mode": "normal" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 2560, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 4, - "mode": "normal" - } - }, - { - "type": "allgather", - "args": { - "num_elements_per_rank": 35840, - "dtype": "bf16", - "world_size": 4 - } - }, - { - "type": "reduce_scatter", - "args": { - "num_elements": 143360, - "dtype": "bf16", - "world_size": 4 - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 5, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 4, - "mode": "low_latency" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 5, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 4, - "mode": "low_latency" - } - }, - { - "type": "allgather", - "args": { - "num_elements_per_rank": 27525120, - "dtype": "bf16", - "world_size": 4 - } - }, - { - "type": "reduce_scatter", - "args": { - "num_elements": 110100480, - "dtype": "bf16", - "world_size": 4 - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 3840, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 4, - "mode": "normal" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 3840, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 4, - "mode": "normal" - } - }, - { - "type": "allgather", - "args": { - "num_elements_per_rank": 50176, - "dtype": "bf16", - "world_size": 4 - } - }, - { - "type": "reduce_scatter", - "args": { - "num_elements": 200704, - "dtype": "bf16", - "world_size": 4 - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 7, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 4, - "mode": "low_latency" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 7, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 4, - "mode": "low_latency" - } - }, - { - "type": "allgather", - "args": { - "num_elements_per_rank": 45875200, - "dtype": "bf16", - "world_size": 4 - } - }, - { - "type": "reduce_scatter", - "args": { - "num_elements": 183500800, - "dtype": "bf16", - "world_size": 4 - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 6400, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 4, - "mode": "normal" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 6400, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 4, - "mode": "normal" - } - }, - { - "type": "allgather", - "args": { - "num_elements_per_rank": 86016, - "dtype": "bf16", - "world_size": 4 - } - }, - { - "type": "reduce_scatter", - "args": { - "num_elements": 344064, - "dtype": "bf16", - "world_size": 4 - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 12, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 4, - "mode": "low_latency" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 12, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 4, - "mode": "low_latency" - } - }, - { - "type": "allgather", - "args": { - "num_elements_per_rank": 82575360, - "dtype": "bf16", - "world_size": 4 - } - }, - { - "type": "reduce_scatter", - "args": { - "num_elements": 330301440, - "dtype": "bf16", - "world_size": 4 - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 11520, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 4, - "mode": "normal" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 11520, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 4, - "mode": "normal" - } - }, - { - "type": "allgather", - "args": { - "num_elements_per_rank": 157696, - "dtype": "bf16", - "world_size": 4 - } - }, - { - "type": "reduce_scatter", - "args": { - "num_elements": 630784, - "dtype": "bf16", - "world_size": 4 - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 22, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 4, - "mode": "low_latency" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 22, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 4, - "mode": "low_latency" - } - }, - { - "type": "allgather", - "args": { - "num_elements_per_rank": 165150720, - "dtype": "bf16", - "world_size": 4 - } - }, - { - "type": "reduce_scatter", - "args": { - "num_elements": 660602880, - "dtype": "bf16", - "world_size": 4 - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 23040, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 4, - "mode": "normal" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 23040, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 4, - "mode": "normal" - } - }, - { - "type": "allgather", - "args": { - "num_elements_per_rank": 322560, - "dtype": "bf16", - "world_size": 4 - } - }, - { - "type": "reduce_scatter", - "args": { - "num_elements": 1290240, - "dtype": "bf16", - "world_size": 4 - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 45, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 4, - "mode": "low_latency" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 45, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 4, - "mode": "low_latency" - } - }, - { - "type": "allgather", - "args": { - "num_elements_per_rank": 330301440, - "dtype": "bf16", - "world_size": 4 - } - }, - { - "type": "reduce_scatter", - "args": { - "num_elements": 1321205760, - "dtype": "bf16", - "world_size": 4 - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 46080, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 4, - "mode": "normal" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 46080, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 4, - "mode": "normal" - } - }, - { - "type": "allgather", - "args": { - "num_elements_per_rank": 645120, - "dtype": "bf16", - "world_size": 4 - } - }, - { - "type": "reduce_scatter", - "args": { - "num_elements": 2580480, - "dtype": "bf16", - "world_size": 4 - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 90, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 4, - "mode": "low_latency" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 90, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 4, - "mode": "low_latency" - } - }, - { - "type": "allreduce", - "args": { - "num_elements": 73400320, - "dtype": "bf16", - "world_size": 8 - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 10240, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 8, - "mode": "normal" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 10240, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 8, - "mode": "normal" - } - }, - { - "type": "allreduce", - "args": { - "num_elements": 143360, - "dtype": "bf16", - "world_size": 8 - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 20, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 8, - "mode": "low_latency" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 20, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 8, - "mode": "low_latency" - } - }, - { - "type": "allreduce", - "args": { - "num_elements": 110100480, - "dtype": "bf16", - "world_size": 8 - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 15360, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 8, - "mode": "normal" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 15360, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 8, - "mode": "normal" - } - }, - { - "type": "allreduce", - "args": { - "num_elements": 215040, - "dtype": "bf16", - "world_size": 8 - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 30, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 8, - "mode": "low_latency" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 30, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 8, - "mode": "low_latency" - } - }, - { - "type": "allreduce", - "args": { - "num_elements": 183500800, - "dtype": "bf16", - "world_size": 8 - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 25600, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 8, - "mode": "normal" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 25600, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 8, - "mode": "normal" - } - }, - { - "type": "allreduce", - "args": { - "num_elements": 358400, - "dtype": "bf16", - "world_size": 8 - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 50, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 8, - "mode": "low_latency" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 50, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 8, - "mode": "low_latency" - } - }, - { - "type": "allreduce", - "args": { - "num_elements": 330301440, - "dtype": "bf16", - "world_size": 8 - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 46080, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 8, - "mode": "normal" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 46080, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 8, - "mode": "normal" - } - }, - { - "type": "allreduce", - "args": { - "num_elements": 645120, - "dtype": "bf16", - "world_size": 8 - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 90, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 8, - "mode": "low_latency" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 90, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 8, - "mode": "low_latency" - } - }, - { - "type": "allreduce", - "args": { - "num_elements": 660602880, - "dtype": "bf16", - "world_size": 8 - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 92160, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 8, - "mode": "normal" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 92160, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 8, - "mode": "normal" - } - }, - { - "type": "allreduce", - "args": { - "num_elements": 1290240, - "dtype": "bf16", - "world_size": 8 - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 180, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 8, - "mode": "low_latency" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 180, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 8, - "mode": "low_latency" - } - }, - { - "type": "allreduce", - "args": { - "num_elements": 1321205760, - "dtype": "bf16", - "world_size": 8 - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 184320, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 8, - "mode": "normal" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 184320, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 8, - "mode": "normal" - } - }, - { - "type": "allreduce", - "args": { - "num_elements": 2580480, - "dtype": "bf16", - "world_size": 8 - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 360, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 8, - "mode": "low_latency" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 360, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 8, - "mode": "low_latency" - } - }, - { - "type": "allgather", - "args": { - "num_elements_per_rank": 9116319744, - "dtype": "bf16", - "world_size": 4 - } - }, - { - "type": "reduce_scatter", - "args": { - "num_elements": 36465278976, - "dtype": "bf16", - "world_size": 4 - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 1271808, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 4, - "mode": "normal" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 1271808, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 4, - "mode": "normal" - } - }, - { - "type": "allgather", - "args": { - "num_elements_per_rank": 17805312, - "dtype": "bf16", - "world_size": 4 - } - }, - { - "type": "reduce_scatter", - "args": { - "num_elements": 71221248, - "dtype": "bf16", - "world_size": 4 - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 2484, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 4, - "mode": "low_latency" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 2484, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 4, - "mode": "low_latency" - } - }, - { - "type": "allgather", - "args": { - "num_elements_per_rank": 4558159872, - "dtype": "bf16", - "world_size": 8 - } - }, - { - "type": "reduce_scatter", - "args": { - "num_elements": 36465278976, - "dtype": "bf16", - "world_size": 8 - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 635904, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 8, - "mode": "normal" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 635904, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 8, - "mode": "normal" - } - }, - { - "type": "allgather", - "args": { - "num_elements_per_rank": 8902656, - "dtype": "bf16", - "world_size": 8 - } - }, - { - "type": "reduce_scatter", - "args": { - "num_elements": 71221248, - "dtype": "bf16", - "world_size": 8 - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 1242, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 8, - "mode": "low_latency" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 1242, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 8, - "mode": "low_latency" - } - }, - { - "type": "allgather", - "args": { - "num_elements_per_rank": 19928186880, - "dtype": "bf16", - "world_size": 4 - } - }, - { - "type": "reduce_scatter", - "args": { - "num_elements": 79712747520, - "dtype": "bf16", - "world_size": 4 - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 2780160, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 4, - "mode": "normal" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 2780160, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 4, - "mode": "normal" - } - }, - { - "type": "allgather", - "args": { - "num_elements_per_rank": 38922240, - "dtype": "bf16", - "world_size": 4 - } - }, - { - "type": "reduce_scatter", - "args": { - "num_elements": 155688960, - "dtype": "bf16", - "world_size": 4 - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 5430, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 4, - "mode": "low_latency" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 5430, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 4, - "mode": "low_latency" - } - }, - { - "type": "allgather", - "args": { - "num_elements_per_rank": 7516192768, - "dtype": "bf16", - "world_size": 4 - } - }, - { - "type": "reduce_scatter", - "args": { - "num_elements": 30064771072, - "dtype": "bf16", - "world_size": 4 - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 1048576, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 4, - "mode": "normal" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 1048576, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 4, - "mode": "normal" - } - }, - { - "type": "allgather", - "args": { - "num_elements_per_rank": 7340032, - "dtype": "bf16", - "world_size": 4 - } - }, - { - "type": "reduce_scatter", - "args": { - "num_elements": 29360128, - "dtype": "bf16", - "world_size": 4 - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 1024, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 4, - "mode": "low_latency" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 1024, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 4, - "mode": "low_latency" - } - }, - { - "type": "allgather", - "args": { - "num_elements_per_rank": 3758096384, - "dtype": "bf16", - "world_size": 8 - } - }, - { - "type": "reduce_scatter", - "args": { - "num_elements": 30064771072, - "dtype": "bf16", - "world_size": 8 - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 524288, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 8, - "mode": "normal" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 524288, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 8, - "mode": "normal" - } - }, - { - "type": "allgather", - "args": { - "num_elements_per_rank": 3670016, - "dtype": "bf16", - "world_size": 8 - } - }, - { - "type": "reduce_scatter", - "args": { - "num_elements": 29360128, - "dtype": "bf16", - "world_size": 8 - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 512, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 8, - "mode": "low_latency" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 512, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 8, - "mode": "low_latency" - } - }, - { - "type": "allgather", - "args": { - "num_elements_per_rank": 4022337536, - "dtype": "bf16", - "world_size": 4 - } - }, - { - "type": "reduce_scatter", - "args": { - "num_elements": 16089350144, - "dtype": "bf16", - "world_size": 4 - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 561152, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 4, - "mode": "normal" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 561152, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 4, - "mode": "normal" - } - }, - { - "type": "allgather", - "args": { - "num_elements_per_rank": 3928064, - "dtype": "bf16", - "world_size": 4 - } - }, - { - "type": "reduce_scatter", - "args": { - "num_elements": 15712256, - "dtype": "bf16", - "world_size": 4 - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 548, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 4, - "mode": "low_latency" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 548, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 4, - "mode": "low_latency" - } - }, - { - "type": "allgather", - "args": { - "num_elements_per_rank": 2011168768, - "dtype": "bf16", - "world_size": 8 - } - }, - { - "type": "reduce_scatter", - "args": { - "num_elements": 16089350144, - "dtype": "bf16", - "world_size": 8 - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 280576, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 8, - "mode": "normal" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 280576, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 8, - "mode": "normal" - } - }, - { - "type": "allgather", - "args": { - "num_elements_per_rank": 1964032, - "dtype": "bf16", - "world_size": 8 - } - }, - { - "type": "reduce_scatter", - "args": { - "num_elements": 15712256, - "dtype": "bf16", - "world_size": 8 - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 274, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 8, - "mode": "low_latency" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 274, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 8, - "mode": "low_latency" - } - }, - { - "type": "allgather", - "args": { - "num_elements_per_rank": 2504785920, - "dtype": "bf16", - "world_size": 4 - } - }, - { - "type": "reduce_scatter", - "args": { - "num_elements": 10019143680, - "dtype": "bf16", - "world_size": 4 - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 349440, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 4, - "mode": "normal" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 349440, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 4, - "mode": "normal" - } - }, - { - "type": "allgather", - "args": { - "num_elements_per_rank": 2444288, - "dtype": "bf16", - "world_size": 4 - } - }, - { - "type": "reduce_scatter", - "args": { - "num_elements": 9777152, - "dtype": "bf16", - "world_size": 4 - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 341, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 4, - "mode": "low_latency" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 341, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 4, - "mode": "low_latency" - } - }, - { - "type": "allgather", - "args": { - "num_elements_per_rank": 1252392960, - "dtype": "bf16", - "world_size": 8 - } - }, - { - "type": "reduce_scatter", - "args": { - "num_elements": 10019143680, - "dtype": "bf16", - "world_size": 8 - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 174720, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 8, - "mode": "normal" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 174720, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 8, - "mode": "normal" - } - }, - { - "type": "allgather", - "args": { - "num_elements_per_rank": 1218560, - "dtype": "bf16", - "world_size": 8 - } - }, - { - "type": "reduce_scatter", - "args": { - "num_elements": 9748480, - "dtype": "bf16", - "world_size": 8 - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 170, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 8, - "mode": "low_latency" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 170, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 8, - "mode": "low_latency" - } - }, - { - "type": "allgather", - "args": { - "num_elements_per_rank": 7168, - "dtype": "bf16", - "world_size": 4 - } - }, - { - "type": "reduce_scatter", - "args": { - "num_elements": 28672, - "dtype": "bf16", - "world_size": 4 - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 1, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 4, - "mode": "low_latency" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 1, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 4, - "mode": "low_latency" - } - }, - { - "type": "allreduce", - "args": { - "num_elements": 43008, - "dtype": "bf16", - "world_size": 8 - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 6, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 8, - "mode": "low_latency" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 6, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 8, - "mode": "low_latency" - } - }, - { - "type": "allgather", - "args": { - "num_elements_per_rank": 14336, - "dtype": "bf16", - "world_size": 4 - } - }, - { - "type": "reduce_scatter", - "args": { - "num_elements": 57344, - "dtype": "bf16", - "world_size": 4 - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 2, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 4, - "mode": "low_latency" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 2, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 4, - "mode": "low_latency" - } - }, - { - "type": "allgather", - "args": { - "num_elements_per_rank": 43008, - "dtype": "bf16", - "world_size": 4 - } - }, - { - "type": "reduce_scatter", - "args": { - "num_elements": 172032, - "dtype": "bf16", - "world_size": 4 - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 6, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 4, - "mode": "low_latency" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 6, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 4, - "mode": "low_latency" - } - }, - { - "type": "allgather", - "args": { - "num_elements_per_rank": 78848, - "dtype": "bf16", - "world_size": 4 - } - }, - { - "type": "reduce_scatter", - "args": { - "num_elements": 315392, - "dtype": "bf16", - "world_size": 4 - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 11, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 4, - "mode": "low_latency" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 11, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 4, - "mode": "low_latency" - } - }, - { - "type": "allreduce", - "args": { - "num_elements": 71680, - "dtype": "bf16", - "world_size": 8 - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 10, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 8, - "mode": "low_latency" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 10, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 8, - "mode": "low_latency" - } - }, - { - "type": "allreduce", - "args": { - "num_elements": 107520, - "dtype": "bf16", - "world_size": 8 - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 15, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 8, - "mode": "low_latency" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 15, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 8, - "mode": "low_latency" - } - }, - { - "type": "allreduce", - "args": { - "num_elements": 179200, - "dtype": "bf16", - "world_size": 8 - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 25, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 8, - "mode": "low_latency" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 25, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 8, - "mode": "low_latency" - } - }, - { - "type": "allreduce", - "args": { - "num_elements": 322560, - "dtype": "bf16", - "world_size": 8 - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 45, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 8, - "mode": "low_latency" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 45, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 8, - "mode": "low_latency" - } - }, - { - "type": "allgather", - "args": { - "num_elements_per_rank": 825753600, - "dtype": "bf16", - "world_size": 4 - } - }, - { - "type": "reduce_scatter", - "args": { - "num_elements": 3303014400, - "dtype": "bf16", - "world_size": 4 - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 115200, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 4, - "mode": "normal" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 115200, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 4, - "mode": "normal" - } - }, - { - "type": "allgather", - "args": { - "num_elements_per_rank": 802816, - "dtype": "bf16", - "world_size": 4 - } - }, - { - "type": "reduce_scatter", - "args": { - "num_elements": 3211264, - "dtype": "bf16", - "world_size": 4 - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 112, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 4, - "mode": "low_latency" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 112, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 4, - "mode": "low_latency" - } - }, - { - "type": "allreduce", - "args": { - "num_elements": 3303014400, - "dtype": "bf16", - "world_size": 8 - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 460800, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 8, - "mode": "normal" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 460800, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 8, - "mode": "normal" - } - }, - { - "type": "allreduce", - "args": { - "num_elements": 3225600, - "dtype": "bf16", - "world_size": 8 - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 450, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 8, - "mode": "low_latency" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 450, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 8, - "mode": "low_latency" - } - }, - { - "type": "allgather", - "args": { - "num_elements_per_rank": 1321205760, - "dtype": "bf16", - "world_size": 4 - } - }, - { - "type": "reduce_scatter", - "args": { - "num_elements": 5284823040, - "dtype": "bf16", - "world_size": 4 - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 184320, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 4, - "mode": "normal" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 184320, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 4, - "mode": "normal" - } - }, - { - "type": "allgather", - "args": { - "num_elements_per_rank": 660602880, - "dtype": "bf16", - "world_size": 8 - } - }, - { - "type": "reduce_scatter", - "args": { - "num_elements": 5284823040, - "dtype": "bf16", - "world_size": 8 - } - }, - { - "type": "allgather", - "args": { - "num_elements_per_rank": 157696, - "dtype": "bf16", - "world_size": 8 - } - }, - { - "type": "reduce_scatter", - "args": { - "num_elements": 1261568, - "dtype": "bf16", - "world_size": 8 - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 22, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 8, - "mode": "low_latency" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 22, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 8, - "mode": "low_latency" - } - }, - { - "type": "allgather", - "args": { - "num_elements_per_rank": 968884224, - "dtype": "bf16", - "world_size": 4 - } - }, - { - "type": "reduce_scatter", - "args": { - "num_elements": 3875536896, - "dtype": "bf16", - "world_size": 4 - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 135168, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 4, - "mode": "normal" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 135168, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 4, - "mode": "normal" - } - }, - { - "type": "allgather", - "args": { - "num_elements_per_rank": 236544, - "dtype": "bf16", - "world_size": 4 - } - }, - { - "type": "reduce_scatter", - "args": { - "num_elements": 946176, - "dtype": "bf16", - "world_size": 4 - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 33, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 4, - "mode": "low_latency" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 33, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 4, - "mode": "low_latency" - } - }, - { - "type": "allreduce", - "args": { - "num_elements": 3875536896, - "dtype": "bf16", - "world_size": 8 - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 540672, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 8, - "mode": "normal" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 540672, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 8, - "mode": "normal" - } - }, - { - "type": "allreduce", - "args": { - "num_elements": 946176, - "dtype": "bf16", - "world_size": 8 - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 132, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 8, - "mode": "low_latency" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 132, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 8, - "mode": "low_latency" - } - }, - { - "type": "allgather", - "args": { - "num_elements_per_rank": 88080384, - "dtype": "bf16", - "world_size": 4 - } - }, - { - "type": "reduce_scatter", - "args": { - "num_elements": 352321536, - "dtype": "bf16", - "world_size": 4 - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 12288, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 4, - "mode": "normal" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 12288, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 4, - "mode": "normal" - } - }, - { - "type": "allreduce", - "args": { - "num_elements": 352321536, - "dtype": "bf16", - "world_size": 8 - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 49152, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 8, - "mode": "normal" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 49152, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 8, - "mode": "normal" - } - }, - { - "type": "allgather", - "args": { - "num_elements_per_rank": 146800640, - "dtype": "bf16", - "world_size": 4 - } - }, - { - "type": "reduce_scatter", - "args": { - "num_elements": 587202560, - "dtype": "bf16", - "world_size": 4 - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 20480, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 4, - "mode": "normal" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 20480, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 4, - "mode": "normal" - } - }, - { - "type": "allgather", - "args": { - "num_elements_per_rank": 220200960, - "dtype": "bf16", - "world_size": 4 - } - }, - { - "type": "reduce_scatter", - "args": { - "num_elements": 880803840, - "dtype": "bf16", - "world_size": 4 - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 30720, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 4, - "mode": "normal" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 30720, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 4, - "mode": "normal" - } - }, - { - "type": "allgather", - "args": { - "num_elements_per_rank": 440401920, - "dtype": "bf16", - "world_size": 4 - } - }, - { - "type": "reduce_scatter", - "args": { - "num_elements": 1761607680, - "dtype": "bf16", - "world_size": 4 - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 61440, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 4, - "mode": "normal" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 61440, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 4, - "mode": "normal" - } - }, - { - "type": "allgather", - "args": { - "num_elements_per_rank": 107520, - "dtype": "bf16", - "world_size": 4 - } - }, - { - "type": "reduce_scatter", - "args": { - "num_elements": 430080, - "dtype": "bf16", - "world_size": 4 - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 15, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 4, - "mode": "low_latency" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 15, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 4, - "mode": "low_latency" - } - }, - { - "type": "allgather", - "args": { - "num_elements_per_rank": 880803840, - "dtype": "bf16", - "world_size": 4 - } - }, - { - "type": "reduce_scatter", - "args": { - "num_elements": 3523215360, - "dtype": "bf16", - "world_size": 4 - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 122880, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 4, - "mode": "normal" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 122880, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 4, - "mode": "normal" - } - }, - { - "type": "allgather", - "args": { - "num_elements_per_rank": 215040, - "dtype": "bf16", - "world_size": 4 - } - }, - { - "type": "reduce_scatter", - "args": { - "num_elements": 860160, - "dtype": "bf16", - "world_size": 4 - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 30, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 4, - "mode": "low_latency" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 30, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 4, - "mode": "low_latency" - } - }, - { - "type": "allreduce", - "args": { - "num_elements": 587202560, - "dtype": "bf16", - "world_size": 8 - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 81920, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 8, - "mode": "normal" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 81920, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 8, - "mode": "normal" - } - }, - { - "type": "allreduce", - "args": { - "num_elements": 880803840, - "dtype": "bf16", - "world_size": 8 - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 122880, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 8, - "mode": "normal" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 122880, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 8, - "mode": "normal" - } - }, - { - "type": "allreduce", - "args": { - "num_elements": 1761607680, - "dtype": "bf16", - "world_size": 8 - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 245760, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 8, - "mode": "normal" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 245760, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 8, - "mode": "normal" - } - }, - { - "type": "allreduce", - "args": { - "num_elements": 430080, - "dtype": "bf16", - "world_size": 8 - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 60, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 8, - "mode": "low_latency" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 60, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 8, - "mode": "low_latency" - } - }, - { - "type": "allreduce", - "args": { - "num_elements": 3523215360, - "dtype": "bf16", - "world_size": 8 - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 491520, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 8, - "mode": "normal" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 491520, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 8, - "mode": "normal" - } - }, - { - "type": "allreduce", - "args": { - "num_elements": 860160, - "dtype": "bf16", - "world_size": 8 - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 120, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 8, - "mode": "low_latency" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 120, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 8, - "mode": "low_latency" - } - }, - { - "type": "allgather", - "args": { - "num_elements_per_rank": 8044675072, - "dtype": "bf16", - "world_size": 4 - } - }, - { - "type": "reduce_scatter", - "args": { - "num_elements": 32178700288, - "dtype": "bf16", - "world_size": 4 - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 1122304, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 4, - "mode": "normal" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 1122304, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 4, - "mode": "normal" - } - }, - { - "type": "allgather", - "args": { - "num_elements_per_rank": 1964032, - "dtype": "bf16", - "world_size": 4 - } - }, - { - "type": "reduce_scatter", - "args": { - "num_elements": 7856128, - "dtype": "bf16", - "world_size": 4 - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 274, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 4, - "mode": "low_latency" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 274, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 4, - "mode": "low_latency" - } - }, - { - "type": "allgather", - "args": { - "num_elements_per_rank": 4022337536, - "dtype": "bf16", - "world_size": 8 - } - }, - { - "type": "reduce_scatter", - "args": { - "num_elements": 32178700288, - "dtype": "bf16", - "world_size": 8 - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 561152, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 8, - "mode": "normal" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 561152, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 8, - "mode": "normal" - } - }, - { - "type": "allgather", - "args": { - "num_elements_per_rank": 982016, - "dtype": "bf16", - "world_size": 8 - } - }, - { - "type": "reduce_scatter", - "args": { - "num_elements": 7856128, - "dtype": "bf16", - "world_size": 8 - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 137, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 8, - "mode": "low_latency" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 137, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 8, - "mode": "low_latency" - } - }, - { - "type": "allgather", - "args": { - "num_elements_per_rank": 16089350144, - "dtype": "bf16", - "world_size": 4 - } - }, - { - "type": "reduce_scatter", - "args": { - "num_elements": 64357400576, - "dtype": "bf16", - "world_size": 4 - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 2244608, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 4, - "mode": "normal" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 2244608, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 4, - "mode": "normal" - } - }, - { - "type": "allgather", - "args": { - "num_elements_per_rank": 24823988224, - "dtype": "bf16", - "world_size": 4 - } - }, - { - "type": "reduce_scatter", - "args": { - "num_elements": 99295952896, - "dtype": "bf16", - "world_size": 4 - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 3463168, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 4, - "mode": "normal" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 3463168, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 4, - "mode": "normal" - } - }, - { - "type": "allgather", - "args": { - "num_elements_per_rank": 6056960, - "dtype": "bf16", - "world_size": 4 - } - }, - { - "type": "reduce_scatter", - "args": { - "num_elements": 24227840, - "dtype": "bf16", - "world_size": 4 - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 845, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 4, - "mode": "low_latency" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 845, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 4, - "mode": "low_latency" - } - }, - { - "type": "allgather", - "args": { - "num_elements_per_rank": 8044675072, - "dtype": "bf16", - "world_size": 8 - } - }, - { - "type": "reduce_scatter", - "args": { - "num_elements": 64357400576, - "dtype": "bf16", - "world_size": 8 - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 1122304, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 8, - "mode": "normal" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 1122304, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 8, - "mode": "normal" - } - }, - { - "type": "allgather", - "args": { - "num_elements_per_rank": 12411994112, - "dtype": "bf16", - "world_size": 8 - } - }, - { - "type": "reduce_scatter", - "args": { - "num_elements": 99295952896, - "dtype": "bf16", - "world_size": 8 - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 1731584, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 8, - "mode": "normal" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 1731584, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 8, - "mode": "normal" - } - }, - { - "type": "allgather", - "args": { - "num_elements_per_rank": 3024896, - "dtype": "bf16", - "world_size": 8 - } - }, - { - "type": "reduce_scatter", - "args": { - "num_elements": 24199168, - "dtype": "bf16", - "world_size": 8 - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 422, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 8, - "mode": "low_latency" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 422, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 8, - "mode": "low_latency" - } - }, - { - "type": "allgather", - "args": { - "num_elements_per_rank": 9659482112, - "dtype": "bf16", - "world_size": 4 - } - }, - { - "type": "reduce_scatter", - "args": { - "num_elements": 38637928448, - "dtype": "bf16", - "world_size": 4 - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 1347584, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 4, - "mode": "normal" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 1347584, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 4, - "mode": "normal" - } - }, - { - "type": "allgather", - "args": { - "num_elements_per_rank": 2358272, - "dtype": "bf16", - "world_size": 4 - } - }, - { - "type": "reduce_scatter", - "args": { - "num_elements": 9433088, - "dtype": "bf16", - "world_size": 4 - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 329, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 4, - "mode": "low_latency" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 329, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 4, - "mode": "low_latency" - } - }, - { - "type": "allgather", - "args": { - "num_elements_per_rank": 4829741056, - "dtype": "bf16", - "world_size": 8 - } - }, - { - "type": "reduce_scatter", - "args": { - "num_elements": 38637928448, - "dtype": "bf16", - "world_size": 8 - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 673792, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 8, - "mode": "normal" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 673792, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 8, - "mode": "normal" - } - }, - { - "type": "allgather", - "args": { - "num_elements_per_rank": 1175552, - "dtype": "bf16", - "world_size": 8 - } - }, - { - "type": "reduce_scatter", - "args": { - "num_elements": 9404416, - "dtype": "bf16", - "world_size": 8 - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 164, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 8, - "mode": "low_latency" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 164, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 8, - "mode": "low_latency" - } - }, - { - "type": "allgather", - "args": { - "num_elements_per_rank": 367001600, - "dtype": "bf16", - "world_size": 4 - } - }, - { - "type": "reduce_scatter", - "args": { - "num_elements": 1468006400, - "dtype": "bf16", - "world_size": 4 - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 51200, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 4, - "mode": "normal" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 51200, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 4, - "mode": "normal" - } - }, - { - "type": "allgather", - "args": { - "num_elements_per_rank": 734003200, - "dtype": "bf16", - "world_size": 4 - } - }, - { - "type": "reduce_scatter", - "args": { - "num_elements": 2936012800, - "dtype": "bf16", - "world_size": 4 - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 102400, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 4, - "mode": "normal" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 102400, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 4, - "mode": "normal" - } - }, - { - "type": "allgather", - "args": { - "num_elements_per_rank": 1468006400, - "dtype": "bf16", - "world_size": 4 - } - }, - { - "type": "reduce_scatter", - "args": { - "num_elements": 5872025600, - "dtype": "bf16", - "world_size": 4 - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 204800, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 4, - "mode": "normal" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 204800, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 4, - "mode": "normal" - } - }, - { - "type": "allgather", - "args": { - "num_elements_per_rank": 179200, - "dtype": "bf16", - "world_size": 4 - } - }, - { - "type": "reduce_scatter", - "args": { - "num_elements": 716800, - "dtype": "bf16", - "world_size": 4 - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 25, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 4, - "mode": "low_latency" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 25, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 4, - "mode": "low_latency" - } - }, - { - "type": "allreduce", - "args": { - "num_elements": 1468006400, - "dtype": "bf16", - "world_size": 8 - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 204800, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 8, - "mode": "normal" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 204800, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 8, - "mode": "normal" - } - }, - { - "type": "allreduce", - "args": { - "num_elements": 2936012800, - "dtype": "bf16", - "world_size": 8 - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 409600, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 8, - "mode": "normal" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 409600, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 8, - "mode": "normal" - } - }, - { - "type": "allreduce", - "args": { - "num_elements": 5872025600, - "dtype": "bf16", - "world_size": 8 - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 819200, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 8, - "mode": "normal" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 819200, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 8, - "mode": "normal" - } - }, - { - "type": "allreduce", - "args": { - "num_elements": 716800, - "dtype": "bf16", - "world_size": 8 - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 100, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 8, - "mode": "low_latency" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 100, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 8, - "mode": "low_latency" - } - }, - { - "type": "allgather", - "args": { - "num_elements_per_rank": 5431623680, - "dtype": "bf16", - "world_size": 4 - } - }, - { - "type": "reduce_scatter", - "args": { - "num_elements": 21726494720, - "dtype": "bf16", - "world_size": 4 - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 757760, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 4, - "mode": "normal" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 757760, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 4, - "mode": "normal" - } - }, - { - "type": "allgather", - "args": { - "num_elements_per_rank": 659456, - "dtype": "bf16", - "world_size": 4 - } - }, - { - "type": "reduce_scatter", - "args": { - "num_elements": 2637824, - "dtype": "bf16", - "world_size": 4 - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 92, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 4, - "mode": "low_latency" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 92, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 4, - "mode": "low_latency" - } - }, - { - "type": "allreduce", - "args": { - "num_elements": 21726494720, - "dtype": "bf16", - "world_size": 8 - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 3031040, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 8, - "mode": "normal" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 3031040, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 8, - "mode": "normal" - } - }, - { - "type": "allreduce", - "args": { - "num_elements": 2652160, - "dtype": "bf16", - "world_size": 8 - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 370, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 8, - "mode": "low_latency" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 370, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 8, - "mode": "low_latency" - } - }, - { - "type": "allgather", - "args": { - "num_elements_per_rank": 23576182784, - "dtype": "bf16", - "world_size": 4 - } - }, - { - "type": "reduce_scatter", - "args": { - "num_elements": 94304731136, - "dtype": "bf16", - "world_size": 4 - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 3289088, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 4, - "mode": "normal" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 3289088, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 4, - "mode": "normal" - } - }, - { - "type": "allgather", - "args": { - "num_elements_per_rank": 2874368, - "dtype": "bf16", - "world_size": 4 - } - }, - { - "type": "reduce_scatter", - "args": { - "num_elements": 11497472, - "dtype": "bf16", - "world_size": 4 - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 401, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 4, - "mode": "low_latency" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 401, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 4, - "mode": "low_latency" - } - }, - { - "type": "allgather", - "args": { - "num_elements_per_rank": 11788091392, - "dtype": "bf16", - "world_size": 8 - } - }, - { - "type": "reduce_scatter", - "args": { - "num_elements": 94304731136, - "dtype": "bf16", - "world_size": 8 - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 1644544, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 8, - "mode": "normal" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 1644544, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 8, - "mode": "normal" - } - }, - { - "type": "allgather", - "args": { - "num_elements_per_rank": 1433600, - "dtype": "bf16", - "world_size": 8 - } - }, - { - "type": "reduce_scatter", - "args": { - "num_elements": 11468800, - "dtype": "bf16", - "world_size": 8 - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 200, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 8, - "mode": "low_latency" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 200, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 8, - "mode": "low_latency" - } - }, - { - "type": "allgather", - "args": { - "num_elements_per_rank": 12287213568, - "dtype": "bf16", - "world_size": 4 - } - }, - { - "type": "reduce_scatter", - "args": { - "num_elements": 49148854272, - "dtype": "bf16", - "world_size": 4 - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 1714176, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 4, - "mode": "normal" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 1714176, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 4, - "mode": "normal" - } - }, - { - "type": "allgather", - "args": { - "num_elements_per_rank": 1498112, - "dtype": "bf16", - "world_size": 4 - } - }, - { - "type": "reduce_scatter", - "args": { - "num_elements": 5992448, - "dtype": "bf16", - "world_size": 4 - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 209, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 4, - "mode": "low_latency" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 209, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 4, - "mode": "low_latency" - } - }, - { - "type": "allgather", - "args": { - "num_elements_per_rank": 6143606784, - "dtype": "bf16", - "world_size": 8 - } - }, - { - "type": "reduce_scatter", - "args": { - "num_elements": 49148854272, - "dtype": "bf16", - "world_size": 8 - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 857088, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 8, - "mode": "normal" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 857088, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 8, - "mode": "normal" - } - }, - { - "type": "allgather", - "args": { - "num_elements_per_rank": 745472, - "dtype": "bf16", - "world_size": 8 - } - }, - { - "type": "reduce_scatter", - "args": { - "num_elements": 5963776, - "dtype": "bf16", - "world_size": 8 - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 104, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 8, - "mode": "low_latency" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 104, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 8, - "mode": "low_latency" - } - }, - { - "type": "allgather", - "args": { - "num_elements_per_rank": 32619102208, - "dtype": "bf16", - "world_size": 4 - } - }, - { - "type": "reduce_scatter", - "args": { - "num_elements": 130476408832, - "dtype": "bf16", - "world_size": 4 - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 4550656, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 4, - "mode": "normal" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 4550656, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 4, - "mode": "normal" - } - }, - { - "type": "allgather", - "args": { - "num_elements_per_rank": 3978240, - "dtype": "bf16", - "world_size": 4 - } - }, - { - "type": "reduce_scatter", - "args": { - "num_elements": 15912960, - "dtype": "bf16", - "world_size": 4 - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 555, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 4, - "mode": "low_latency" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 555, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 4, - "mode": "low_latency" - } - }, - { - "type": "allgather", - "args": { - "num_elements_per_rank": 16309551104, - "dtype": "bf16", - "world_size": 8 - } - }, - { - "type": "reduce_scatter", - "args": { - "num_elements": 130476408832, - "dtype": "bf16", - "world_size": 8 - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 2275328, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 8, - "mode": "normal" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 2275328, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 8, - "mode": "normal" - } - }, - { - "type": "allgather", - "args": { - "num_elements_per_rank": 1985536, - "dtype": "bf16", - "world_size": 8 - } - }, - { - "type": "reduce_scatter", - "args": { - "num_elements": 15884288, - "dtype": "bf16", - "world_size": 8 - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 277, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 8, - "mode": "low_latency" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 277, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 8, - "mode": "low_latency" - } - }, - { - "type": "allgather", - "args": { - "num_elements_per_rank": 14680064, - "dtype": "bf16", - "world_size": 4 - } - }, - { - "type": "reduce_scatter", - "args": { - "num_elements": 58720256, - "dtype": "bf16", - "world_size": 4 - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 2048, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 4, - "mode": "normal" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 2048, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 4, - "mode": "normal" - } - }, - { - "type": "allgather", - "args": { - "num_elements_per_rank": 0, - "dtype": "bf16", - "world_size": 4 - } - }, - { - "type": "reduce_scatter", - "args": { - "num_elements": 0, - "dtype": "bf16", - "world_size": 4 - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 0, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 4, - "mode": "low_latency" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 0, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 4, - "mode": "low_latency" - } - }, - { - "type": "allreduce", - "args": { - "num_elements": 58720256, - "dtype": "bf16", - "world_size": 8 - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 8192, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 8, - "mode": "normal" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 8192, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 8, - "mode": "normal" - } - }, - { - "type": "allreduce", - "args": { - "num_elements": 7168, - "dtype": "bf16", - "world_size": 8 - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 1, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 8, - "mode": "low_latency" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 1, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 8, - "mode": "low_latency" - } - }, - { - "type": "allgather", - "args": { - "num_elements_per_rank": 29360128, - "dtype": "bf16", - "world_size": 4 - } - }, - { - "type": "reduce_scatter", - "args": { - "num_elements": 117440512, - "dtype": "bf16", - "world_size": 4 - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 4096, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 4, - "mode": "normal" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 4096, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 4, - "mode": "normal" - } - }, - { - "type": "allreduce", - "args": { - "num_elements": 117440512, - "dtype": "bf16", - "world_size": 8 - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 16384, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 8, - "mode": "normal" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 16384, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 8, - "mode": "normal" - } - }, - { - "type": "allreduce", - "args": { - "num_elements": 14336, - "dtype": "bf16", - "world_size": 8 - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 2, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 8, - "mode": "low_latency" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 2, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 8, - "mode": "low_latency" - } - }, - { - "type": "allgather", - "args": { - "num_elements_per_rank": 58720256, - "dtype": "bf16", - "world_size": 4 - } - }, - { - "type": "reduce_scatter", - "args": { - "num_elements": 234881024, - "dtype": "bf16", - "world_size": 4 - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 8192, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 4, - "mode": "normal" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 8192, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 4, - "mode": "normal" - } - }, - { - "type": "allreduce", - "args": { - "num_elements": 234881024, - "dtype": "bf16", - "world_size": 8 - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 32768, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 8, - "mode": "normal" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 32768, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 8, - "mode": "normal" - } - }, - { - "type": "allreduce", - "args": { - "num_elements": 28672, - "dtype": "bf16", - "world_size": 8 - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 4, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 8, - "mode": "low_latency" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 4, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 8, - "mode": "low_latency" - } - }, - { - "type": "allgather", - "args": { - "num_elements_per_rank": 117440512, - "dtype": "bf16", - "world_size": 4 - } - }, - { - "type": "reduce_scatter", - "args": { - "num_elements": 469762048, - "dtype": "bf16", - "world_size": 4 - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 16384, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 4, - "mode": "normal" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 16384, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 4, - "mode": "normal" - } - }, - { - "type": "allreduce", - "args": { - "num_elements": 469762048, - "dtype": "bf16", - "world_size": 8 - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 65536, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 8, - "mode": "normal" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 65536, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 8, - "mode": "normal" - } - }, - { - "type": "allreduce", - "args": { - "num_elements": 57344, - "dtype": "bf16", - "world_size": 8 - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 8, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 8, - "mode": "low_latency" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 8, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 8, - "mode": "low_latency" - } - }, - { - "type": "allgather", - "args": { - "num_elements_per_rank": 234881024, - "dtype": "bf16", - "world_size": 4 - } - }, - { - "type": "reduce_scatter", - "args": { - "num_elements": 939524096, - "dtype": "bf16", - "world_size": 4 - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 32768, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 4, - "mode": "normal" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 32768, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 4, - "mode": "normal" - } - }, - { - "type": "allgather", - "args": { - "num_elements_per_rank": 28672, - "dtype": "bf16", - "world_size": 4 - } - }, - { - "type": "reduce_scatter", - "args": { - "num_elements": 114688, - "dtype": "bf16", - "world_size": 4 - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 4, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 4, - "mode": "low_latency" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 4, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 4, - "mode": "low_latency" - } - }, - { - "type": "allreduce", - "args": { - "num_elements": 939524096, - "dtype": "bf16", - "world_size": 8 - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 131072, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 8, - "mode": "normal" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 131072, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 8, - "mode": "normal" - } - }, - { - "type": "allreduce", - "args": { - "num_elements": 114688, - "dtype": "bf16", - "world_size": 8 - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 16, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 8, - "mode": "low_latency" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 16, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 8, - "mode": "low_latency" - } - }, - { - "type": "allgather", - "args": { - "num_elements_per_rank": 469762048, - "dtype": "bf16", - "world_size": 4 - } - }, - { - "type": "reduce_scatter", - "args": { - "num_elements": 1879048192, - "dtype": "bf16", - "world_size": 4 - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 65536, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 4, - "mode": "normal" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 65536, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 4, - "mode": "normal" - } - }, - { - "type": "allgather", - "args": { - "num_elements_per_rank": 57344, - "dtype": "bf16", - "world_size": 4 - } - }, - { - "type": "reduce_scatter", - "args": { - "num_elements": 229376, - "dtype": "bf16", - "world_size": 4 - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 8, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 4, - "mode": "low_latency" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 8, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 4, - "mode": "low_latency" - } - }, - { - "type": "allreduce", - "args": { - "num_elements": 1879048192, - "dtype": "bf16", - "world_size": 8 - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 262144, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 8, - "mode": "normal" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 262144, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 8, - "mode": "normal" - } - }, - { - "type": "allreduce", - "args": { - "num_elements": 229376, - "dtype": "bf16", - "world_size": 8 - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 32, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 8, - "mode": "low_latency" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 32, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 8, - "mode": "low_latency" - } - }, - { - "type": "allgather", - "args": { - "num_elements_per_rank": 7340032, - "dtype": "bf16", - "world_size": 8 - } - }, - { - "type": "reduce_scatter", - "args": { - "num_elements": 58720256, - "dtype": "bf16", - "world_size": 8 - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 1024, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 8, - "mode": "normal" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 1024, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 8, - "mode": "normal" - } - }, - { - "type": "allgather", - "args": { - "num_elements_per_rank": 14336, - "dtype": "bf16", - "world_size": 8 - } - }, - { - "type": "reduce_scatter", - "args": { - "num_elements": 114688, - "dtype": "bf16", - "world_size": 8 - } - }, - { - "type": "allgather", - "args": { - "num_elements_per_rank": 29360128, - "dtype": "bf16", - "world_size": 8 - } - }, - { - "type": "reduce_scatter", - "args": { - "num_elements": 234881024, - "dtype": "bf16", - "world_size": 8 - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 4096, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 8, - "mode": "normal" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 4096, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 8, - "mode": "normal" - } - }, - { - "type": "allgather", - "args": { - "num_elements_per_rank": 57344, - "dtype": "bf16", - "world_size": 8 - } - }, - { - "type": "reduce_scatter", - "args": { - "num_elements": 458752, - "dtype": "bf16", - "world_size": 8 - } - }, - { - "type": "allreduce", - "args": { - "num_elements": 458752, - "dtype": "bf16", - "world_size": 8 - } - }, - { - "type": "allgather", - "args": { - "num_elements_per_rank": 58720256, - "dtype": "bf16", - "world_size": 8 - } - }, - { - "type": "reduce_scatter", - "args": { - "num_elements": 469762048, - "dtype": "bf16", - "world_size": 8 - } - }, - { - "type": "allgather", - "args": { - "num_elements_per_rank": 114688, - "dtype": "bf16", - "world_size": 8 - } - }, - { - "type": "reduce_scatter", - "args": { - "num_elements": 917504, - "dtype": "bf16", - "world_size": 8 - } - }, - { - "type": "allreduce", - "args": { - "num_elements": 917504, - "dtype": "bf16", - "world_size": 8 - } - }, - { - "type": "allgather", - "args": { - "num_elements_per_rank": 234881024, - "dtype": "bf16", - "world_size": 8 - } - }, - { - "type": "reduce_scatter", - "args": { - "num_elements": 1879048192, - "dtype": "bf16", - "world_size": 8 - } - }, - { - "type": "allgather", - "args": { - "num_elements_per_rank": 458752, - "dtype": "bf16", - "world_size": 8 - } - }, - { - "type": "reduce_scatter", - "args": { - "num_elements": 3670016, - "dtype": "bf16", - "world_size": 8 - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 64, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 8, - "mode": "low_latency" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 64, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 8, - "mode": "low_latency" - } - }, - { - "type": "allreduce", - "args": { - "num_elements": 3670016, - "dtype": "bf16", - "world_size": 8 - } - }, - { - "type": "allgather", - "args": { - "num_elements_per_rank": 822083584, - "dtype": "bf16", - "world_size": 8 - } - }, - { - "type": "reduce_scatter", - "args": { - "num_elements": 6576668672, - "dtype": "bf16", - "world_size": 8 - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 114688, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 8, - "mode": "normal" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 114688, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 8, - "mode": "normal" - } - }, - { - "type": "allgather", - "args": { - "num_elements_per_rank": 1605632, - "dtype": "bf16", - "world_size": 8 - } - }, - { - "type": "reduce_scatter", - "args": { - "num_elements": 12845056, - "dtype": "bf16", - "world_size": 8 - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 224, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 8, - "mode": "low_latency" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 224, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 8, - "mode": "low_latency" - } - }, - { - "type": "allgather", - "args": { - "num_elements_per_rank": 939524096, - "dtype": "bf16", - "world_size": 8 - } - }, - { - "type": "reduce_scatter", - "args": { - "num_elements": 7516192768, - "dtype": "bf16", - "world_size": 8 - } - }, - { - "type": "allgather", - "args": { - "num_elements_per_rank": 1835008, - "dtype": "bf16", - "world_size": 8 - } - }, - { - "type": "reduce_scatter", - "args": { - "num_elements": 14680064, - "dtype": "bf16", - "world_size": 8 - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 256, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 8, - "mode": "low_latency" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 256, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 8, - "mode": "low_latency" - } - }, - { - "type": "allgather", - "args": { - "num_elements_per_rank": 1086324736, - "dtype": "bf16", - "world_size": 8 - } - }, - { - "type": "reduce_scatter", - "args": { - "num_elements": 8690597888, - "dtype": "bf16", - "world_size": 8 - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 151552, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 8, - "mode": "normal" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 151552, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 8, - "mode": "normal" - } - }, - { - "type": "allgather", - "args": { - "num_elements_per_rank": 2121728, - "dtype": "bf16", - "world_size": 8 - } - }, - { - "type": "reduce_scatter", - "args": { - "num_elements": 16973824, - "dtype": "bf16", - "world_size": 8 - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 296, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 8, - "mode": "low_latency" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 296, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 8, - "mode": "low_latency" - } - }, - { - "type": "allgather", - "args": { - "num_elements_per_rank": 1468006400, - "dtype": "bf16", - "world_size": 8 - } - }, - { - "type": "reduce_scatter", - "args": { - "num_elements": 11744051200, - "dtype": "bf16", - "world_size": 8 - } - }, - { - "type": "allgather", - "args": { - "num_elements_per_rank": 2867200, - "dtype": "bf16", - "world_size": 8 - } - }, - { - "type": "reduce_scatter", - "args": { - "num_elements": 22937600, - "dtype": "bf16", - "world_size": 8 - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 400, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 8, - "mode": "low_latency" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 400, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 8, - "mode": "low_latency" - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 512, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 8, - "mode": "normal" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 512, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 8, - "mode": "normal" - } - }, - { - "type": "allgather", - "args": { - "num_elements_per_rank": 0, - "dtype": "bf16", - "world_size": 8 - } - }, - { - "type": "reduce_scatter", - "args": { - "num_elements": 0, - "dtype": "bf16", - "world_size": 8 - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 0, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 8, - "mode": "low_latency" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 0, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 8, - "mode": "low_latency" - } - }, - { - "type": "allreduce", - "args": { - "num_elements": 29360128, - "dtype": "bf16", - "world_size": 8 - } - }, - { - "type": "allgather", - "args": { - "num_elements_per_rank": 28672, - "dtype": "bf16", - "world_size": 8 - } - }, - { - "type": "reduce_scatter", - "args": { - "num_elements": 229376, - "dtype": "bf16", - "world_size": 8 - } - }, - { - "type": "allgather", - "args": { - "num_elements_per_rank": 117440512, - "dtype": "bf16", - "world_size": 8 - } - }, - { - "type": "reduce_scatter", - "args": { - "num_elements": 939524096, - "dtype": "bf16", - "world_size": 8 - } - }, - { - "type": "allgather", - "args": { - "num_elements_per_rank": 1761607680, - "dtype": "bf16", - "world_size": 8 - } - }, - { - "type": "reduce_scatter", - "args": { - "num_elements": 14092861440, - "dtype": "bf16", - "world_size": 8 - } - }, - { - "type": "allgather", - "args": { - "num_elements_per_rank": 1720320, - "dtype": "bf16", - "world_size": 8 - } - }, - { - "type": "reduce_scatter", - "args": { - "num_elements": 13762560, - "dtype": "bf16", - "world_size": 8 - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 240, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 8, - "mode": "low_latency" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 240, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 8, - "mode": "low_latency" - } - }, - { - "type": "allgather", - "args": { - "num_elements_per_rank": 4726980608, - "dtype": "bf16", - "world_size": 8 - } - }, - { - "type": "reduce_scatter", - "args": { - "num_elements": 37815844864, - "dtype": "bf16", - "world_size": 8 - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 659456, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 8, - "mode": "normal" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 659456, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 8, - "mode": "normal" - } - }, - { - "type": "allgather", - "args": { - "num_elements_per_rank": 4616192, - "dtype": "bf16", - "world_size": 8 - } - }, - { - "type": "reduce_scatter", - "args": { - "num_elements": 36929536, - "dtype": "bf16", - "world_size": 8 - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 644, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 8, - "mode": "low_latency" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 644, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 8, - "mode": "low_latency" - } - }, - { - "type": "allgather", - "args": { - "num_elements_per_rank": 352321536, - "dtype": "bf16", - "world_size": 8 - } - }, - { - "type": "reduce_scatter", - "args": { - "num_elements": 2818572288, - "dtype": "bf16", - "world_size": 8 - } - }, - { - "type": "allgather", - "args": { - "num_elements_per_rank": 86016, - "dtype": "bf16", - "world_size": 8 - } - }, - { - "type": "reduce_scatter", - "args": { - "num_elements": 688128, - "dtype": "bf16", - "world_size": 8 - } - }, - { - "type": "allreduce", - "args": { - "num_elements": 2818572288, - "dtype": "bf16", - "world_size": 8 - } - }, - { - "type": "allreduce", - "args": { - "num_elements": 688128, - "dtype": "bf16", - "world_size": 8 - } - }, - { - "type": "allgather", - "args": { - "num_elements_per_rank": 469762048, - "dtype": "bf16", - "world_size": 8 - } - }, - { - "type": "reduce_scatter", - "args": { - "num_elements": 3758096384, - "dtype": "bf16", - "world_size": 8 - } - }, - { - "type": "allreduce", - "args": { - "num_elements": 3758096384, - "dtype": "bf16", - "world_size": 8 - } - }, - { - "type": "allgather", - "args": { - "num_elements_per_rank": 1644167168, - "dtype": "bf16", - "world_size": 8 - } - }, - { - "type": "reduce_scatter", - "args": { - "num_elements": 13153337344, - "dtype": "bf16", - "world_size": 8 - } - }, - { - "type": "allgather", - "args": { - "num_elements_per_rank": 401408, - "dtype": "bf16", - "world_size": 8 - } - }, - { - "type": "reduce_scatter", - "args": { - "num_elements": 3211264, - "dtype": "bf16", - "world_size": 8 - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 229376, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 8, - "mode": "normal" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 229376, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 8, - "mode": "normal" - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 56, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 8, - "mode": "low_latency" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 56, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 8, - "mode": "low_latency" - } - }, - { - "type": "allgather", - "args": { - "num_elements_per_rank": 2113929216, - "dtype": "bf16", - "world_size": 8 - } - }, - { - "type": "reduce_scatter", - "args": { - "num_elements": 16911433728, - "dtype": "bf16", - "world_size": 8 - } - }, - { - "type": "allgather", - "args": { - "num_elements_per_rank": 516096, - "dtype": "bf16", - "world_size": 8 - } - }, - { - "type": "reduce_scatter", - "args": { - "num_elements": 4128768, - "dtype": "bf16", - "world_size": 8 - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 294912, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 8, - "mode": "normal" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 294912, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 8, - "mode": "normal" - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 72, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 8, - "mode": "low_latency" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 72, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 8, - "mode": "low_latency" - } - }, - { - "type": "allgather", - "args": { - "num_elements_per_rank": 7985954816, - "dtype": "bf16", - "world_size": 8 - } - }, - { - "type": "reduce_scatter", - "args": { - "num_elements": 63887638528, - "dtype": "bf16", - "world_size": 8 - } - }, - { - "type": "allgather", - "args": { - "num_elements_per_rank": 1949696, - "dtype": "bf16", - "world_size": 8 - } - }, - { - "type": "reduce_scatter", - "args": { - "num_elements": 15597568, - "dtype": "bf16", - "world_size": 8 - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 1114112, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 8, - "mode": "normal" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 1114112, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 8, - "mode": "normal" - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 272, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 8, - "mode": "low_latency" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 272, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 8, - "mode": "low_latency" - } - }, - { - "type": "allreduce", - "args": { - "num_elements": 7516192768, - "dtype": "bf16", - "world_size": 8 - } - }, - { - "type": "allgather", - "args": { - "num_elements_per_rank": 704643072, - "dtype": "bf16", - "world_size": 8 - } - }, - { - "type": "reduce_scatter", - "args": { - "num_elements": 5637144576, - "dtype": "bf16", - "world_size": 8 - } - }, - { - "type": "allreduce", - "args": { - "num_elements": 5637144576, - "dtype": "bf16", - "world_size": 8 - } - }, - { - "type": "allgather", - "args": { - "num_elements_per_rank": 1879048192, - "dtype": "bf16", - "world_size": 8 - } - }, - { - "type": "reduce_scatter", - "args": { - "num_elements": 15032385536, - "dtype": "bf16", - "world_size": 8 - } - }, - { - "type": "allgather", - "args": { - "num_elements_per_rank": 229376, - "dtype": "bf16", - "world_size": 8 - } - }, - { - "type": "reduce_scatter", - "args": { - "num_elements": 1835008, - "dtype": "bf16", - "world_size": 8 - } - }, - { - "type": "allgather", - "args": { - "num_elements_per_rank": 4697620480, - "dtype": "bf16", - "world_size": 8 - } - }, - { - "type": "reduce_scatter", - "args": { - "num_elements": 37580963840, - "dtype": "bf16", - "world_size": 8 - } - }, - { - "type": "allgather", - "args": { - "num_elements_per_rank": 573440, - "dtype": "bf16", - "world_size": 8 - } - }, - { - "type": "reduce_scatter", - "args": { - "num_elements": 4587520, - "dtype": "bf16", - "world_size": 8 - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 655360, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 8, - "mode": "normal" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 655360, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 8, - "mode": "normal" - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 80, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 8, - "mode": "low_latency" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 80, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 8, - "mode": "low_latency" - } - }, - { - "type": "allgather", - "args": { - "num_elements_per_rank": 2400190464, - "dtype": "bf16", - "world_size": 2 - } - }, - { - "type": "reduce_scatter", - "args": { - "num_elements": 4800380928, - "dtype": "bf16", - "world_size": 2 - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 334848, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 2, - "mode": "normal" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 334848, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 2, - "mode": "normal" - } - }, - { - "type": "allgather", - "args": { - "num_elements_per_rank": 4687872, - "dtype": "bf16", - "world_size": 2 - } - }, - { - "type": "reduce_scatter", - "args": { - "num_elements": 9375744, - "dtype": "bf16", - "world_size": 2 - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 654, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 2, - "mode": "low_latency" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 654, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 2, - "mode": "low_latency" - } - }, - { - "type": "allgather", - "args": { - "num_elements_per_rank": 600047616, - "dtype": "bf16", - "world_size": 8 - } - }, - { - "type": "reduce_scatter", - "args": { - "num_elements": 4800380928, - "dtype": "bf16", - "world_size": 8 - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 83712, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 8, - "mode": "normal" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 83712, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 8, - "mode": "normal" - } - }, - { - "type": "allgather", - "args": { - "num_elements_per_rank": 1168384, - "dtype": "bf16", - "world_size": 8 - } - }, - { - "type": "reduce_scatter", - "args": { - "num_elements": 9347072, - "dtype": "bf16", - "world_size": 8 - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 163, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 8, - "mode": "low_latency" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 163, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 8, - "mode": "low_latency" - } - }, - { - "type": "allgather", - "args": { - "num_elements_per_rank": 994574336, - "dtype": "bf16", - "world_size": 2 - } - }, - { - "type": "reduce_scatter", - "args": { - "num_elements": 1989148672, - "dtype": "bf16", - "world_size": 2 - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 138752, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 2, - "mode": "normal" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 138752, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 2, - "mode": "normal" - } - }, - { - "type": "allgather", - "args": { - "num_elements_per_rank": 1942528, - "dtype": "bf16", - "world_size": 2 - } - }, - { - "type": "reduce_scatter", - "args": { - "num_elements": 3885056, - "dtype": "bf16", - "world_size": 2 - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 271, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 2, - "mode": "low_latency" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 271, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 2, - "mode": "low_latency" - } - }, - { - "type": "allgather", - "args": { - "num_elements_per_rank": 248643584, - "dtype": "bf16", - "world_size": 8 - } - }, - { - "type": "reduce_scatter", - "args": { - "num_elements": 1989148672, - "dtype": "bf16", - "world_size": 8 - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 34688, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 8, - "mode": "normal" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 34688, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 8, - "mode": "normal" - } - }, - { - "type": "allgather", - "args": { - "num_elements_per_rank": 480256, - "dtype": "bf16", - "world_size": 8 - } - }, - { - "type": "reduce_scatter", - "args": { - "num_elements": 3842048, - "dtype": "bf16", - "world_size": 8 - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 67, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 8, - "mode": "low_latency" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 67, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 8, - "mode": "low_latency" - } - }, - { - "type": "allgather", - "args": { - "num_elements_per_rank": 40370176, - "dtype": "bf16", - "world_size": 2 - } - }, - { - "type": "reduce_scatter", - "args": { - "num_elements": 80740352, - "dtype": "bf16", - "world_size": 2 - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 5632, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 2, - "mode": "normal" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 5632, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 2, - "mode": "normal" - } - }, - { - "type": "allgather", - "args": { - "num_elements_per_rank": 78848, - "dtype": "bf16", - "world_size": 2 - } - }, - { - "type": "reduce_scatter", - "args": { - "num_elements": 157696, - "dtype": "bf16", - "world_size": 2 - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 11, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 2, - "mode": "low_latency" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 11, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 2, - "mode": "low_latency" - } - }, - { - "type": "allreduce", - "args": { - "num_elements": 80740352, - "dtype": "bf16", - "world_size": 8 - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 11264, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 8, - "mode": "normal" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 11264, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 8, - "mode": "normal" - } - }, - { - "type": "allreduce", - "args": { - "num_elements": 157696, - "dtype": "bf16", - "world_size": 8 - } - }, - { - "type": "allgather", - "args": { - "num_elements_per_rank": 36700160, - "dtype": "bf16", - "world_size": 2 - } - }, - { - "type": "reduce_scatter", - "args": { - "num_elements": 73400320, - "dtype": "bf16", - "world_size": 2 - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 5120, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 2, - "mode": "normal" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 5120, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 2, - "mode": "normal" - } - }, - { - "type": "allgather", - "args": { - "num_elements_per_rank": 71680, - "dtype": "bf16", - "world_size": 2 - } - }, - { - "type": "reduce_scatter", - "args": { - "num_elements": 143360, - "dtype": "bf16", - "world_size": 2 - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 10, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 2, - "mode": "low_latency" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 10, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 2, - "mode": "low_latency" - } - }, - { - "type": "allgather", - "args": { - "num_elements_per_rank": 73400320, - "dtype": "bf16", - "world_size": 2 - } - }, - { - "type": "reduce_scatter", - "args": { - "num_elements": 146800640, - "dtype": "bf16", - "world_size": 2 - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 10240, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 2, - "mode": "normal" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 10240, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 2, - "mode": "normal" - } - }, - { - "type": "allgather", - "args": { - "num_elements_per_rank": 143360, - "dtype": "bf16", - "world_size": 2 - } - }, - { - "type": "reduce_scatter", - "args": { - "num_elements": 286720, - "dtype": "bf16", - "world_size": 2 - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 20, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 2, - "mode": "low_latency" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 20, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 2, - "mode": "low_latency" - } - }, - { - "type": "allgather", - "args": { - "num_elements_per_rank": 91750400, - "dtype": "bf16", - "world_size": 2 - } - }, - { - "type": "reduce_scatter", - "args": { - "num_elements": 183500800, - "dtype": "bf16", - "world_size": 2 - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 12800, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 2, - "mode": "normal" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 12800, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 2, - "mode": "normal" - } - }, - { - "type": "allgather", - "args": { - "num_elements_per_rank": 179200, - "dtype": "bf16", - "world_size": 2 - } - }, - { - "type": "reduce_scatter", - "args": { - "num_elements": 358400, - "dtype": "bf16", - "world_size": 2 - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 25, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 2, - "mode": "low_latency" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 25, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 2, - "mode": "low_latency" - } - }, - { - "type": "allgather", - "args": { - "num_elements_per_rank": 220200960, - "dtype": "bf16", - "world_size": 2 - } - }, - { - "type": "reduce_scatter", - "args": { - "num_elements": 440401920, - "dtype": "bf16", - "world_size": 2 - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 30720, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 2, - "mode": "normal" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 30720, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 2, - "mode": "normal" - } - }, - { - "type": "allgather", - "args": { - "num_elements_per_rank": 430080, - "dtype": "bf16", - "world_size": 2 - } - }, - { - "type": "reduce_scatter", - "args": { - "num_elements": 860160, - "dtype": "bf16", - "world_size": 2 - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 60, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 2, - "mode": "low_latency" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 60, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 2, - "mode": "low_latency" - } - }, - { - "type": "allgather", - "args": { - "num_elements_per_rank": 440401920, - "dtype": "bf16", - "world_size": 2 - } - }, - { - "type": "reduce_scatter", - "args": { - "num_elements": 880803840, - "dtype": "bf16", - "world_size": 2 - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 61440, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 2, - "mode": "normal" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 61440, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 2, - "mode": "normal" - } - }, - { - "type": "allgather", - "args": { - "num_elements_per_rank": 860160, - "dtype": "bf16", - "world_size": 2 - } - }, - { - "type": "reduce_scatter", - "args": { - "num_elements": 1720320, - "dtype": "bf16", - "world_size": 2 - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 120, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 2, - "mode": "low_latency" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 120, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 2, - "mode": "low_latency" - } - }, - { - "type": "allgather", - "args": { - "num_elements_per_rank": 734003200, - "dtype": "bf16", - "world_size": 2 - } - }, - { - "type": "reduce_scatter", - "args": { - "num_elements": 1468006400, - "dtype": "bf16", - "world_size": 2 - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 102400, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 2, - "mode": "normal" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 102400, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 2, - "mode": "normal" - } - }, - { - "type": "allgather", - "args": { - "num_elements_per_rank": 1433600, - "dtype": "bf16", - "world_size": 2 - } - }, - { - "type": "reduce_scatter", - "args": { - "num_elements": 2867200, - "dtype": "bf16", - "world_size": 2 - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 200, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 2, - "mode": "low_latency" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 200, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 2, - "mode": "low_latency" - } - }, - { - "type": "allreduce", - "args": { - "num_elements": 146800640, - "dtype": "bf16", - "world_size": 8 - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 20480, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 8, - "mode": "normal" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 20480, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 8, - "mode": "normal" - } - }, - { - "type": "allreduce", - "args": { - "num_elements": 286720, - "dtype": "bf16", - "world_size": 8 - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 40, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 8, - "mode": "low_latency" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 40, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 8, - "mode": "low_latency" - } - }, - { - "type": "allreduce", - "args": { - "num_elements": 440401920, - "dtype": "bf16", - "world_size": 8 - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 61440, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 8, - "mode": "normal" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 61440, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 8, - "mode": "normal" - } - }, - { - "type": "allreduce", - "args": { - "num_elements": 1720320, - "dtype": "bf16", - "world_size": 8 - } - }, - { - "type": "allreduce", - "args": { - "num_elements": 2867200, - "dtype": "bf16", - "world_size": 8 - } - }, - { - "type": "allgather", - "args": { - "num_elements_per_rank": 8595177472, - "dtype": "bf16", - "world_size": 2 - } - }, - { - "type": "reduce_scatter", - "args": { - "num_elements": 17190354944, - "dtype": "bf16", - "world_size": 2 - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 1199104, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 2, - "mode": "normal" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 1199104, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 2, - "mode": "normal" - } - }, - { - "type": "allgather", - "args": { - "num_elements_per_rank": 16787456, - "dtype": "bf16", - "world_size": 2 - } - }, - { - "type": "reduce_scatter", - "args": { - "num_elements": 33574912, - "dtype": "bf16", - "world_size": 2 - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 2342, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 2, - "mode": "low_latency" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 2342, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 2, - "mode": "low_latency" - } - }, - { - "type": "allgather", - "args": { - "num_elements_per_rank": 2148794368, - "dtype": "bf16", - "world_size": 8 - } - }, - { - "type": "reduce_scatter", - "args": { - "num_elements": 17190354944, - "dtype": "bf16", - "world_size": 8 - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 299776, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 8, - "mode": "normal" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 299776, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 8, - "mode": "normal" - } - }, - { - "type": "allgather", - "args": { - "num_elements_per_rank": 4193280, - "dtype": "bf16", - "world_size": 8 - } - }, - { - "type": "reduce_scatter", - "args": { - "num_elements": 33546240, - "dtype": "bf16", - "world_size": 8 - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 585, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 8, - "mode": "low_latency" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 585, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 8, - "mode": "low_latency" - } - }, - { - "type": "allgather", - "args": { - "num_elements_per_rank": 31595167744, - "dtype": "bf16", - "world_size": 2 - } - }, - { - "type": "reduce_scatter", - "args": { - "num_elements": 63190335488, - "dtype": "bf16", - "world_size": 2 - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 4407808, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 2, - "mode": "normal" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 4407808, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 2, - "mode": "normal" - } - }, - { - "type": "allgather", - "args": { - "num_elements_per_rank": 61709312, - "dtype": "bf16", - "world_size": 2 - } - }, - { - "type": "reduce_scatter", - "args": { - "num_elements": 123418624, - "dtype": "bf16", - "world_size": 2 - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 8609, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 2, - "mode": "low_latency" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 8609, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 2, - "mode": "low_latency" - } - }, - { - "type": "allgather", - "args": { - "num_elements_per_rank": 7898791936, - "dtype": "bf16", - "world_size": 8 - } - }, - { - "type": "reduce_scatter", - "args": { - "num_elements": 63190335488, - "dtype": "bf16", - "world_size": 8 - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 1101952, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 8, - "mode": "normal" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 1101952, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 8, - "mode": "normal" - } - }, - { - "type": "allgather", - "args": { - "num_elements_per_rank": 15425536, - "dtype": "bf16", - "world_size": 8 - } - }, - { - "type": "reduce_scatter", - "args": { - "num_elements": 123404288, - "dtype": "bf16", - "world_size": 8 - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 2152, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 8, - "mode": "low_latency" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 2152, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 8, - "mode": "low_latency" - } - }, - { - "type": "allgather", - "args": { - "num_elements_per_rank": 47438626816, - "dtype": "bf16", - "world_size": 2 - } - }, - { - "type": "reduce_scatter", - "args": { - "num_elements": 94877253632, - "dtype": "bf16", - "world_size": 2 - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 6618112, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 2, - "mode": "normal" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 6618112, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 2, - "mode": "normal" - } - }, - { - "type": "allgather", - "args": { - "num_elements_per_rank": 92653568, - "dtype": "bf16", - "world_size": 2 - } - }, - { - "type": "reduce_scatter", - "args": { - "num_elements": 185307136, - "dtype": "bf16", - "world_size": 2 - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 12926, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 2, - "mode": "low_latency" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 12926, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 2, - "mode": "low_latency" - } - }, - { - "type": "allgather", - "args": { - "num_elements_per_rank": 11859656704, - "dtype": "bf16", - "world_size": 8 - } - }, - { - "type": "reduce_scatter", - "args": { - "num_elements": 94877253632, - "dtype": "bf16", - "world_size": 8 - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 1654528, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 8, - "mode": "normal" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 1654528, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 8, - "mode": "normal" - } - }, - { - "type": "allgather", - "args": { - "num_elements_per_rank": 23159808, - "dtype": "bf16", - "world_size": 8 - } - }, - { - "type": "reduce_scatter", - "args": { - "num_elements": 185278464, - "dtype": "bf16", - "world_size": 8 - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 3231, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 8, - "mode": "low_latency" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 3231, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 8, - "mode": "low_latency" - } - }, - { - "type": "allgather", - "args": { - "num_elements_per_rank": 4315938816, - "dtype": "bf16", - "world_size": 2 - } - }, - { - "type": "reduce_scatter", - "args": { - "num_elements": 8631877632, - "dtype": "bf16", - "world_size": 2 - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 602112, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 2, - "mode": "normal" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 602112, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 2, - "mode": "normal" - } - }, - { - "type": "allgather", - "args": { - "num_elements_per_rank": 4214784, - "dtype": "bf16", - "world_size": 2 - } - }, - { - "type": "reduce_scatter", - "args": { - "num_elements": 8429568, - "dtype": "bf16", - "world_size": 2 - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 588, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 2, - "mode": "low_latency" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 588, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 2, - "mode": "low_latency" - } - }, - { - "type": "allgather", - "args": { - "num_elements_per_rank": 1078984704, - "dtype": "bf16", - "world_size": 8 - } - }, - { - "type": "reduce_scatter", - "args": { - "num_elements": 8631877632, - "dtype": "bf16", - "world_size": 8 - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 150528, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 8, - "mode": "normal" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 150528, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 8, - "mode": "normal" - } - }, - { - "type": "allgather", - "args": { - "num_elements_per_rank": 1053696, - "dtype": "bf16", - "world_size": 8 - } - }, - { - "type": "reduce_scatter", - "args": { - "num_elements": 8429568, - "dtype": "bf16", - "world_size": 8 - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 147, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 8, - "mode": "low_latency" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 147, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 8, - "mode": "low_latency" - } - }, - { - "type": "allgather", - "args": { - "num_elements_per_rank": 22020096, - "dtype": "bf16", - "world_size": 2 - } - }, - { - "type": "reduce_scatter", - "args": { - "num_elements": 44040192, - "dtype": "bf16", - "world_size": 2 - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 3072, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 2, - "mode": "normal" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 3072, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 2, - "mode": "normal" - } - }, - { - "type": "allgather", - "args": { - "num_elements_per_rank": 21504, - "dtype": "bf16", - "world_size": 2 - } - }, - { - "type": "reduce_scatter", - "args": { - "num_elements": 43008, - "dtype": "bf16", - "world_size": 2 - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 3, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 2, - "mode": "low_latency" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 3, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 2, - "mode": "low_latency" - } - }, - { - "type": "allgather", - "args": { - "num_elements_per_rank": 18350080, - "dtype": "bf16", - "world_size": 2 - } - }, - { - "type": "reduce_scatter", - "args": { - "num_elements": 36700160, - "dtype": "bf16", - "world_size": 2 - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 2560, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 2, - "mode": "normal" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 2560, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 2, - "mode": "normal" - } - }, - { - "type": "allgather", - "args": { - "num_elements_per_rank": 14336, - "dtype": "bf16", - "world_size": 2 - } - }, - { - "type": "reduce_scatter", - "args": { - "num_elements": 28672, - "dtype": "bf16", - "world_size": 2 - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 2, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 2, - "mode": "low_latency" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 2, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 2, - "mode": "low_latency" - } - }, - { - "type": "allgather", - "args": { - "num_elements_per_rank": 35840, - "dtype": "bf16", - "world_size": 2 - } - }, - { - "type": "reduce_scatter", - "args": { - "num_elements": 71680, - "dtype": "bf16", - "world_size": 2 - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 5, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 2, - "mode": "low_latency" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 5, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 2, - "mode": "low_latency" - } - }, - { - "type": "allgather", - "args": { - "num_elements_per_rank": 55050240, - "dtype": "bf16", - "world_size": 2 - } - }, - { - "type": "reduce_scatter", - "args": { - "num_elements": 110100480, - "dtype": "bf16", - "world_size": 2 - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 7680, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 2, - "mode": "normal" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 7680, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 2, - "mode": "normal" - } - }, - { - "type": "allgather", - "args": { - "num_elements_per_rank": 50176, - "dtype": "bf16", - "world_size": 2 - } - }, - { - "type": "reduce_scatter", - "args": { - "num_elements": 100352, - "dtype": "bf16", - "world_size": 2 - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 7, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 2, - "mode": "low_latency" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 7, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 2, - "mode": "low_latency" - } - }, - { - "type": "allgather", - "args": { - "num_elements_per_rank": 86016, - "dtype": "bf16", - "world_size": 2 - } - }, - { - "type": "reduce_scatter", - "args": { - "num_elements": 172032, - "dtype": "bf16", - "world_size": 2 - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 12, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 2, - "mode": "low_latency" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 12, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 2, - "mode": "low_latency" - } - }, - { - "type": "allreduce", - "args": { - "num_elements": 36700160, - "dtype": "bf16", - "world_size": 4 - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 5120, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 4, - "mode": "normal" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 5120, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 4, - "mode": "normal" - } - }, - { - "type": "allreduce", - "args": { - "num_elements": 35840, - "dtype": "bf16", - "world_size": 4 - } - }, - { - "type": "allreduce", - "args": { - "num_elements": 73400320, - "dtype": "bf16", - "world_size": 4 - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 10240, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 4, - "mode": "normal" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 10240, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 4, - "mode": "normal" - } - }, - { - "type": "allreduce", - "args": { - "num_elements": 71680, - "dtype": "bf16", - "world_size": 4 - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 10, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 4, - "mode": "low_latency" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 10, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 4, - "mode": "low_latency" - } - }, - { - "type": "allreduce", - "args": { - "num_elements": 110100480, - "dtype": "bf16", - "world_size": 4 - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 15360, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 4, - "mode": "normal" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 15360, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 4, - "mode": "normal" - } - }, - { - "type": "allreduce", - "args": { - "num_elements": 107520, - "dtype": "bf16", - "world_size": 4 - } - }, - { - "type": "allreduce", - "args": { - "num_elements": 183500800, - "dtype": "bf16", - "world_size": 4 - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 25600, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 4, - "mode": "normal" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 25600, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 4, - "mode": "normal" - } - }, - { - "type": "allreduce", - "args": { - "num_elements": 179200, - "dtype": "bf16", - "world_size": 4 - } - }, - { - "type": "allgather", - "args": { - "num_elements_per_rank": 215040, - "dtype": "bf16", - "world_size": 2 - } - }, - { - "type": "reduce_scatter", - "args": { - "num_elements": 430080, - "dtype": "bf16", - "world_size": 2 - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 30, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 2, - "mode": "low_latency" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 30, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 2, - "mode": "low_latency" - } - }, - { - "type": "allgather", - "args": { - "num_elements_per_rank": 403701760, - "dtype": "bf16", - "world_size": 2 - } - }, - { - "type": "reduce_scatter", - "args": { - "num_elements": 807403520, - "dtype": "bf16", - "world_size": 2 - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 56320, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 2, - "mode": "normal" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 56320, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 2, - "mode": "normal" - } - }, - { - "type": "allgather", - "args": { - "num_elements_per_rank": 394240, - "dtype": "bf16", - "world_size": 2 - } - }, - { - "type": "reduce_scatter", - "args": { - "num_elements": 788480, - "dtype": "bf16", - "world_size": 2 - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 55, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 2, - "mode": "low_latency" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 55, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 2, - "mode": "low_latency" - } - }, - { - "type": "allgather", - "args": { - "num_elements_per_rank": 715653120, - "dtype": "bf16", - "world_size": 2 - } - }, - { - "type": "reduce_scatter", - "args": { - "num_elements": 1431306240, - "dtype": "bf16", - "world_size": 2 - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 99840, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 2, - "mode": "normal" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 99840, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 2, - "mode": "normal" - } - }, - { - "type": "allgather", - "args": { - "num_elements_per_rank": 695296, - "dtype": "bf16", - "world_size": 2 - } - }, - { - "type": "reduce_scatter", - "args": { - "num_elements": 1390592, - "dtype": "bf16", - "world_size": 2 - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 97, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 2, - "mode": "low_latency" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 97, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 2, - "mode": "low_latency" - } - }, - { - "type": "allgather", - "args": { - "num_elements_per_rank": 1449656320, - "dtype": "bf16", - "world_size": 2 - } - }, - { - "type": "reduce_scatter", - "args": { - "num_elements": 2899312640, - "dtype": "bf16", - "world_size": 2 - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 202240, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 2, - "mode": "normal" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 202240, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 2, - "mode": "normal" - } - }, - { - "type": "allgather", - "args": { - "num_elements_per_rank": 1412096, - "dtype": "bf16", - "world_size": 2 - } - }, - { - "type": "reduce_scatter", - "args": { - "num_elements": 2824192, - "dtype": "bf16", - "world_size": 2 - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 197, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 2, - "mode": "low_latency" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 197, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 2, - "mode": "low_latency" - } - }, - { - "type": "allreduce", - "args": { - "num_elements": 807403520, - "dtype": "bf16", - "world_size": 8 - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 112640, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 8, - "mode": "normal" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 112640, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 8, - "mode": "normal" - } - }, - { - "type": "allreduce", - "args": { - "num_elements": 788480, - "dtype": "bf16", - "world_size": 8 - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 110, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 8, - "mode": "low_latency" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 110, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 8, - "mode": "low_latency" - } - }, - { - "type": "allreduce", - "args": { - "num_elements": 1431306240, - "dtype": "bf16", - "world_size": 8 - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 199680, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 8, - "mode": "normal" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 199680, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 8, - "mode": "normal" - } - }, - { - "type": "allreduce", - "args": { - "num_elements": 1397760, - "dtype": "bf16", - "world_size": 8 - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 195, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 8, - "mode": "low_latency" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 195, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 8, - "mode": "low_latency" - } - }, - { - "type": "allreduce", - "args": { - "num_elements": 2899312640, - "dtype": "bf16", - "world_size": 8 - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 404480, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 8, - "mode": "normal" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 404480, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 8, - "mode": "normal" - } - }, - { - "type": "allreduce", - "args": { - "num_elements": 2831360, - "dtype": "bf16", - "world_size": 8 - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 395, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 8, - "mode": "low_latency" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 395, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 8, - "mode": "low_latency" - } - }, - { - "type": "allgather", - "args": { - "num_elements_per_rank": 16166420480, - "dtype": "bf16", - "world_size": 2 - } - }, - { - "type": "reduce_scatter", - "args": { - "num_elements": 32332840960, - "dtype": "bf16", - "world_size": 2 - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 2255360, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 2, - "mode": "normal" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 2255360, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 2, - "mode": "normal" - } - }, - { - "type": "allgather", - "args": { - "num_elements_per_rank": 15783936, - "dtype": "bf16", - "world_size": 2 - } - }, - { - "type": "reduce_scatter", - "args": { - "num_elements": 31567872, - "dtype": "bf16", - "world_size": 2 - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 2202, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 2, - "mode": "low_latency" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 2202, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 2, - "mode": "low_latency" - } - }, - { - "type": "allgather", - "args": { - "num_elements_per_rank": 4041605120, - "dtype": "bf16", - "world_size": 8 - } - }, - { - "type": "reduce_scatter", - "args": { - "num_elements": 32332840960, - "dtype": "bf16", - "world_size": 8 - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 563840, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 8, - "mode": "normal" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 563840, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 8, - "mode": "normal" - } - }, - { - "type": "allgather", - "args": { - "num_elements_per_rank": 3942400, - "dtype": "bf16", - "world_size": 8 - } - }, - { - "type": "reduce_scatter", - "args": { - "num_elements": 31539200, - "dtype": "bf16", - "world_size": 8 - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 550, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 8, - "mode": "low_latency" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 550, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 8, - "mode": "low_latency" - } - }, - { - "type": "allgather", - "args": { - "num_elements_per_rank": 30064771072, - "dtype": "bf16", - "world_size": 2 - } - }, - { - "type": "reduce_scatter", - "args": { - "num_elements": 60129542144, - "dtype": "bf16", - "world_size": 2 - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 4194304, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 2, - "mode": "normal" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 4194304, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 2, - "mode": "normal" - } - }, - { - "type": "allgather", - "args": { - "num_elements_per_rank": 29360128, - "dtype": "bf16", - "world_size": 2 - } - }, - { - "type": "reduce_scatter", - "args": { - "num_elements": 58720256, - "dtype": "bf16", - "world_size": 2 - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 4096, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 2, - "mode": "low_latency" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 4096, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 2, - "mode": "low_latency" - } - }, - { - "type": "allgather", - "args": { - "num_elements_per_rank": 7516192768, - "dtype": "bf16", - "world_size": 8 - } - }, - { - "type": "reduce_scatter", - "args": { - "num_elements": 60129542144, - "dtype": "bf16", - "world_size": 8 - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 1048576, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 8, - "mode": "normal" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 1048576, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 8, - "mode": "normal" - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 1024, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 8, - "mode": "low_latency" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 1024, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 8, - "mode": "low_latency" - } - }, - { - "type": "allgather", - "args": { - "num_elements_per_rank": 16922443776, - "dtype": "bf16", - "world_size": 2 - } - }, - { - "type": "reduce_scatter", - "args": { - "num_elements": 33844887552, - "dtype": "bf16", - "world_size": 2 - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 2360832, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 2, - "mode": "normal" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 2360832, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 2, - "mode": "normal" - } - }, - { - "type": "allgather", - "args": { - "num_elements_per_rank": 16522240, - "dtype": "bf16", - "world_size": 2 - } - }, - { - "type": "reduce_scatter", - "args": { - "num_elements": 33044480, - "dtype": "bf16", - "world_size": 2 - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 2305, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 2, - "mode": "low_latency" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 2305, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 2, - "mode": "low_latency" - } - }, - { - "type": "allgather", - "args": { - "num_elements_per_rank": 4230610944, - "dtype": "bf16", - "world_size": 8 - } - }, - { - "type": "reduce_scatter", - "args": { - "num_elements": 33844887552, - "dtype": "bf16", - "world_size": 8 - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 590208, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 8, - "mode": "normal" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 590208, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 8, - "mode": "normal" - } - }, - { - "type": "allgather", - "args": { - "num_elements_per_rank": 4128768, - "dtype": "bf16", - "world_size": 8 - } - }, - { - "type": "reduce_scatter", - "args": { - "num_elements": 33030144, - "dtype": "bf16", - "world_size": 8 - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 576, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 8, - "mode": "low_latency" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 576, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 8, - "mode": "low_latency" - } - }, - { - "type": "allgather", - "args": { - "num_elements_per_rank": 64533561344, - "dtype": "bf16", - "world_size": 2 - } - }, - { - "type": "reduce_scatter", - "args": { - "num_elements": 129067122688, - "dtype": "bf16", - "world_size": 2 - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 9003008, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 2, - "mode": "normal" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 9003008, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 2, - "mode": "normal" - } - }, - { - "type": "allgather", - "args": { - "num_elements_per_rank": 15755264, - "dtype": "bf16", - "world_size": 2 - } - }, - { - "type": "reduce_scatter", - "args": { - "num_elements": 31510528, - "dtype": "bf16", - "world_size": 2 - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 2198, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 2, - "mode": "low_latency" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 2198, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 2, - "mode": "low_latency" - } - }, - { - "type": "allgather", - "args": { - "num_elements_per_rank": 16133390336, - "dtype": "bf16", - "world_size": 8 - } - }, - { - "type": "reduce_scatter", - "args": { - "num_elements": 129067122688, - "dtype": "bf16", - "world_size": 8 - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 2250752, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 8, - "mode": "normal" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 2250752, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 8, - "mode": "normal" - } - }, - { - "type": "allgather", - "args": { - "num_elements_per_rank": 3935232, - "dtype": "bf16", - "world_size": 8 - } - }, - { - "type": "reduce_scatter", - "args": { - "num_elements": 31481856, - "dtype": "bf16", - "world_size": 8 - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 549, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 8, - "mode": "low_latency" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 549, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 8, - "mode": "low_latency" - } - }, - { - "type": "allgather", - "args": { - "num_elements_per_rank": 1526726656, - "dtype": "bf16", - "world_size": 2 - } - }, - { - "type": "reduce_scatter", - "args": { - "num_elements": 3053453312, - "dtype": "bf16", - "world_size": 2 - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 212992, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 2, - "mode": "normal" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 212992, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 2, - "mode": "normal" - } - }, - { - "type": "allgather", - "args": { - "num_elements_per_rank": 372736, - "dtype": "bf16", - "world_size": 2 - } - }, - { - "type": "reduce_scatter", - "args": { - "num_elements": 745472, - "dtype": "bf16", - "world_size": 2 - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 52, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 2, - "mode": "low_latency" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 52, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 2, - "mode": "low_latency" - } - }, - { - "type": "allreduce", - "args": { - "num_elements": 3053453312, - "dtype": "bf16", - "world_size": 4 - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 425984, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 4, - "mode": "normal" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 425984, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 4, - "mode": "normal" - } - }, - { - "type": "allreduce", - "args": { - "num_elements": 745472, - "dtype": "bf16", - "world_size": 4 - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 104, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 4, - "mode": "low_latency" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 104, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 4, - "mode": "low_latency" - } - }, - { - "type": "allgather", - "args": { - "num_elements_per_rank": 234881024, - "dtype": "bf16", - "world_size": 2 - } - }, - { - "type": "reduce_scatter", - "args": { - "num_elements": 469762048, - "dtype": "bf16", - "world_size": 2 - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 32768, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 2, - "mode": "normal" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 32768, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 2, - "mode": "normal" - } - }, - { - "type": "allgather", - "args": { - "num_elements_per_rank": 57344, - "dtype": "bf16", - "world_size": 2 - } - }, - { - "type": "reduce_scatter", - "args": { - "num_elements": 114688, - "dtype": "bf16", - "world_size": 2 - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 8, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 2, - "mode": "low_latency" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 8, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 2, - "mode": "low_latency" - } - }, - { - "type": "allgather", - "args": { - "num_elements_per_rank": 939524096, - "dtype": "bf16", - "world_size": 2 - } - }, - { - "type": "reduce_scatter", - "args": { - "num_elements": 1879048192, - "dtype": "bf16", - "world_size": 2 - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 131072, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 2, - "mode": "normal" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 131072, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 2, - "mode": "normal" - } - }, - { - "type": "allgather", - "args": { - "num_elements_per_rank": 229376, - "dtype": "bf16", - "world_size": 2 - } - }, - { - "type": "reduce_scatter", - "args": { - "num_elements": 458752, - "dtype": "bf16", - "world_size": 2 - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 32, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 2, - "mode": "low_latency" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 32, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 2, - "mode": "low_latency" - } - }, - { - "type": "allgather", - "args": { - "num_elements_per_rank": 5314183168, - "dtype": "bf16", - "world_size": 2 - } - }, - { - "type": "reduce_scatter", - "args": { - "num_elements": 10628366336, - "dtype": "bf16", - "world_size": 2 - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 741376, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 2, - "mode": "normal" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 741376, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 2, - "mode": "normal" - } - }, - { - "type": "allgather", - "args": { - "num_elements_per_rank": 1297408, - "dtype": "bf16", - "world_size": 2 - } - }, - { - "type": "reduce_scatter", - "args": { - "num_elements": 2594816, - "dtype": "bf16", - "world_size": 2 - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 181, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 2, - "mode": "low_latency" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 181, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 2, - "mode": "low_latency" - } - }, - { - "type": "allgather", - "args": { - "num_elements_per_rank": 1328545792, - "dtype": "bf16", - "world_size": 8 - } - }, - { - "type": "reduce_scatter", - "args": { - "num_elements": 10628366336, - "dtype": "bf16", - "world_size": 8 - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 185344, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 8, - "mode": "normal" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 185344, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 8, - "mode": "normal" - } - }, - { - "type": "allgather", - "args": { - "num_elements_per_rank": 322560, - "dtype": "bf16", - "world_size": 8 - } - }, - { - "type": "reduce_scatter", - "args": { - "num_elements": 2580480, - "dtype": "bf16", - "world_size": 8 - } - }, - { - "type": "allgather", - "args": { - "num_elements_per_rank": 35144073216, - "dtype": "bf16", - "world_size": 2 - } - }, - { - "type": "reduce_scatter", - "args": { - "num_elements": 70288146432, - "dtype": "bf16", - "world_size": 2 - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 4902912, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 2, - "mode": "normal" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 4902912, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 2, - "mode": "normal" - } - }, - { - "type": "allgather", - "args": { - "num_elements_per_rank": 8580096, - "dtype": "bf16", - "world_size": 2 - } - }, - { - "type": "reduce_scatter", - "args": { - "num_elements": 17160192, - "dtype": "bf16", - "world_size": 2 - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 1197, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 2, - "mode": "low_latency" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 1197, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 2, - "mode": "low_latency" - } - }, - { - "type": "allgather", - "args": { - "num_elements_per_rank": 8786018304, - "dtype": "bf16", - "world_size": 8 - } - }, - { - "type": "reduce_scatter", - "args": { - "num_elements": 70288146432, - "dtype": "bf16", - "world_size": 8 - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 1225728, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 8, - "mode": "normal" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 1225728, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 8, - "mode": "normal" - } - }, - { - "type": "allgather", - "args": { - "num_elements_per_rank": 2143232, - "dtype": "bf16", - "world_size": 8 - } - }, - { - "type": "reduce_scatter", - "args": { - "num_elements": 17145856, - "dtype": "bf16", - "world_size": 8 - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 299, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 8, - "mode": "low_latency" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 299, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 8, - "mode": "low_latency" - } - }, - { - "type": "allgather", - "args": { - "num_elements_per_rank": 3082813440, - "dtype": "bf16", - "world_size": 2 - } - }, - { - "type": "reduce_scatter", - "args": { - "num_elements": 6165626880, - "dtype": "bf16", - "world_size": 2 - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 430080, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 2, - "mode": "normal" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 430080, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 2, - "mode": "normal" - } - }, - { - "type": "allreduce", - "args": { - "num_elements": 6165626880, - "dtype": "bf16", - "world_size": 4 - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 860160, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 4, - "mode": "normal" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 860160, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 4, - "mode": "normal" - } - }, - { - "type": "allreduce", - "args": { - "num_elements": 752640, - "dtype": "bf16", - "world_size": 4 - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 105, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 4, - "mode": "low_latency" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 105, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 4, - "mode": "low_latency" - } - }, - { - "type": "allgather", - "args": { - "num_elements_per_rank": 1849688064, - "dtype": "bf16", - "world_size": 2 - } - }, - { - "type": "reduce_scatter", - "args": { - "num_elements": 3699376128, - "dtype": "bf16", - "world_size": 2 - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 258048, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 2, - "mode": "normal" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 258048, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 2, - "mode": "normal" - } - }, - { - "type": "allgather", - "args": { - "num_elements_per_rank": 222208, - "dtype": "bf16", - "world_size": 2 - } - }, - { - "type": "reduce_scatter", - "args": { - "num_elements": 444416, - "dtype": "bf16", - "world_size": 2 - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 31, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 2, - "mode": "low_latency" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 31, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 2, - "mode": "low_latency" - } - }, - { - "type": "allreduce", - "args": { - "num_elements": 3699376128, - "dtype": "bf16", - "world_size": 8 - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 516096, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 8, - "mode": "normal" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 516096, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 8, - "mode": "normal" - } - }, - { - "type": "allreduce", - "args": { - "num_elements": 451584, - "dtype": "bf16", - "world_size": 8 - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 63, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 8, - "mode": "low_latency" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 63, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 8, - "mode": "low_latency" - } - }, - { - "type": "allgather", - "args": { - "num_elements_per_rank": 117440512, - "dtype": "bf16", - "world_size": 2 - } - }, - { - "type": "reduce_scatter", - "args": { - "num_elements": 234881024, - "dtype": "bf16", - "world_size": 2 - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 16384, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 2, - "mode": "normal" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 16384, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 2, - "mode": "normal" - } - }, - { - "type": "allgather", - "args": { - "num_elements_per_rank": 352321536, - "dtype": "bf16", - "world_size": 2 - } - }, - { - "type": "reduce_scatter", - "args": { - "num_elements": 704643072, - "dtype": "bf16", - "world_size": 2 - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 49152, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 2, - "mode": "normal" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 49152, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 2, - "mode": "normal" - } - }, - { - "type": "allgather", - "args": { - "num_elements_per_rank": 43008, - "dtype": "bf16", - "world_size": 2 - } - }, - { - "type": "reduce_scatter", - "args": { - "num_elements": 86016, - "dtype": "bf16", - "world_size": 2 - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 6, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 2, - "mode": "low_latency" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 6, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 2, - "mode": "low_latency" - } - }, - { - "type": "allreduce", - "args": { - "num_elements": 704643072, - "dtype": "bf16", - "world_size": 4 - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 98304, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 4, - "mode": "normal" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 98304, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 4, - "mode": "normal" - } - }, - { - "type": "allreduce", - "args": { - "num_elements": 86016, - "dtype": "bf16", - "world_size": 4 - } - }, - { - "type": "allgather", - "args": { - "num_elements_per_rank": 17293115392, - "dtype": "bf16", - "world_size": 2 - } - }, - { - "type": "reduce_scatter", - "args": { - "num_elements": 34586230784, - "dtype": "bf16", - "world_size": 2 - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 2412544, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 2, - "mode": "normal" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 2412544, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 2, - "mode": "normal" - } - }, - { - "type": "allgather", - "args": { - "num_elements_per_rank": 2107392, - "dtype": "bf16", - "world_size": 2 - } - }, - { - "type": "reduce_scatter", - "args": { - "num_elements": 4214784, - "dtype": "bf16", - "world_size": 2 - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 294, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 2, - "mode": "low_latency" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 294, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 2, - "mode": "low_latency" - } - }, - { - "type": "allgather", - "args": { - "num_elements_per_rank": 4323278848, - "dtype": "bf16", - "world_size": 8 - } - }, - { - "type": "reduce_scatter", - "args": { - "num_elements": 34586230784, - "dtype": "bf16", - "world_size": 8 - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 603136, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 8, - "mode": "normal" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 603136, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 8, - "mode": "normal" - } - }, - { - "type": "allgather", - "args": { - "num_elements_per_rank": 523264, - "dtype": "bf16", - "world_size": 8 - } - }, - { - "type": "reduce_scatter", - "args": { - "num_elements": 4186112, - "dtype": "bf16", - "world_size": 8 - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 73, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 8, - "mode": "low_latency" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 73, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 8, - "mode": "low_latency" - } - }, - { - "type": "allgather", - "args": { - "num_elements_per_rank": 32090619904, - "dtype": "bf16", - "world_size": 2 - } - }, - { - "type": "reduce_scatter", - "args": { - "num_elements": 64181239808, - "dtype": "bf16", - "world_size": 2 - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 4476928, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 2, - "mode": "normal" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 4476928, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 2, - "mode": "normal" - } - }, - { - "type": "allgather", - "args": { - "num_elements_per_rank": 3913728, - "dtype": "bf16", - "world_size": 2 - } - }, - { - "type": "reduce_scatter", - "args": { - "num_elements": 7827456, - "dtype": "bf16", - "world_size": 2 - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 546, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 2, - "mode": "low_latency" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 546, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 2, - "mode": "low_latency" - } - }, - { - "type": "allgather", - "args": { - "num_elements_per_rank": 8022654976, - "dtype": "bf16", - "world_size": 8 - } - }, - { - "type": "reduce_scatter", - "args": { - "num_elements": 64181239808, - "dtype": "bf16", - "world_size": 8 - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 1119232, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 8, - "mode": "normal" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 1119232, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 8, - "mode": "normal" - } - }, - { - "type": "allgather", - "args": { - "num_elements_per_rank": 974848, - "dtype": "bf16", - "world_size": 8 - } - }, - { - "type": "reduce_scatter", - "args": { - "num_elements": 7798784, - "dtype": "bf16", - "world_size": 8 - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 136, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 8, - "mode": "low_latency" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 136, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 8, - "mode": "low_latency" - } - }, - { - "type": "allgather", - "args": { - "num_elements_per_rank": 60129542144, - "dtype": "bf16", - "world_size": 2 - } - }, - { - "type": "reduce_scatter", - "args": { - "num_elements": 120259084288, - "dtype": "bf16", - "world_size": 2 - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 8388608, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 2, - "mode": "normal" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 8388608, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 2, - "mode": "normal" - } - }, - { - "type": "allgather", - "args": { - "num_elements_per_rank": 7340032, - "dtype": "bf16", - "world_size": 2 - } - }, - { - "type": "reduce_scatter", - "args": { - "num_elements": 14680064, - "dtype": "bf16", - "world_size": 2 - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 1024, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 2, - "mode": "low_latency" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 1024, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 2, - "mode": "low_latency" - } - }, - { - "type": "allgather", - "args": { - "num_elements_per_rank": 15032385536, - "dtype": "bf16", - "world_size": 8 - } - }, - { - "type": "reduce_scatter", - "args": { - "num_elements": 120259084288, - "dtype": "bf16", - "world_size": 8 - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 2097152, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 8, - "mode": "normal" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 2097152, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 8, - "mode": "normal" - } - }, - { - "type": "allgather", - "args": { - "num_elements_per_rank": 293601280, - "dtype": "bf16", - "world_size": 4 - } - }, - { - "type": "reduce_scatter", - "args": { - "num_elements": 1174405120, - "dtype": "bf16", - "world_size": 4 - } - }, - { - "type": "allgather", - "args": { - "num_elements_per_rank": 573440, - "dtype": "bf16", - "world_size": 4 - } - }, - { - "type": "reduce_scatter", - "args": { - "num_elements": 2293760, - "dtype": "bf16", - "world_size": 4 - } - }, - { - "type": "allreduce", - "args": { - "num_elements": 1174405120, - "dtype": "bf16", - "world_size": 8 - } - }, - { - "type": "allreduce", - "args": { - "num_elements": 2293760, - "dtype": "bf16", - "world_size": 8 - } - }, - { - "type": "allgather", - "args": { - "num_elements_per_rank": 5637144576, - "dtype": "bf16", - "world_size": 4 - } - }, - { - "type": "reduce_scatter", - "args": { - "num_elements": 22548578304, - "dtype": "bf16", - "world_size": 4 - } - }, - { - "type": "allgather", - "args": { - "num_elements_per_rank": 2818572288, - "dtype": "bf16", - "world_size": 8 - } - }, - { - "type": "reduce_scatter", - "args": { - "num_elements": 22548578304, - "dtype": "bf16", - "world_size": 8 - } - }, - { - "type": "allgather", - "args": { - "num_elements_per_rank": 5505024, - "dtype": "bf16", - "world_size": 8 - } - }, - { - "type": "reduce_scatter", - "args": { - "num_elements": 44040192, - "dtype": "bf16", - "world_size": 8 - } - }, - { - "type": "allgather", - "args": { - "num_elements_per_rank": 4697620480, - "dtype": "bf16", - "world_size": 4 - } - }, - { - "type": "reduce_scatter", - "args": { - "num_elements": 18790481920, - "dtype": "bf16", - "world_size": 4 - } - }, - { - "type": "allgather", - "args": { - "num_elements_per_rank": 9175040, - "dtype": "bf16", - "world_size": 4 - } - }, - { - "type": "reduce_scatter", - "args": { - "num_elements": 36700160, - "dtype": "bf16", - "world_size": 4 - } - }, - { - "type": "allgather", - "args": { - "num_elements_per_rank": 2348810240, - "dtype": "bf16", - "world_size": 8 - } - }, - { - "type": "reduce_scatter", - "args": { - "num_elements": 18790481920, - "dtype": "bf16", - "world_size": 8 - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 327680, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 8, - "mode": "normal" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 327680, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 8, - "mode": "normal" - } - }, - { - "type": "allgather", - "args": { - "num_elements_per_rank": 4587520, - "dtype": "bf16", - "world_size": 8 - } - }, - { - "type": "reduce_scatter", - "args": { - "num_elements": 36700160, - "dtype": "bf16", - "world_size": 8 - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 640, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 8, - "mode": "low_latency" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 640, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 8, - "mode": "low_latency" - } - }, - { - "type": "allgather", - "args": { - "num_elements_per_rank": 2580480, - "dtype": "bf16", - "world_size": 4 - } - }, - { - "type": "reduce_scatter", - "args": { - "num_elements": 10321920, - "dtype": "bf16", - "world_size": 4 - } - }, - { - "type": "allgather", - "args": { - "num_elements_per_rank": 1290240, - "dtype": "bf16", - "world_size": 8 - } - }, - { - "type": "reduce_scatter", - "args": { - "num_elements": 10321920, - "dtype": "bf16", - "world_size": 8 - } - }, - { - "type": "allgather", - "args": { - "num_elements_per_rank": 20669530112, - "dtype": "bf16", - "world_size": 4 - } - }, - { - "type": "reduce_scatter", - "args": { - "num_elements": 82678120448, - "dtype": "bf16", - "world_size": 4 - } - }, - { - "type": "allgather", - "args": { - "num_elements_per_rank": 40370176, - "dtype": "bf16", - "world_size": 4 - } - }, - { - "type": "reduce_scatter", - "args": { - "num_elements": 161480704, - "dtype": "bf16", - "world_size": 4 - } - }, - { - "type": "allgather", - "args": { - "num_elements_per_rank": 10334765056, - "dtype": "bf16", - "world_size": 8 - } - }, - { - "type": "reduce_scatter", - "args": { - "num_elements": 82678120448, - "dtype": "bf16", - "world_size": 8 - } - }, - { - "type": "allgather", - "args": { - "num_elements_per_rank": 20185088, - "dtype": "bf16", - "world_size": 8 - } - }, - { - "type": "reduce_scatter", - "args": { - "num_elements": 161480704, - "dtype": "bf16", - "world_size": 8 - } - }, - { - "type": "allgather", - "args": { - "num_elements_per_rank": 3875536896, - "dtype": "bf16", - "world_size": 4 - } - }, - { - "type": "reduce_scatter", - "args": { - "num_elements": 15502147584, - "dtype": "bf16", - "world_size": 4 - } - }, - { - "type": "allgather", - "args": { - "num_elements_per_rank": 3784704, - "dtype": "bf16", - "world_size": 4 - } - }, - { - "type": "reduce_scatter", - "args": { - "num_elements": 15138816, - "dtype": "bf16", - "world_size": 4 - } - }, - { - "type": "allgather", - "args": { - "num_elements_per_rank": 1937768448, - "dtype": "bf16", - "world_size": 8 - } - }, - { - "type": "reduce_scatter", - "args": { - "num_elements": 15502147584, - "dtype": "bf16", - "world_size": 8 - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 270336, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 8, - "mode": "normal" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 270336, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 8, - "mode": "normal" - } - }, - { - "type": "allgather", - "args": { - "num_elements_per_rank": 1892352, - "dtype": "bf16", - "world_size": 8 - } - }, - { - "type": "reduce_scatter", - "args": { - "num_elements": 15138816, - "dtype": "bf16", - "world_size": 8 - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 264, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 8, - "mode": "low_latency" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 264, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 8, - "mode": "low_latency" - } - }, - { - "type": "allgather", - "args": { - "num_elements_per_rank": 5505024, - "dtype": "bf16", - "world_size": 4 - } - }, - { - "type": "reduce_scatter", - "args": { - "num_elements": 22020096, - "dtype": "bf16", - "world_size": 4 - } - }, - { - "type": "allgather", - "args": { - "num_elements_per_rank": 2752512, - "dtype": "bf16", - "world_size": 8 - } - }, - { - "type": "reduce_scatter", - "args": { - "num_elements": 22020096, - "dtype": "bf16", - "world_size": 8 - } - }, - { - "type": "allgather", - "args": { - "num_elements_per_rank": 2348810240, - "dtype": "bf16", - "world_size": 4 - } - }, - { - "type": "reduce_scatter", - "args": { - "num_elements": 9395240960, - "dtype": "bf16", - "world_size": 4 - } - }, - { - "type": "allgather", - "args": { - "num_elements_per_rank": 2293760, - "dtype": "bf16", - "world_size": 4 - } - }, - { - "type": "reduce_scatter", - "args": { - "num_elements": 9175040, - "dtype": "bf16", - "world_size": 4 - } - }, - { - "type": "allgather", - "args": { - "num_elements_per_rank": 1174405120, - "dtype": "bf16", - "world_size": 8 - } - }, - { - "type": "reduce_scatter", - "args": { - "num_elements": 9395240960, - "dtype": "bf16", - "world_size": 8 - } - }, - { - "type": "allgather", - "args": { - "num_elements_per_rank": 1146880, - "dtype": "bf16", - "world_size": 8 - } - }, - { - "type": "reduce_scatter", - "args": { - "num_elements": 9175040, - "dtype": "bf16", - "world_size": 8 - } - }, - { - "type": "allgather", - "args": { - "num_elements_per_rank": 22020096, - "dtype": "bf16", - "world_size": 4 - } - }, - { - "type": "reduce_scatter", - "args": { - "num_elements": 88080384, - "dtype": "bf16", - "world_size": 4 - } - }, - { - "type": "allreduce", - "args": { - "num_elements": 88080384, - "dtype": "bf16", - "world_size": 8 - } - }, - { - "type": "allgather", - "args": { - "num_elements_per_rank": 229376, - "dtype": "bf16", - "world_size": 4 - } - }, - { - "type": "reduce_scatter", - "args": { - "num_elements": 917504, - "dtype": "bf16", - "world_size": 4 - } - }, - { - "type": "allgather", - "args": { - "num_elements_per_rank": 704643072, - "dtype": "bf16", - "world_size": 4 - } - }, - { - "type": "reduce_scatter", - "args": { - "num_elements": 2818572288, - "dtype": "bf16", - "world_size": 4 - } - }, - { - "type": "allgather", - "args": { - "num_elements_per_rank": 688128, - "dtype": "bf16", - "world_size": 4 - } - }, - { - "type": "reduce_scatter", - "args": { - "num_elements": 2752512, - "dtype": "bf16", - "world_size": 4 - } - }, - { - "type": "allreduce", - "args": { - "num_elements": 2752512, - "dtype": "bf16", - "world_size": 8 - } - }, - { - "type": "allgather", - "args": { - "num_elements_per_rank": 30064771072, - "dtype": "bf16", - "world_size": 4 - } - }, - { - "type": "reduce_scatter", - "args": { - "num_elements": 120259084288, - "dtype": "bf16", - "world_size": 4 - } - }, - { - "type": "allgather", - "args": { - "num_elements_per_rank": 14680064, - "dtype": "bf16", - "world_size": 8 - } - }, - { - "type": "reduce_scatter", - "args": { - "num_elements": 117440512, - "dtype": "bf16", - "world_size": 8 - } - }, - { - "type": "allgather", - "args": { - "num_elements_per_rank": 587202560, - "dtype": "bf16", - "world_size": 4 - } - }, - { - "type": "reduce_scatter", - "args": { - "num_elements": 2348810240, - "dtype": "bf16", - "world_size": 4 - } - }, - { - "type": "allgather", - "args": { - "num_elements_per_rank": 143360, - "dtype": "bf16", - "world_size": 4 - } - }, - { - "type": "reduce_scatter", - "args": { - "num_elements": 573440, - "dtype": "bf16", - "world_size": 4 - } - }, - { - "type": "allreduce", - "args": { - "num_elements": 2348810240, - "dtype": "bf16", - "world_size": 8 - } - }, - { - "type": "allreduce", - "args": { - "num_elements": 573440, - "dtype": "bf16", - "world_size": 8 - } - }, - { - "type": "allgather", - "args": { - "num_elements_per_rank": 71680, - "dtype": "bf16", - "world_size": 4 - } - }, - { - "type": "reduce_scatter", - "args": { - "num_elements": 286720, - "dtype": "bf16", - "world_size": 4 - } - }, - { - "type": "allgather", - "args": { - "num_elements_per_rank": 1056964608, - "dtype": "bf16", - "world_size": 4 - } - }, - { - "type": "reduce_scatter", - "args": { - "num_elements": 4227858432, - "dtype": "bf16", - "world_size": 4 - } - }, - { - "type": "allgather", - "args": { - "num_elements_per_rank": 258048, - "dtype": "bf16", - "world_size": 4 - } - }, - { - "type": "reduce_scatter", - "args": { - "num_elements": 1032192, - "dtype": "bf16", - "world_size": 4 - } - }, - { - "type": "allgather", - "args": { - "num_elements_per_rank": 528482304, - "dtype": "bf16", - "world_size": 8 - } - }, - { - "type": "reduce_scatter", - "args": { - "num_elements": 4227858432, - "dtype": "bf16", - "world_size": 8 - } - }, - { - "type": "allgather", - "args": { - "num_elements_per_rank": 129024, - "dtype": "bf16", - "world_size": 8 - } - }, - { - "type": "reduce_scatter", - "args": { - "num_elements": 1032192, - "dtype": "bf16", - "world_size": 8 - } - }, - { - "type": "allgather", - "args": { - "num_elements_per_rank": 2113929216, - "dtype": "bf16", - "world_size": 4 - } - }, - { - "type": "reduce_scatter", - "args": { - "num_elements": 8455716864, - "dtype": "bf16", - "world_size": 4 - } - }, - { - "type": "allgather", - "args": { - "num_elements_per_rank": 516096, - "dtype": "bf16", - "world_size": 4 - } - }, - { - "type": "reduce_scatter", - "args": { - "num_elements": 2064384, - "dtype": "bf16", - "world_size": 4 - } - }, - { - "type": "allgather", - "args": { - "num_elements_per_rank": 1056964608, - "dtype": "bf16", - "world_size": 8 - } - }, - { - "type": "reduce_scatter", - "args": { - "num_elements": 8455716864, - "dtype": "bf16", - "world_size": 8 - } - }, - { - "type": "allgather", - "args": { - "num_elements_per_rank": 258048, - "dtype": "bf16", - "world_size": 8 - } - }, - { - "type": "reduce_scatter", - "args": { - "num_elements": 2064384, - "dtype": "bf16", - "world_size": 8 - } - }, - { - "type": "allgather", - "args": { - "num_elements_per_rank": 1835008, - "dtype": "bf16", - "world_size": 4 - } - }, - { - "type": "reduce_scatter", - "args": { - "num_elements": 7340032, - "dtype": "bf16", - "world_size": 4 - } - }, - { - "type": "allgather", - "args": { - "num_elements_per_rank": 917504, - "dtype": "bf16", - "world_size": 8 - } - }, - { - "type": "reduce_scatter", - "args": { - "num_elements": 7340032, - "dtype": "bf16", - "world_size": 8 - } - }, - { - "type": "allgather", - "args": { - "num_elements_per_rank": 939524096, - "dtype": "bf16", - "world_size": 4 - } - }, - { - "type": "reduce_scatter", - "args": { - "num_elements": 3758096384, - "dtype": "bf16", - "world_size": 4 - } - }, - { - "type": "allgather", - "args": { - "num_elements_per_rank": 114688, - "dtype": "bf16", - "world_size": 4 - } - }, - { - "type": "reduce_scatter", - "args": { - "num_elements": 458752, - "dtype": "bf16", - "world_size": 4 - } - }, - { - "type": "allgather", - "args": { - "num_elements_per_rank": 3758096384, - "dtype": "bf16", - "world_size": 4 - } - }, - { - "type": "reduce_scatter", - "args": { - "num_elements": 15032385536, - "dtype": "bf16", - "world_size": 4 - } - }, - { - "type": "allgather", - "args": { - "num_elements_per_rank": 458752, - "dtype": "bf16", - "world_size": 4 - } - }, - { - "type": "reduce_scatter", - "args": { - "num_elements": 1835008, - "dtype": "bf16", - "world_size": 4 - } - }, - { - "type": "allgather", - "args": { - "num_elements_per_rank": 917504, - "dtype": "bf16", - "world_size": 4 - } - }, - { - "type": "reduce_scatter", - "args": { - "num_elements": 3670016, - "dtype": "bf16", - "world_size": 4 - } - }, - { - "type": "allreduce", - "args": { - "num_elements": 15032385536, - "dtype": "bf16", - "world_size": 8 - } - }, - { - "type": "allreduce", - "args": { - "num_elements": 1835008, - "dtype": "bf16", - "world_size": 8 - } - }, - { - "type": "allgather", - "args": { - "num_elements_per_rank": 15781068800, - "dtype": "bf16", - "world_size": 4 - } - }, - { - "type": "reduce_scatter", - "args": { - "num_elements": 63124275200, - "dtype": "bf16", - "world_size": 4 - } - }, - { - "type": "allgather", - "args": { - "num_elements_per_rank": 1921024, - "dtype": "bf16", - "world_size": 4 - } - }, - { - "type": "reduce_scatter", - "args": { - "num_elements": 7684096, - "dtype": "bf16", - "world_size": 4 - } - }, - { - "type": "allgather", - "args": { - "num_elements_per_rank": 7890534400, - "dtype": "bf16", - "world_size": 8 - } - }, - { - "type": "reduce_scatter", - "args": { - "num_elements": 63124275200, - "dtype": "bf16", - "world_size": 8 - } - }, - { - "type": "allgather", - "args": { - "num_elements_per_rank": 960512, - "dtype": "bf16", - "world_size": 8 - } - }, - { - "type": "reduce_scatter", - "args": { - "num_elements": 7684096, - "dtype": "bf16", - "world_size": 8 - } - }, - { - "type": "allgather", - "args": { - "num_elements_per_rank": 45097156608, - "dtype": "bf16", - "world_size": 4 - } - }, - { - "type": "reduce_scatter", - "args": { - "num_elements": 180388626432, - "dtype": "bf16", - "world_size": 4 - } - }, - { - "type": "allgather", - "args": { - "num_elements_per_rank": 22548578304, - "dtype": "bf16", - "world_size": 8 - } - }, - { - "type": "reduce_scatter", - "args": { - "num_elements": 180388626432, - "dtype": "bf16", - "world_size": 8 - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 3145728, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 8, - "mode": "normal" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 3145728, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 8, - "mode": "normal" - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 384, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 8, - "mode": "low_latency" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 384, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 8, - "mode": "low_latency" - } - }, - { - "type": "allreduce", - "args": { - "num_elements": 29360128, - "dtype": "bf16", - "world_size": 4 - } - }, - { - "type": "allreduce", - "args": { - "num_elements": 28672, - "dtype": "bf16", - "world_size": 4 - } - }, - { - "type": "allreduce", - "args": { - "num_elements": 58720256, - "dtype": "bf16", - "world_size": 4 - } - }, - { - "type": "allreduce", - "args": { - "num_elements": 57344, - "dtype": "bf16", - "world_size": 4 - } - }, - { - "type": "allreduce", - "args": { - "num_elements": 117440512, - "dtype": "bf16", - "world_size": 4 - } - }, - { - "type": "allreduce", - "args": { - "num_elements": 114688, - "dtype": "bf16", - "world_size": 4 - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 16, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 4, - "mode": "low_latency" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 16, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 4, - "mode": "low_latency" - } - }, - { - "type": "allreduce", - "args": { - "num_elements": 234881024, - "dtype": "bf16", - "world_size": 4 - } - }, - { - "type": "allreduce", - "args": { - "num_elements": 229376, - "dtype": "bf16", - "world_size": 4 - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 32, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 4, - "mode": "low_latency" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 32, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 4, - "mode": "low_latency" - } - }, - { - "type": "allreduce", - "args": { - "num_elements": 469762048, - "dtype": "bf16", - "world_size": 4 - } - }, - { - "type": "allreduce", - "args": { - "num_elements": 458752, - "dtype": "bf16", - "world_size": 4 - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 64, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 4, - "mode": "low_latency" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 64, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 4, - "mode": "low_latency" - } - }, - { - "type": "allreduce", - "args": { - "num_elements": 939524096, - "dtype": "bf16", - "world_size": 4 - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 131072, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 4, - "mode": "normal" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 131072, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 4, - "mode": "normal" - } - }, - { - "type": "allreduce", - "args": { - "num_elements": 917504, - "dtype": "bf16", - "world_size": 4 - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 128, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 4, - "mode": "low_latency" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 128, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 4, - "mode": "low_latency" - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 128, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 8, - "mode": "low_latency" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 128, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 8, - "mode": "low_latency" - } - }, - { - "type": "allreduce", - "args": { - "num_elements": 1879048192, - "dtype": "bf16", - "world_size": 4 - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 262144, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 4, - "mode": "normal" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 262144, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 4, - "mode": "normal" - } - }, - { - "type": "allreduce", - "args": { - "num_elements": 3758096384, - "dtype": "bf16", - "world_size": 4 - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 524288, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 4, - "mode": "normal" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 524288, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 4, - "mode": "normal" - } - }, - { - "type": "allreduce", - "args": { - "num_elements": 7516192768, - "dtype": "bf16", - "world_size": 4 - } - }, - { - "type": "allreduce", - "args": { - "num_elements": 7168, - "dtype": "bf16", - "world_size": 4 - } - }, - { - "type": "allreduce", - "args": { - "num_elements": 14336, - "dtype": "bf16", - "world_size": 4 - } - }, - { - "type": "allreduce", - "args": { - "num_elements": 1409286144, - "dtype": "bf16", - "world_size": 4 - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 196608, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 4, - "mode": "normal" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 196608, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 4, - "mode": "normal" - } - }, - { - "type": "allreduce", - "args": { - "num_elements": 172032, - "dtype": "bf16", - "world_size": 4 - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 24, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 4, - "mode": "low_latency" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 24, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 4, - "mode": "low_latency" - } - }, - { - "type": "allreduce", - "args": { - "num_elements": 2818572288, - "dtype": "bf16", - "world_size": 4 - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 393216, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 4, - "mode": "normal" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 393216, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 4, - "mode": "normal" - } - }, - { - "type": "allreduce", - "args": { - "num_elements": 344064, - "dtype": "bf16", - "world_size": 4 - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 48, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 4, - "mode": "low_latency" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 48, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 4, - "mode": "low_latency" - } - }, - { - "type": "allreduce", - "args": { - "num_elements": 15032385536, - "dtype": "bf16", - "world_size": 4 - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 2097152, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 4, - "mode": "normal" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 2097152, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 4, - "mode": "normal" - } - }, - { - "type": "allreduce", - "args": { - "num_elements": 1835008, - "dtype": "bf16", - "world_size": 4 - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 256, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 4, - "mode": "low_latency" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 256, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 4, - "mode": "low_latency" - } - }, - { - "type": "allreduce", - "args": { - "num_elements": 704643072, - "dtype": "bf16", - "world_size": 8 - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 98304, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 8, - "mode": "normal" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 98304, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 8, - "mode": "normal" - } - }, - { - "type": "allreduce", - "args": { - "num_elements": 30064771072, - "dtype": "bf16", - "world_size": 8 - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 4194304, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 8, - "mode": "normal" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 4194304, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 8, - "mode": "normal" - } - }, - { - "type": "allreduce", - "args": { - "num_elements": 7340032, - "dtype": "bf16", - "world_size": 8 - } - }, - { - "type": "allreduce", - "args": { - "num_elements": 14680064, - "dtype": "bf16", - "world_size": 8 - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 8192, - "num_experts": 384, - "top_k": 6, - "hidden": 7168, - "dtype": "bf16", - "world_size": 8, - "mode": "normal" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 8192, - "num_experts": 384, - "top_k": 6, - "hidden": 7168, - "dtype": "bf16", - "world_size": 8, - "mode": "normal" - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 8, - "num_experts": 384, - "top_k": 6, - "hidden": 7168, - "dtype": "bf16", - "world_size": 8, - "mode": "low_latency" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 8, - "num_experts": 384, - "top_k": 6, - "hidden": 7168, - "dtype": "bf16", - "world_size": 8, - "mode": "low_latency" - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 16384, - "num_experts": 384, - "top_k": 6, - "hidden": 7168, - "dtype": "bf16", - "world_size": 8, - "mode": "normal" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 16384, - "num_experts": 384, - "top_k": 6, - "hidden": 7168, - "dtype": "bf16", - "world_size": 8, - "mode": "normal" - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 16, - "num_experts": 384, - "top_k": 6, - "hidden": 7168, - "dtype": "bf16", - "world_size": 8, - "mode": "low_latency" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 16, - "num_experts": 384, - "top_k": 6, - "hidden": 7168, - "dtype": "bf16", - "world_size": 8, - "mode": "low_latency" - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 32768, - "num_experts": 384, - "top_k": 6, - "hidden": 7168, - "dtype": "bf16", - "world_size": 8, - "mode": "normal" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 32768, - "num_experts": 384, - "top_k": 6, - "hidden": 7168, - "dtype": "bf16", - "world_size": 8, - "mode": "normal" - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 32, - "num_experts": 384, - "top_k": 6, - "hidden": 7168, - "dtype": "bf16", - "world_size": 8, - "mode": "low_latency" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 32, - "num_experts": 384, - "top_k": 6, - "hidden": 7168, - "dtype": "bf16", - "world_size": 8, - "mode": "low_latency" - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 65536, - "num_experts": 384, - "top_k": 6, - "hidden": 7168, - "dtype": "bf16", - "world_size": 8, - "mode": "normal" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 65536, - "num_experts": 384, - "top_k": 6, - "hidden": 7168, - "dtype": "bf16", - "world_size": 8, - "mode": "normal" - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 64, - "num_experts": 384, - "top_k": 6, - "hidden": 7168, - "dtype": "bf16", - "world_size": 8, - "mode": "low_latency" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 64, - "num_experts": 384, - "top_k": 6, - "hidden": 7168, - "dtype": "bf16", - "world_size": 8, - "mode": "low_latency" - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 131072, - "num_experts": 384, - "top_k": 6, - "hidden": 7168, - "dtype": "bf16", - "world_size": 8, - "mode": "normal" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 131072, - "num_experts": 384, - "top_k": 6, - "hidden": 7168, - "dtype": "bf16", - "world_size": 8, - "mode": "normal" - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 128, - "num_experts": 384, - "top_k": 6, - "hidden": 7168, - "dtype": "bf16", - "world_size": 8, - "mode": "low_latency" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 128, - "num_experts": 384, - "top_k": 6, - "hidden": 7168, - "dtype": "bf16", - "world_size": 8, - "mode": "low_latency" - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 262144, - "num_experts": 384, - "top_k": 6, - "hidden": 7168, - "dtype": "bf16", - "world_size": 8, - "mode": "normal" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 262144, - "num_experts": 384, - "top_k": 6, - "hidden": 7168, - "dtype": "bf16", - "world_size": 8, - "mode": "normal" - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 524288, - "num_experts": 384, - "top_k": 6, - "hidden": 7168, - "dtype": "bf16", - "world_size": 8, - "mode": "normal" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 524288, - "num_experts": 384, - "top_k": 6, - "hidden": 7168, - "dtype": "bf16", - "world_size": 8, - "mode": "normal" - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 256, - "num_experts": 384, - "top_k": 6, - "hidden": 7168, - "dtype": "bf16", - "world_size": 8, - "mode": "low_latency" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 256, - "num_experts": 384, - "top_k": 6, - "hidden": 7168, - "dtype": "bf16", - "world_size": 8, - "mode": "low_latency" - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 512, - "num_experts": 384, - "top_k": 6, - "hidden": 7168, - "dtype": "bf16", - "world_size": 8, - "mode": "low_latency" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 512, - "num_experts": 384, - "top_k": 6, - "hidden": 7168, - "dtype": "bf16", - "world_size": 8, - "mode": "low_latency" - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 1048576, - "num_experts": 384, - "top_k": 6, - "hidden": 7168, - "dtype": "bf16", - "world_size": 8, - "mode": "normal" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 1048576, - "num_experts": 384, - "top_k": 6, - "hidden": 7168, - "dtype": "bf16", - "world_size": 8, - "mode": "normal" - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 4096, - "num_experts": 384, - "top_k": 6, - "hidden": 7168, - "dtype": "bf16", - "world_size": 8, - "mode": "normal" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 4096, - "num_experts": 384, - "top_k": 6, - "hidden": 7168, - "dtype": "bf16", - "world_size": 8, - "mode": "normal" - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 4, - "num_experts": 384, - "top_k": 6, - "hidden": 7168, - "dtype": "bf16", - "world_size": 8, - "mode": "low_latency" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 4, - "num_experts": 384, - "top_k": 6, - "hidden": 7168, - "dtype": "bf16", - "world_size": 8, - "mode": "low_latency" - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 1024, - "num_experts": 384, - "top_k": 6, - "hidden": 7168, - "dtype": "bf16", - "world_size": 8, - "mode": "low_latency" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 1024, - "num_experts": 384, - "top_k": 6, - "hidden": 7168, - "dtype": "bf16", - "world_size": 8, - "mode": "low_latency" - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 131072, - "num_experts": 384, - "top_k": 6, - "hidden": 7168, - "dtype": "bf16", - "world_size": 4, - "mode": "normal" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 131072, - "num_experts": 384, - "top_k": 6, - "hidden": 7168, - "dtype": "bf16", - "world_size": 4, - "mode": "normal" - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 128, - "num_experts": 384, - "top_k": 6, - "hidden": 7168, - "dtype": "bf16", - "world_size": 4, - "mode": "low_latency" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 128, - "num_experts": 384, - "top_k": 6, - "hidden": 7168, - "dtype": "bf16", - "world_size": 4, - "mode": "low_latency" - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 1048576, - "num_experts": 384, - "top_k": 6, - "hidden": 7168, - "dtype": "bf16", - "world_size": 4, - "mode": "normal" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 1048576, - "num_experts": 384, - "top_k": 6, - "hidden": 7168, - "dtype": "bf16", - "world_size": 4, - "mode": "normal" - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 2097152, - "num_experts": 384, - "top_k": 6, - "hidden": 7168, - "dtype": "bf16", - "world_size": 8, - "mode": "normal" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 2097152, - "num_experts": 384, - "top_k": 6, - "hidden": 7168, - "dtype": "bf16", - "world_size": 8, - "mode": "normal" - } - }, - { - "type": "allgather", - "args": { - "num_elements_per_rank": 30064771072, - "dtype": "bf16", - "world_size": 8 - } - }, - { - "type": "reduce_scatter", - "args": { - "num_elements": 240518168576, - "dtype": "bf16", - "world_size": 8 - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 4194304, - "num_experts": 384, - "top_k": 6, - "hidden": 7168, - "dtype": "bf16", - "world_size": 8, - "mode": "normal" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 4194304, - "num_experts": 384, - "top_k": 6, - "hidden": 7168, - "dtype": "bf16", - "world_size": 8, - "mode": "normal" - } - }, - { - "type": "allreduce", - "args": { - "num_elements": 16777216, - "dtype": "bf16", - "world_size": 8 - } - }, - { - "type": "allreduce", - "args": { - "num_elements": 16384, - "dtype": "bf16", - "world_size": 8 - } - }, - { - "type": "allreduce", - "args": { - "num_elements": 33554432, - "dtype": "bf16", - "world_size": 8 - } - }, - { - "type": "allreduce", - "args": { - "num_elements": 32768, - "dtype": "bf16", - "world_size": 8 - } - }, - { - "type": "allreduce", - "args": { - "num_elements": 65536, - "dtype": "bf16", - "world_size": 8 - } - }, - { - "type": "allreduce", - "args": { - "num_elements": 134217728, - "dtype": "bf16", - "world_size": 8 - } - }, - { - "type": "allreduce", - "args": { - "num_elements": 131072, - "dtype": "bf16", - "world_size": 8 - } - }, - { - "type": "allreduce", - "args": { - "num_elements": 268435456, - "dtype": "bf16", - "world_size": 8 - } - }, - { - "type": "allreduce", - "args": { - "num_elements": 536870912, - "dtype": "bf16", - "world_size": 8 - } - }, - { - "type": "allreduce", - "args": { - "num_elements": 1073741824, - "dtype": "bf16", - "world_size": 8 - } - }, - { - "type": "allreduce", - "args": { - "num_elements": 2147483648, - "dtype": "bf16", - "world_size": 8 - } - }, - { - "type": "allreduce", - "args": { - "num_elements": 524288, - "dtype": "bf16", - "world_size": 8 - } - }, - { - "type": "allreduce", - "args": { - "num_elements": 16777216, - "dtype": "bf16", - "world_size": 4 - } - }, - { - "type": "allreduce", - "args": { - "num_elements": 16384, - "dtype": "bf16", - "world_size": 4 - } - }, - { - "type": "allreduce", - "args": { - "num_elements": 33554432, - "dtype": "bf16", - "world_size": 4 - } - }, - { - "type": "allreduce", - "args": { - "num_elements": 32768, - "dtype": "bf16", - "world_size": 4 - } - }, - { - "type": "allreduce", - "args": { - "num_elements": 65536, - "dtype": "bf16", - "world_size": 4 - } - }, - { - "type": "allreduce", - "args": { - "num_elements": 134217728, - "dtype": "bf16", - "world_size": 4 - } - }, - { - "type": "allreduce", - "args": { - "num_elements": 131072, - "dtype": "bf16", - "world_size": 4 - } - }, - { - "type": "allreduce", - "args": { - "num_elements": 268435456, - "dtype": "bf16", - "world_size": 4 - } - }, - { - "type": "allreduce", - "args": { - "num_elements": 536870912, - "dtype": "bf16", - "world_size": 4 - } - }, - { - "type": "allreduce", - "args": { - "num_elements": 524288, - "dtype": "bf16", - "world_size": 4 - } - }, - { - "type": "allreduce", - "args": { - "num_elements": 1073741824, - "dtype": "bf16", - "world_size": 4 - } - }, - { - "type": "allreduce", - "args": { - "num_elements": 1048576, - "dtype": "bf16", - "world_size": 4 - } - }, - { - "type": "allreduce", - "args": { - "num_elements": 2147483648, - "dtype": "bf16", - "world_size": 4 - } - }, - { - "type": "allreduce", - "args": { - "num_elements": 4294967296, - "dtype": "bf16", - "world_size": 4 - } - }, - { - "type": "allreduce", - "args": { - "num_elements": 8589934592, - "dtype": "bf16", - "world_size": 4 - } - }, - { - "type": "allreduce", - "args": { - "num_elements": 16777216, - "dtype": "bf16", - "world_size": 2 - } - }, - { - "type": "allreduce", - "args": { - "num_elements": 16384, - "dtype": "bf16", - "world_size": 2 - } - }, - { - "type": "allreduce", - "args": { - "num_elements": 33554432, - "dtype": "bf16", - "world_size": 2 - } - }, - { - "type": "allreduce", - "args": { - "num_elements": 32768, - "dtype": "bf16", - "world_size": 2 - } - }, - { - "type": "allreduce", - "args": { - "num_elements": 65536, - "dtype": "bf16", - "world_size": 2 - } - }, - { - "type": "allreduce", - "args": { - "num_elements": 134217728, - "dtype": "bf16", - "world_size": 2 - } - }, - { - "type": "allreduce", - "args": { - "num_elements": 131072, - "dtype": "bf16", - "world_size": 2 - } - }, - { - "type": "allreduce", - "args": { - "num_elements": 268435456, - "dtype": "bf16", - "world_size": 2 - } - }, - { - "type": "allreduce", - "args": { - "num_elements": 536870912, - "dtype": "bf16", - "world_size": 2 - } - }, - { - "type": "allreduce", - "args": { - "num_elements": 524288, - "dtype": "bf16", - "world_size": 2 - } - }, - { - "type": "allreduce", - "args": { - "num_elements": 1073741824, - "dtype": "bf16", - "world_size": 2 - } - }, - { - "type": "allreduce", - "args": { - "num_elements": 2147483648, - "dtype": "bf16", - "world_size": 2 - } - }, - { - "type": "allreduce", - "args": { - "num_elements": 4294967296, - "dtype": "bf16", - "world_size": 2 - } - }, - { - "type": "allreduce", - "args": { - "num_elements": 25165824, - "dtype": "bf16", - "world_size": 8 - } - }, - { - "type": "allreduce", - "args": { - "num_elements": 24576, - "dtype": "bf16", - "world_size": 8 - } - }, - { - "type": "allreduce", - "args": { - "num_elements": 50331648, - "dtype": "bf16", - "world_size": 8 - } - }, - { - "type": "allreduce", - "args": { - "num_elements": 49152, - "dtype": "bf16", - "world_size": 8 - } - }, - { - "type": "allreduce", - "args": { - "num_elements": 100663296, - "dtype": "bf16", - "world_size": 8 - } - }, - { - "type": "allreduce", - "args": { - "num_elements": 98304, - "dtype": "bf16", - "world_size": 8 - } - }, - { - "type": "allreduce", - "args": { - "num_elements": 201326592, - "dtype": "bf16", - "world_size": 8 - } - }, - { - "type": "allreduce", - "args": { - "num_elements": 196608, - "dtype": "bf16", - "world_size": 8 - } - }, - { - "type": "allreduce", - "args": { - "num_elements": 402653184, - "dtype": "bf16", - "world_size": 8 - } - }, - { - "type": "allreduce", - "args": { - "num_elements": 393216, - "dtype": "bf16", - "world_size": 8 - } - }, - { - "type": "allreduce", - "args": { - "num_elements": 805306368, - "dtype": "bf16", - "world_size": 8 - } - }, - { - "type": "allreduce", - "args": { - "num_elements": 786432, - "dtype": "bf16", - "world_size": 8 - } - }, - { - "type": "allreduce", - "args": { - "num_elements": 1610612736, - "dtype": "bf16", - "world_size": 8 - } - }, - { - "type": "allreduce", - "args": { - "num_elements": 1572864, - "dtype": "bf16", - "world_size": 8 - } - }, - { - "type": "allreduce", - "args": { - "num_elements": 3221225472, - "dtype": "bf16", - "world_size": 8 - } - }, - { - "type": "allreduce", - "args": { - "num_elements": 6442450944, - "dtype": "bf16", - "world_size": 8 - } - }, - { - "type": "allreduce", - "args": { - "num_elements": 12884901888, - "dtype": "bf16", - "world_size": 8 - } - }, - { - "type": "allreduce", - "args": { - "num_elements": 3145728, - "dtype": "bf16", - "world_size": 8 - } - }, - { - "type": "allreduce", - "args": { - "num_elements": 25165824, - "dtype": "bf16", - "world_size": 4 - } - }, - { - "type": "allreduce", - "args": { - "num_elements": 24576, - "dtype": "bf16", - "world_size": 4 - } - }, - { - "type": "allreduce", - "args": { - "num_elements": 50331648, - "dtype": "bf16", - "world_size": 4 - } - }, - { - "type": "allreduce", - "args": { - "num_elements": 49152, - "dtype": "bf16", - "world_size": 4 - } - }, - { - "type": "allreduce", - "args": { - "num_elements": 100663296, - "dtype": "bf16", - "world_size": 4 - } - }, - { - "type": "allreduce", - "args": { - "num_elements": 98304, - "dtype": "bf16", - "world_size": 4 - } - }, - { - "type": "allreduce", - "args": { - "num_elements": 201326592, - "dtype": "bf16", - "world_size": 4 - } - }, - { - "type": "allreduce", - "args": { - "num_elements": 196608, - "dtype": "bf16", - "world_size": 4 - } - }, - { - "type": "allreduce", - "args": { - "num_elements": 402653184, - "dtype": "bf16", - "world_size": 4 - } - }, - { - "type": "allreduce", - "args": { - "num_elements": 393216, - "dtype": "bf16", - "world_size": 4 - } - }, - { - "type": "allreduce", - "args": { - "num_elements": 805306368, - "dtype": "bf16", - "world_size": 4 - } - }, - { - "type": "allreduce", - "args": { - "num_elements": 786432, - "dtype": "bf16", - "world_size": 4 - } - }, - { - "type": "allreduce", - "args": { - "num_elements": 1610612736, - "dtype": "bf16", - "world_size": 4 - } - }, - { - "type": "allreduce", - "args": { - "num_elements": 1572864, - "dtype": "bf16", - "world_size": 4 - } - }, - { - "type": "allreduce", - "args": { - "num_elements": 3221225472, - "dtype": "bf16", - "world_size": 4 - } - }, - { - "type": "allreduce", - "args": { - "num_elements": 6442450944, - "dtype": "bf16", - "world_size": 4 - } - }, - { - "type": "allreduce", - "args": { - "num_elements": 12884901888, - "dtype": "bf16", - "world_size": 4 - } - }, - { - "type": "allreduce", - "args": { - "num_elements": 3145728, - "dtype": "bf16", - "world_size": 4 - } - }, - { - "type": "allreduce", - "args": { - "num_elements": 2097152, - "dtype": "bf16", - "world_size": 4 - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 4096, - "num_experts": 512, - "top_k": 10, - "hidden": 4096, - "dtype": "bf16", - "world_size": 2, - "mode": "normal" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 4096, - "num_experts": 512, - "top_k": 10, - "hidden": 4096, - "dtype": "bf16", - "world_size": 2, - "mode": "normal" - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 4, - "num_experts": 512, - "top_k": 10, - "hidden": 4096, - "dtype": "bf16", - "world_size": 2, - "mode": "low_latency" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 4, - "num_experts": 512, - "top_k": 10, - "hidden": 4096, - "dtype": "bf16", - "world_size": 2, - "mode": "low_latency" - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 8192, - "num_experts": 512, - "top_k": 10, - "hidden": 4096, - "dtype": "bf16", - "world_size": 2, - "mode": "normal" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 8192, - "num_experts": 512, - "top_k": 10, - "hidden": 4096, - "dtype": "bf16", - "world_size": 2, - "mode": "normal" - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 8, - "num_experts": 512, - "top_k": 10, - "hidden": 4096, - "dtype": "bf16", - "world_size": 2, - "mode": "low_latency" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 8, - "num_experts": 512, - "top_k": 10, - "hidden": 4096, - "dtype": "bf16", - "world_size": 2, - "mode": "low_latency" - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 16384, - "num_experts": 512, - "top_k": 10, - "hidden": 4096, - "dtype": "bf16", - "world_size": 2, - "mode": "normal" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 16384, - "num_experts": 512, - "top_k": 10, - "hidden": 4096, - "dtype": "bf16", - "world_size": 2, - "mode": "normal" - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 16, - "num_experts": 512, - "top_k": 10, - "hidden": 4096, - "dtype": "bf16", - "world_size": 2, - "mode": "low_latency" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 16, - "num_experts": 512, - "top_k": 10, - "hidden": 4096, - "dtype": "bf16", - "world_size": 2, - "mode": "low_latency" - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 32768, - "num_experts": 512, - "top_k": 10, - "hidden": 4096, - "dtype": "bf16", - "world_size": 2, - "mode": "normal" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 32768, - "num_experts": 512, - "top_k": 10, - "hidden": 4096, - "dtype": "bf16", - "world_size": 2, - "mode": "normal" - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 32, - "num_experts": 512, - "top_k": 10, - "hidden": 4096, - "dtype": "bf16", - "world_size": 2, - "mode": "low_latency" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 32, - "num_experts": 512, - "top_k": 10, - "hidden": 4096, - "dtype": "bf16", - "world_size": 2, - "mode": "low_latency" - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 65536, - "num_experts": 512, - "top_k": 10, - "hidden": 4096, - "dtype": "bf16", - "world_size": 2, - "mode": "normal" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 65536, - "num_experts": 512, - "top_k": 10, - "hidden": 4096, - "dtype": "bf16", - "world_size": 2, - "mode": "normal" - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 64, - "num_experts": 512, - "top_k": 10, - "hidden": 4096, - "dtype": "bf16", - "world_size": 2, - "mode": "low_latency" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 64, - "num_experts": 512, - "top_k": 10, - "hidden": 4096, - "dtype": "bf16", - "world_size": 2, - "mode": "low_latency" - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 131072, - "num_experts": 512, - "top_k": 10, - "hidden": 4096, - "dtype": "bf16", - "world_size": 2, - "mode": "normal" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 131072, - "num_experts": 512, - "top_k": 10, - "hidden": 4096, - "dtype": "bf16", - "world_size": 2, - "mode": "normal" - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 128, - "num_experts": 512, - "top_k": 10, - "hidden": 4096, - "dtype": "bf16", - "world_size": 2, - "mode": "low_latency" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 128, - "num_experts": 512, - "top_k": 10, - "hidden": 4096, - "dtype": "bf16", - "world_size": 2, - "mode": "low_latency" - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 262144, - "num_experts": 512, - "top_k": 10, - "hidden": 4096, - "dtype": "bf16", - "world_size": 2, - "mode": "normal" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 262144, - "num_experts": 512, - "top_k": 10, - "hidden": 4096, - "dtype": "bf16", - "world_size": 2, - "mode": "normal" - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 524288, - "num_experts": 512, - "top_k": 10, - "hidden": 4096, - "dtype": "bf16", - "world_size": 2, - "mode": "normal" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 524288, - "num_experts": 512, - "top_k": 10, - "hidden": 4096, - "dtype": "bf16", - "world_size": 2, - "mode": "normal" - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 1048576, - "num_experts": 512, - "top_k": 10, - "hidden": 4096, - "dtype": "bf16", - "world_size": 2, - "mode": "normal" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 1048576, - "num_experts": 512, - "top_k": 10, - "hidden": 4096, - "dtype": "bf16", - "world_size": 2, - "mode": "normal" - } - }, - { - "type": "allreduce", - "args": { - "num_elements": 1048576, - "dtype": "bf16", - "world_size": 2 - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 256, - "num_experts": 512, - "top_k": 10, - "hidden": 4096, - "dtype": "bf16", - "world_size": 2, - "mode": "low_latency" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 256, - "num_experts": 512, - "top_k": 10, - "hidden": 4096, - "dtype": "bf16", - "world_size": 2, - "mode": "low_latency" - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 128, - "num_experts": 384, - "top_k": 6, - "hidden": 7168, - "dtype": "bf16", - "world_size": 8, - "mode": "normal" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 128, - "num_experts": 384, - "top_k": 6, - "hidden": 7168, - "dtype": "bf16", - "world_size": 8, - "mode": "normal" - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 0, - "num_experts": 384, - "top_k": 6, - "hidden": 7168, - "dtype": "bf16", - "world_size": 8, - "mode": "low_latency" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 0, - "num_experts": 384, - "top_k": 6, - "hidden": 7168, - "dtype": "bf16", - "world_size": 8, - "mode": "low_latency" - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 256, - "num_experts": 384, - "top_k": 6, - "hidden": 7168, - "dtype": "bf16", - "world_size": 8, - "mode": "normal" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 256, - "num_experts": 384, - "top_k": 6, - "hidden": 7168, - "dtype": "bf16", - "world_size": 8, - "mode": "normal" - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 512, - "num_experts": 384, - "top_k": 6, - "hidden": 7168, - "dtype": "bf16", - "world_size": 8, - "mode": "normal" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 512, - "num_experts": 384, - "top_k": 6, - "hidden": 7168, - "dtype": "bf16", - "world_size": 8, - "mode": "normal" - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 1024, - "num_experts": 384, - "top_k": 6, - "hidden": 7168, - "dtype": "bf16", - "world_size": 8, - "mode": "normal" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 1024, - "num_experts": 384, - "top_k": 6, - "hidden": 7168, - "dtype": "bf16", - "world_size": 8, - "mode": "normal" - } - }, - { - "type": "allgather", - "args": { - "num_elements_per_rank": 7168, - "dtype": "bf16", - "world_size": 8 - } - }, - { - "type": "reduce_scatter", - "args": { - "num_elements": 57344, - "dtype": "bf16", - "world_size": 8 - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 1, - "num_experts": 384, - "top_k": 6, - "hidden": 7168, - "dtype": "bf16", - "world_size": 8, - "mode": "low_latency" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 1, - "num_experts": 384, - "top_k": 6, - "hidden": 7168, - "dtype": "bf16", - "world_size": 8, - "mode": "low_latency" - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 2048, - "num_experts": 384, - "top_k": 6, - "hidden": 7168, - "dtype": "bf16", - "world_size": 8, - "mode": "normal" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 2048, - "num_experts": 384, - "top_k": 6, - "hidden": 7168, - "dtype": "bf16", - "world_size": 8, - "mode": "normal" - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 2, - "num_experts": 384, - "top_k": 6, - "hidden": 7168, - "dtype": "bf16", - "world_size": 8, - "mode": "low_latency" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 2, - "num_experts": 384, - "top_k": 6, - "hidden": 7168, - "dtype": "bf16", - "world_size": 8, - "mode": "low_latency" - } - }, - { - "type": "allreduce", - "args": { - "num_elements": 7340032, - "dtype": "bf16", - "world_size": 4 - } - }, - { - "type": "allreduce", - "args": { - "num_elements": 14680064, - "dtype": "bf16", - "world_size": 4 - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 65536, - "num_experts": 384, - "top_k": 6, - "hidden": 7168, - "dtype": "bf16", - "world_size": 4, - "mode": "normal" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 65536, - "num_experts": 384, - "top_k": 6, - "hidden": 7168, - "dtype": "bf16", - "world_size": 4, - "mode": "normal" - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 64, - "num_experts": 384, - "top_k": 6, - "hidden": 7168, - "dtype": "bf16", - "world_size": 4, - "mode": "low_latency" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 64, - "num_experts": 384, - "top_k": 6, - "hidden": 7168, - "dtype": "bf16", - "world_size": 4, - "mode": "low_latency" - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 524288, - "num_experts": 384, - "top_k": 6, - "hidden": 7168, - "dtype": "bf16", - "world_size": 4, - "mode": "normal" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 524288, - "num_experts": 384, - "top_k": 6, - "hidden": 7168, - "dtype": "bf16", - "world_size": 4, - "mode": "normal" - } - }, - { - "type": "allgather", - "args": { - "num_elements_per_rank": 3670016, - "dtype": "bf16", - "world_size": 4 - } - }, - { - "type": "reduce_scatter", - "args": { - "num_elements": 14680064, - "dtype": "bf16", - "world_size": 4 - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 512, - "num_experts": 384, - "top_k": 6, - "hidden": 7168, - "dtype": "bf16", - "world_size": 4, - "mode": "low_latency" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 512, - "num_experts": 384, - "top_k": 6, - "hidden": 7168, - "dtype": "bf16", - "world_size": 4, - "mode": "low_latency" - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 16384, - "num_experts": 384, - "top_k": 6, - "hidden": 7168, - "dtype": "bf16", - "world_size": 4, - "mode": "normal" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 16384, - "num_experts": 384, - "top_k": 6, - "hidden": 7168, - "dtype": "bf16", - "world_size": 4, - "mode": "normal" - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 16, - "num_experts": 384, - "top_k": 6, - "hidden": 7168, - "dtype": "bf16", - "world_size": 4, - "mode": "low_latency" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 16, - "num_experts": 384, - "top_k": 6, - "hidden": 7168, - "dtype": "bf16", - "world_size": 4, - "mode": "low_latency" - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 32768, - "num_experts": 384, - "top_k": 6, - "hidden": 7168, - "dtype": "bf16", - "world_size": 4, - "mode": "normal" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 32768, - "num_experts": 384, - "top_k": 6, - "hidden": 7168, - "dtype": "bf16", - "world_size": 4, - "mode": "normal" - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 32, - "num_experts": 384, - "top_k": 6, - "hidden": 7168, - "dtype": "bf16", - "world_size": 4, - "mode": "low_latency" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 32, - "num_experts": 384, - "top_k": 6, - "hidden": 7168, - "dtype": "bf16", - "world_size": 4, - "mode": "low_latency" - } - }, - { - "type": "allreduce", - "args": { - "num_elements": 3670016, - "dtype": "bf16", - "world_size": 4 - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 256, - "num_experts": 384, - "top_k": 6, - "hidden": 7168, - "dtype": "bf16", - "world_size": 4, - "mode": "low_latency" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 256, - "num_experts": 384, - "top_k": 6, - "hidden": 7168, - "dtype": "bf16", - "world_size": 4, - "mode": "low_latency" - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 2097152, - "num_experts": 384, - "top_k": 6, - "hidden": 7168, - "dtype": "bf16", - "world_size": 4, - "mode": "normal" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 2097152, - "num_experts": 384, - "top_k": 6, - "hidden": 7168, - "dtype": "bf16", - "world_size": 4, - "mode": "normal" - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 4096, - "num_experts": 512, - "top_k": 10, - "hidden": 4096, - "dtype": "bf16", - "world_size": 8, - "mode": "normal" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 4096, - "num_experts": 512, - "top_k": 10, - "hidden": 4096, - "dtype": "bf16", - "world_size": 8, - "mode": "normal" - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 4, - "num_experts": 512, - "top_k": 10, - "hidden": 4096, - "dtype": "bf16", - "world_size": 8, - "mode": "low_latency" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 4, - "num_experts": 512, - "top_k": 10, - "hidden": 4096, - "dtype": "bf16", - "world_size": 8, - "mode": "low_latency" - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 8192, - "num_experts": 512, - "top_k": 10, - "hidden": 4096, - "dtype": "bf16", - "world_size": 8, - "mode": "normal" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 8192, - "num_experts": 512, - "top_k": 10, - "hidden": 4096, - "dtype": "bf16", - "world_size": 8, - "mode": "normal" - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 8, - "num_experts": 512, - "top_k": 10, - "hidden": 4096, - "dtype": "bf16", - "world_size": 8, - "mode": "low_latency" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 8, - "num_experts": 512, - "top_k": 10, - "hidden": 4096, - "dtype": "bf16", - "world_size": 8, - "mode": "low_latency" - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 16384, - "num_experts": 512, - "top_k": 10, - "hidden": 4096, - "dtype": "bf16", - "world_size": 8, - "mode": "normal" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 16384, - "num_experts": 512, - "top_k": 10, - "hidden": 4096, - "dtype": "bf16", - "world_size": 8, - "mode": "normal" - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 16, - "num_experts": 512, - "top_k": 10, - "hidden": 4096, - "dtype": "bf16", - "world_size": 8, - "mode": "low_latency" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 16, - "num_experts": 512, - "top_k": 10, - "hidden": 4096, - "dtype": "bf16", - "world_size": 8, - "mode": "low_latency" - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 32768, - "num_experts": 512, - "top_k": 10, - "hidden": 4096, - "dtype": "bf16", - "world_size": 8, - "mode": "normal" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 32768, - "num_experts": 512, - "top_k": 10, - "hidden": 4096, - "dtype": "bf16", - "world_size": 8, - "mode": "normal" - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 32, - "num_experts": 512, - "top_k": 10, - "hidden": 4096, - "dtype": "bf16", - "world_size": 8, - "mode": "low_latency" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 32, - "num_experts": 512, - "top_k": 10, - "hidden": 4096, - "dtype": "bf16", - "world_size": 8, - "mode": "low_latency" - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 65536, - "num_experts": 512, - "top_k": 10, - "hidden": 4096, - "dtype": "bf16", - "world_size": 8, - "mode": "normal" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 65536, - "num_experts": 512, - "top_k": 10, - "hidden": 4096, - "dtype": "bf16", - "world_size": 8, - "mode": "normal" - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 64, - "num_experts": 512, - "top_k": 10, - "hidden": 4096, - "dtype": "bf16", - "world_size": 8, - "mode": "low_latency" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 64, - "num_experts": 512, - "top_k": 10, - "hidden": 4096, - "dtype": "bf16", - "world_size": 8, - "mode": "low_latency" - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 131072, - "num_experts": 512, - "top_k": 10, - "hidden": 4096, - "dtype": "bf16", - "world_size": 8, - "mode": "normal" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 131072, - "num_experts": 512, - "top_k": 10, - "hidden": 4096, - "dtype": "bf16", - "world_size": 8, - "mode": "normal" - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 262144, - "num_experts": 512, - "top_k": 10, - "hidden": 4096, - "dtype": "bf16", - "world_size": 8, - "mode": "normal" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 262144, - "num_experts": 512, - "top_k": 10, - "hidden": 4096, - "dtype": "bf16", - "world_size": 8, - "mode": "normal" - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 524288, - "num_experts": 512, - "top_k": 10, - "hidden": 4096, - "dtype": "bf16", - "world_size": 8, - "mode": "normal" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 524288, - "num_experts": 512, - "top_k": 10, - "hidden": 4096, - "dtype": "bf16", - "world_size": 8, - "mode": "normal" - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 128, - "num_experts": 512, - "top_k": 10, - "hidden": 4096, - "dtype": "bf16", - "world_size": 8, - "mode": "low_latency" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 128, - "num_experts": 512, - "top_k": 10, - "hidden": 4096, - "dtype": "bf16", - "world_size": 8, - "mode": "low_latency" - } - }, - { - "type": "allreduce", - "args": { - "num_elements": 1048576, - "dtype": "bf16", - "world_size": 8 - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 256, - "num_experts": 512, - "top_k": 10, - "hidden": 4096, - "dtype": "bf16", - "world_size": 8, - "mode": "low_latency" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 256, - "num_experts": 512, - "top_k": 10, - "hidden": 4096, - "dtype": "bf16", - "world_size": 8, - "mode": "low_latency" - } - }, - { - "type": "allreduce", - "args": { - "num_elements": 4294967296, - "dtype": "bf16", - "world_size": 8 - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 1048576, - "num_experts": 512, - "top_k": 10, - "hidden": 4096, - "dtype": "bf16", - "world_size": 8, - "mode": "normal" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 1048576, - "num_experts": 512, - "top_k": 10, - "hidden": 4096, - "dtype": "bf16", - "world_size": 8, - "mode": "normal" - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 128, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 8, - "mode": "normal" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 128, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 8, - "mode": "normal" - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 2048, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 8, - "mode": "normal" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 2048, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 8, - "mode": "normal" - } - }, - { - "type": "allgather", - "args": { - "num_elements_per_rank": 2752512, - "dtype": "bf16", - "world_size": 16 - } - }, - { - "type": "reduce_scatter", - "args": { - "num_elements": 44040192, - "dtype": "bf16", - "world_size": 16 - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 384, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 16, - "mode": "normal" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 384, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 16, - "mode": "normal" - } - }, - { - "type": "allgather", - "args": { - "num_elements_per_rank": 0, - "dtype": "bf16", - "world_size": 16 - } - }, - { - "type": "reduce_scatter", - "args": { - "num_elements": 0, - "dtype": "bf16", - "world_size": 16 - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 0, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 16, - "mode": "low_latency" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 0, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 16, - "mode": "low_latency" - } - }, - { - "type": "allreduce", - "args": { - "num_elements": 44040192, - "dtype": "bf16", - "world_size": 16 - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 6144, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 16, - "mode": "normal" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 6144, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 16, - "mode": "normal" - } - }, - { - "type": "allreduce", - "args": { - "num_elements": 86016, - "dtype": "bf16", - "world_size": 16 - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 12, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 16, - "mode": "low_latency" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 12, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 16, - "mode": "low_latency" - } - }, - { - "type": "allgather", - "args": { - "num_elements_per_rank": 4128768, - "dtype": "bf16", - "world_size": 16 - } - }, - { - "type": "reduce_scatter", - "args": { - "num_elements": 66060288, - "dtype": "bf16", - "world_size": 16 - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 576, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 16, - "mode": "normal" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 576, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 16, - "mode": "normal" - } - }, - { - "type": "allgather", - "args": { - "num_elements_per_rank": 7168, - "dtype": "bf16", - "world_size": 16 - } - }, - { - "type": "reduce_scatter", - "args": { - "num_elements": 114688, - "dtype": "bf16", - "world_size": 16 - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 1, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 16, - "mode": "low_latency" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 1, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 16, - "mode": "low_latency" - } - }, - { - "type": "allreduce", - "args": { - "num_elements": 66060288, - "dtype": "bf16", - "world_size": 16 - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 9216, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 16, - "mode": "normal" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 9216, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 16, - "mode": "normal" - } - }, - { - "type": "allreduce", - "args": { - "num_elements": 129024, - "dtype": "bf16", - "world_size": 16 - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 18, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 16, - "mode": "low_latency" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 18, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 16, - "mode": "low_latency" - } - }, - { - "type": "allgather", - "args": { - "num_elements_per_rank": 13762560, - "dtype": "bf16", - "world_size": 16 - } - }, - { - "type": "reduce_scatter", - "args": { - "num_elements": 220200960, - "dtype": "bf16", - "world_size": 16 - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 1920, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 16, - "mode": "normal" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 1920, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 16, - "mode": "normal" - } - }, - { - "type": "allgather", - "args": { - "num_elements_per_rank": 21504, - "dtype": "bf16", - "world_size": 16 - } - }, - { - "type": "reduce_scatter", - "args": { - "num_elements": 344064, - "dtype": "bf16", - "world_size": 16 - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 3, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 16, - "mode": "low_latency" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 3, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 16, - "mode": "low_latency" - } - }, - { - "type": "allreduce", - "args": { - "num_elements": 220200960, - "dtype": "bf16", - "world_size": 16 - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 30720, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 16, - "mode": "normal" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 30720, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 16, - "mode": "normal" - } - }, - { - "type": "allreduce", - "args": { - "num_elements": 430080, - "dtype": "bf16", - "world_size": 16 - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 60, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 16, - "mode": "low_latency" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 60, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 16, - "mode": "low_latency" - } - }, - { - "type": "allgather", - "args": { - "num_elements_per_rank": 27525120, - "dtype": "bf16", - "world_size": 16 - } - }, - { - "type": "reduce_scatter", - "args": { - "num_elements": 440401920, - "dtype": "bf16", - "world_size": 16 - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 3840, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 16, - "mode": "normal" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 3840, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 16, - "mode": "normal" - } - }, - { - "type": "allgather", - "args": { - "num_elements_per_rank": 50176, - "dtype": "bf16", - "world_size": 16 - } - }, - { - "type": "reduce_scatter", - "args": { - "num_elements": 802816, - "dtype": "bf16", - "world_size": 16 - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 7, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 16, - "mode": "low_latency" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 7, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 16, - "mode": "low_latency" - } - }, - { - "type": "allreduce", - "args": { - "num_elements": 440401920, - "dtype": "bf16", - "world_size": 16 - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 61440, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 16, - "mode": "normal" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 61440, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 16, - "mode": "normal" - } - }, - { - "type": "allreduce", - "args": { - "num_elements": 860160, - "dtype": "bf16", - "world_size": 16 - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 120, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 16, - "mode": "low_latency" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 120, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 16, - "mode": "low_latency" - } - }, - { - "type": "allgather", - "args": { - "num_elements_per_rank": 53673984, - "dtype": "bf16", - "world_size": 16 - } - }, - { - "type": "reduce_scatter", - "args": { - "num_elements": 858783744, - "dtype": "bf16", - "world_size": 16 - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 7488, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 16, - "mode": "normal" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 7488, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 16, - "mode": "normal" - } - }, - { - "type": "allgather", - "args": { - "num_elements_per_rank": 100352, - "dtype": "bf16", - "world_size": 16 - } - }, - { - "type": "reduce_scatter", - "args": { - "num_elements": 1605632, - "dtype": "bf16", - "world_size": 16 - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 14, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 16, - "mode": "low_latency" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 14, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 16, - "mode": "low_latency" - } - }, - { - "type": "allreduce", - "args": { - "num_elements": 858783744, - "dtype": "bf16", - "world_size": 16 - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 119808, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 16, - "mode": "normal" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 119808, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 16, - "mode": "normal" - } - }, - { - "type": "allreduce", - "args": { - "num_elements": 1677312, - "dtype": "bf16", - "world_size": 16 - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 234, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 16, - "mode": "low_latency" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 234, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 16, - "mode": "low_latency" - } - }, - { - "type": "allgather", - "args": { - "num_elements_per_rank": 105971712, - "dtype": "bf16", - "world_size": 16 - } - }, - { - "type": "reduce_scatter", - "args": { - "num_elements": 1695547392, - "dtype": "bf16", - "world_size": 16 - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 14784, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 16, - "mode": "normal" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 14784, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 16, - "mode": "normal" - } - }, - { - "type": "allgather", - "args": { - "num_elements_per_rank": 200704, - "dtype": "bf16", - "world_size": 16 - } - }, - { - "type": "reduce_scatter", - "args": { - "num_elements": 3211264, - "dtype": "bf16", - "world_size": 16 - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 28, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 16, - "mode": "low_latency" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 28, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 16, - "mode": "low_latency" - } - }, - { - "type": "allgather", - "args": { - "num_elements_per_rank": 211943424, - "dtype": "bf16", - "world_size": 16 - } - }, - { - "type": "reduce_scatter", - "args": { - "num_elements": 3391094784, - "dtype": "bf16", - "world_size": 16 - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 29568, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 16, - "mode": "normal" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 29568, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 16, - "mode": "normal" - } - }, - { - "type": "allgather", - "args": { - "num_elements_per_rank": 408576, - "dtype": "bf16", - "world_size": 16 - } - }, - { - "type": "reduce_scatter", - "args": { - "num_elements": 6537216, - "dtype": "bf16", - "world_size": 16 - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 57, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 16, - "mode": "low_latency" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 57, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 16, - "mode": "low_latency" - } - }, - { - "type": "allreduce", - "args": { - "num_elements": 3391094784, - "dtype": "bf16", - "world_size": 16 - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 473088, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 16, - "mode": "normal" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 473088, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 16, - "mode": "normal" - } - }, - { - "type": "allreduce", - "args": { - "num_elements": 6623232, - "dtype": "bf16", - "world_size": 16 - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 924, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 16, - "mode": "low_latency" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 924, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 16, - "mode": "low_latency" - } - }, - { - "type": "allgather", - "args": { - "num_elements_per_rank": 282132480, - "dtype": "bf16", - "world_size": 16 - } - }, - { - "type": "reduce_scatter", - "args": { - "num_elements": 4514119680, - "dtype": "bf16", - "world_size": 16 - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 39360, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 16, - "mode": "normal" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 39360, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 16, - "mode": "normal" - } - }, - { - "type": "allgather", - "args": { - "num_elements_per_rank": 544768, - "dtype": "bf16", - "world_size": 16 - } - }, - { - "type": "reduce_scatter", - "args": { - "num_elements": 8716288, - "dtype": "bf16", - "world_size": 16 - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 76, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 16, - "mode": "low_latency" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 76, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 16, - "mode": "low_latency" - } - }, - { - "type": "allgather", - "args": { - "num_elements_per_rank": 563806208, - "dtype": "bf16", - "world_size": 16 - } - }, - { - "type": "reduce_scatter", - "args": { - "num_elements": 9020899328, - "dtype": "bf16", - "world_size": 16 - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 78656, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 16, - "mode": "normal" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 78656, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 16, - "mode": "normal" - } - }, - { - "type": "allgather", - "args": { - "num_elements_per_rank": 1096704, - "dtype": "bf16", - "world_size": 16 - } - }, - { - "type": "reduce_scatter", - "args": { - "num_elements": 17547264, - "dtype": "bf16", - "world_size": 16 - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 153, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 16, - "mode": "low_latency" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 153, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 16, - "mode": "low_latency" - } - }, - { - "type": "allreduce", - "args": { - "num_elements": 43008, - "dtype": "bf16", - "world_size": 16 - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 6, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 16, - "mode": "low_latency" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 6, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 16, - "mode": "low_latency" - } - }, - { - "type": "allreduce", - "args": { - "num_elements": 64512, - "dtype": "bf16", - "world_size": 16 - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 9, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 16, - "mode": "low_latency" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 9, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 16, - "mode": "low_latency" - } - }, - { - "type": "allreduce", - "args": { - "num_elements": 215040, - "dtype": "bf16", - "world_size": 16 - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 30, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 16, - "mode": "low_latency" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 30, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 16, - "mode": "low_latency" - } - }, - { - "type": "allgather", - "args": { - "num_elements_per_rank": 423886848, - "dtype": "bf16", - "world_size": 16 - } - }, - { - "type": "reduce_scatter", - "args": { - "num_elements": 6782189568, - "dtype": "bf16", - "world_size": 16 - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 59136, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 16, - "mode": "normal" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 59136, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 16, - "mode": "normal" - } - }, - { - "type": "allgather", - "args": { - "num_elements_per_rank": 846397440, - "dtype": "bf16", - "world_size": 16 - } - }, - { - "type": "reduce_scatter", - "args": { - "num_elements": 13542359040, - "dtype": "bf16", - "world_size": 16 - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 118080, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 16, - "mode": "normal" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 118080, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 16, - "mode": "normal" - } - }, - { - "type": "allgather", - "args": { - "num_elements_per_rank": 824320, - "dtype": "bf16", - "world_size": 16 - } - }, - { - "type": "reduce_scatter", - "args": { - "num_elements": 13189120, - "dtype": "bf16", - "world_size": 16 - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 115, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 16, - "mode": "low_latency" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 115, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 16, - "mode": "low_latency" - } - }, - { - "type": "allgather", - "args": { - "num_elements_per_rank": 2255224832, - "dtype": "bf16", - "world_size": 16 - } - }, - { - "type": "reduce_scatter", - "args": { - "num_elements": 36083597312, - "dtype": "bf16", - "world_size": 16 - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 314624, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 16, - "mode": "normal" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 314624, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 16, - "mode": "normal" - } - }, - { - "type": "allgather", - "args": { - "num_elements_per_rank": 2200576, - "dtype": "bf16", - "world_size": 16 - } - }, - { - "type": "reduce_scatter", - "args": { - "num_elements": 35209216, - "dtype": "bf16", - "world_size": 16 - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 307, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 16, - "mode": "low_latency" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 307, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 16, - "mode": "low_latency" - } - }, - { - "type": "allgather", - "args": { - "num_elements_per_rank": 22020096, - "dtype": "bf16", - "world_size": 16 - } - }, - { - "type": "reduce_scatter", - "args": { - "num_elements": 352321536, - "dtype": "bf16", - "world_size": 16 - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 3072, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 16, - "mode": "normal" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 3072, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 16, - "mode": "normal" - } - }, - { - "type": "allreduce", - "args": { - "num_elements": 352321536, - "dtype": "bf16", - "world_size": 16 - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 49152, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 16, - "mode": "normal" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 49152, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 16, - "mode": "normal" - } - }, - { - "type": "allgather", - "args": { - "num_elements_per_rank": 33030144, - "dtype": "bf16", - "world_size": 16 - } - }, - { - "type": "reduce_scatter", - "args": { - "num_elements": 528482304, - "dtype": "bf16", - "world_size": 16 - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 4608, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 16, - "mode": "normal" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 4608, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 16, - "mode": "normal" - } - }, - { - "type": "allreduce", - "args": { - "num_elements": 528482304, - "dtype": "bf16", - "world_size": 16 - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 73728, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 16, - "mode": "normal" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 73728, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 16, - "mode": "normal" - } - }, - { - "type": "allgather", - "args": { - "num_elements_per_rank": 110100480, - "dtype": "bf16", - "world_size": 16 - } - }, - { - "type": "reduce_scatter", - "args": { - "num_elements": 1761607680, - "dtype": "bf16", - "world_size": 16 - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 15360, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 16, - "mode": "normal" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 15360, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 16, - "mode": "normal" - } - }, - { - "type": "allreduce", - "args": { - "num_elements": 1761607680, - "dtype": "bf16", - "world_size": 16 - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 245760, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 16, - "mode": "normal" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 245760, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 16, - "mode": "normal" - } - }, - { - "type": "allgather", - "args": { - "num_elements_per_rank": 282591232, - "dtype": "bf16", - "world_size": 16 - } - }, - { - "type": "reduce_scatter", - "args": { - "num_elements": 4521459712, - "dtype": "bf16", - "world_size": 16 - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 39424, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 16, - "mode": "normal" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 39424, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 16, - "mode": "normal" - } - }, - { - "type": "allgather", - "args": { - "num_elements_per_rank": 64512, - "dtype": "bf16", - "world_size": 16 - } - }, - { - "type": "reduce_scatter", - "args": { - "num_elements": 1032192, - "dtype": "bf16", - "world_size": 16 - } - }, - { - "type": "allgather", - "args": { - "num_elements_per_rank": 565182464, - "dtype": "bf16", - "world_size": 16 - } - }, - { - "type": "reduce_scatter", - "args": { - "num_elements": 9042919424, - "dtype": "bf16", - "world_size": 16 - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 78848, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 16, - "mode": "normal" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 78848, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 16, - "mode": "normal" - } - }, - { - "type": "allgather", - "args": { - "num_elements_per_rank": 136192, - "dtype": "bf16", - "world_size": 16 - } - }, - { - "type": "reduce_scatter", - "args": { - "num_elements": 2179072, - "dtype": "bf16", - "world_size": 16 - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 19, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 16, - "mode": "low_latency" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 19, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 16, - "mode": "low_latency" - } - }, - { - "type": "allreduce", - "args": { - "num_elements": 9042919424, - "dtype": "bf16", - "world_size": 16 - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 1261568, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 16, - "mode": "normal" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 1261568, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 16, - "mode": "normal" - } - }, - { - "type": "allreduce", - "args": { - "num_elements": 1103872, - "dtype": "bf16", - "world_size": 16 - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 154, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 16, - "mode": "low_latency" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 154, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 16, - "mode": "low_latency" - } - }, - { - "type": "allgather", - "args": { - "num_elements_per_rank": 1130364928, - "dtype": "bf16", - "world_size": 16 - } - }, - { - "type": "reduce_scatter", - "args": { - "num_elements": 18085838848, - "dtype": "bf16", - "world_size": 16 - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 157696, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 16, - "mode": "normal" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 157696, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 16, - "mode": "normal" - } - }, - { - "type": "allgather", - "args": { - "num_elements_per_rank": 377487360, - "dtype": "bf16", - "world_size": 2 - } - }, - { - "type": "reduce_scatter", - "args": { - "num_elements": 754974720, - "dtype": "bf16", - "world_size": 2 - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 131072, - "num_experts": 128, - "top_k": 4, - "hidden": 2880, - "dtype": "bf16", - "world_size": 2, - "mode": "normal" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 131072, - "num_experts": 128, - "top_k": 4, - "hidden": 2880, - "dtype": "bf16", - "world_size": 2, - "mode": "normal" - } - }, - { - "type": "allgather", - "args": { - "num_elements_per_rank": 368640, - "dtype": "bf16", - "world_size": 2 - } - }, - { - "type": "reduce_scatter", - "args": { - "num_elements": 737280, - "dtype": "bf16", - "world_size": 2 - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 128, - "num_experts": 128, - "top_k": 4, - "hidden": 2880, - "dtype": "bf16", - "world_size": 2, - "mode": "low_latency" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 128, - "num_experts": 128, - "top_k": 4, - "hidden": 2880, - "dtype": "bf16", - "world_size": 2, - "mode": "low_latency" - } - }, - { - "type": "allreduce", - "args": { - "num_elements": 11796480, - "dtype": "bf16", - "world_size": 2 - } - }, - { - "type": "allreduce", - "args": { - "num_elements": 11520, - "dtype": "bf16", - "world_size": 2 - } - }, - { - "type": "allreduce", - "args": { - "num_elements": 23592960, - "dtype": "bf16", - "world_size": 2 - } - }, - { - "type": "allreduce", - "args": { - "num_elements": 23040, - "dtype": "bf16", - "world_size": 2 - } - }, - { - "type": "allreduce", - "args": { - "num_elements": 47185920, - "dtype": "bf16", - "world_size": 2 - } - }, - { - "type": "allreduce", - "args": { - "num_elements": 46080, - "dtype": "bf16", - "world_size": 2 - } - }, - { - "type": "allreduce", - "args": { - "num_elements": 94371840, - "dtype": "bf16", - "world_size": 2 - } - }, - { - "type": "allreduce", - "args": { - "num_elements": 92160, - "dtype": "bf16", - "world_size": 2 - } - }, - { - "type": "allreduce", - "args": { - "num_elements": 188743680, - "dtype": "bf16", - "world_size": 2 - } - }, - { - "type": "allreduce", - "args": { - "num_elements": 184320, - "dtype": "bf16", - "world_size": 2 - } - }, - { - "type": "allreduce", - "args": { - "num_elements": 377487360, - "dtype": "bf16", - "world_size": 2 - } - }, - { - "type": "allreduce", - "args": { - "num_elements": 368640, - "dtype": "bf16", - "world_size": 2 - } - }, - { - "type": "allreduce", - "args": { - "num_elements": 754974720, - "dtype": "bf16", - "world_size": 2 - } - }, - { - "type": "allreduce", - "args": { - "num_elements": 737280, - "dtype": "bf16", - "world_size": 2 - } - }, - { - "type": "allgather", - "args": { - "num_elements_per_rank": 47185920, - "dtype": "bf16", - "world_size": 4 - } - }, - { - "type": "reduce_scatter", - "args": { - "num_elements": 188743680, - "dtype": "bf16", - "world_size": 4 - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 16384, - "num_experts": 128, - "top_k": 4, - "hidden": 2880, - "dtype": "bf16", - "world_size": 4, - "mode": "normal" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 16384, - "num_experts": 128, - "top_k": 4, - "hidden": 2880, - "dtype": "bf16", - "world_size": 4, - "mode": "normal" - } - }, - { - "type": "allgather", - "args": { - "num_elements_per_rank": 46080, - "dtype": "bf16", - "world_size": 4 - } - }, - { - "type": "reduce_scatter", - "args": { - "num_elements": 184320, - "dtype": "bf16", - "world_size": 4 - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 16, - "num_experts": 128, - "top_k": 4, - "hidden": 2880, - "dtype": "bf16", - "world_size": 4, - "mode": "low_latency" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 16, - "num_experts": 128, - "top_k": 4, - "hidden": 2880, - "dtype": "bf16", - "world_size": 4, - "mode": "low_latency" - } - }, - { - "type": "allreduce", - "args": { - "num_elements": 11796480, - "dtype": "bf16", - "world_size": 4 - } - }, - { - "type": "allreduce", - "args": { - "num_elements": 11520, - "dtype": "bf16", - "world_size": 4 - } - }, - { - "type": "allreduce", - "args": { - "num_elements": 11796480, - "dtype": "bf16", - "world_size": 8 - } - }, - { - "type": "allreduce", - "args": { - "num_elements": 11520, - "dtype": "bf16", - "world_size": 8 - } - }, - { - "type": "allreduce", - "args": { - "num_elements": 1509949440, - "dtype": "bf16", - "world_size": 2 - } - }, - { - "type": "allreduce", - "args": { - "num_elements": 3019898880, - "dtype": "bf16", - "world_size": 2 - } - }, - { - "type": "allreduce", - "args": { - "num_elements": 6039797760, - "dtype": "bf16", - "world_size": 2 - } - }, - { - "type": "allreduce", - "args": { - "num_elements": 94371840, - "dtype": "bf16", - "world_size": 4 - } - }, - { - "type": "allreduce", - "args": { - "num_elements": 188743680, - "dtype": "bf16", - "world_size": 4 - } - }, - { - "type": "allreduce", - "args": { - "num_elements": 23040, - "dtype": "bf16", - "world_size": 4 - } - }, - { - "type": "allreduce", - "args": { - "num_elements": 377487360, - "dtype": "bf16", - "world_size": 4 - } - }, - { - "type": "allreduce", - "args": { - "num_elements": 46080, - "dtype": "bf16", - "world_size": 4 - } - }, - { - "type": "allreduce", - "args": { - "num_elements": 754974720, - "dtype": "bf16", - "world_size": 4 - } - }, - { - "type": "allreduce", - "args": { - "num_elements": 92160, - "dtype": "bf16", - "world_size": 4 - } - }, - { - "type": "allreduce", - "args": { - "num_elements": 94371840, - "dtype": "bf16", - "world_size": 8 - } - }, - { - "type": "allreduce", - "args": { - "num_elements": 23592960, - "dtype": "bf16", - "world_size": 4 - } - }, - { - "type": "allreduce", - "args": { - "num_elements": 47185920, - "dtype": "bf16", - "world_size": 4 - } - }, - { - "type": "allreduce", - "args": { - "num_elements": 184320, - "dtype": "bf16", - "world_size": 4 - } - }, - { - "type": "allreduce", - "args": { - "num_elements": 23592960, - "dtype": "bf16", - "world_size": 8 - } - }, - { - "type": "allreduce", - "args": { - "num_elements": 23040, - "dtype": "bf16", - "world_size": 8 - } - }, - { - "type": "allreduce", - "args": { - "num_elements": 1509949440, - "dtype": "bf16", - "world_size": 4 - } - }, - { - "type": "allreduce", - "args": { - "num_elements": 1610612736, - "dtype": "bf16", - "world_size": 2 - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 524288, - "num_experts": 256, - "top_k": 8, - "hidden": 3072, - "dtype": "bf16", - "world_size": 2, - "mode": "normal" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 524288, - "num_experts": 256, - "top_k": 8, - "hidden": 3072, - "dtype": "bf16", - "world_size": 2, - "mode": "normal" - } - }, - { - "type": "allreduce", - "args": { - "num_elements": 1572864, - "dtype": "bf16", - "world_size": 2 - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 512, - "num_experts": 256, - "top_k": 8, - "hidden": 3072, - "dtype": "bf16", - "world_size": 2, - "mode": "low_latency" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 512, - "num_experts": 256, - "top_k": 8, - "hidden": 3072, - "dtype": "bf16", - "world_size": 2, - "mode": "low_latency" - } - }, - { - "type": "allreduce", - "args": { - "num_elements": 12582912, - "dtype": "bf16", - "world_size": 4 - } - }, - { - "type": "allreduce", - "args": { - "num_elements": 12288, - "dtype": "bf16", - "world_size": 4 - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 262144, - "num_experts": 256, - "top_k": 8, - "hidden": 3072, - "dtype": "bf16", - "world_size": 4, - "mode": "normal" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 262144, - "num_experts": 256, - "top_k": 8, - "hidden": 3072, - "dtype": "bf16", - "world_size": 4, - "mode": "normal" - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 256, - "num_experts": 256, - "top_k": 8, - "hidden": 3072, - "dtype": "bf16", - "world_size": 4, - "mode": "low_latency" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 256, - "num_experts": 256, - "top_k": 8, - "hidden": 3072, - "dtype": "bf16", - "world_size": 4, - "mode": "low_latency" - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 524288, - "num_experts": 256, - "top_k": 8, - "hidden": 3072, - "dtype": "bf16", - "world_size": 4, - "mode": "normal" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 524288, - "num_experts": 256, - "top_k": 8, - "hidden": 3072, - "dtype": "bf16", - "world_size": 4, - "mode": "normal" - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 512, - "num_experts": 256, - "top_k": 8, - "hidden": 3072, - "dtype": "bf16", - "world_size": 4, - "mode": "low_latency" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 512, - "num_experts": 256, - "top_k": 8, - "hidden": 3072, - "dtype": "bf16", - "world_size": 4, - "mode": "low_latency" - } - }, - { - "type": "allreduce", - "args": { - "num_elements": 100663296, - "dtype": "bf16", - "world_size": 2 - } - }, - { - "type": "allreduce", - "args": { - "num_elements": 12288, - "dtype": "bf16", - "world_size": 2 - } - }, - { - "type": "allreduce", - "args": { - "num_elements": 201326592, - "dtype": "bf16", - "world_size": 2 - } - }, - { - "type": "allreduce", - "args": { - "num_elements": 24576, - "dtype": "bf16", - "world_size": 2 - } - }, - { - "type": "allreduce", - "args": { - "num_elements": 402653184, - "dtype": "bf16", - "world_size": 2 - } - }, - { - "type": "allreduce", - "args": { - "num_elements": 49152, - "dtype": "bf16", - "world_size": 2 - } - }, - { - "type": "allreduce", - "args": { - "num_elements": 805306368, - "dtype": "bf16", - "world_size": 2 - } - }, - { - "type": "allreduce", - "args": { - "num_elements": 98304, - "dtype": "bf16", - "world_size": 2 - } - }, - { - "type": "allreduce", - "args": { - "num_elements": 196608, - "dtype": "bf16", - "world_size": 2 - } - }, - { - "type": "allreduce", - "args": { - "num_elements": 3221225472, - "dtype": "bf16", - "world_size": 2 - } - }, - { - "type": "allreduce", - "args": { - "num_elements": 393216, - "dtype": "bf16", - "world_size": 2 - } - }, - { - "type": "allreduce", - "args": { - "num_elements": 6442450944, - "dtype": "bf16", - "world_size": 2 - } - }, - { - "type": "allreduce", - "args": { - "num_elements": 786432, - "dtype": "bf16", - "world_size": 2 - } - }, - { - "type": "allreduce", - "args": { - "num_elements": 12884901888, - "dtype": "bf16", - "world_size": 2 - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 1048576, - "num_experts": 256, - "top_k": 8, - "hidden": 3072, - "dtype": "bf16", - "world_size": 2, - "mode": "normal" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 1048576, - "num_experts": 256, - "top_k": 8, - "hidden": 3072, - "dtype": "bf16", - "world_size": 2, - "mode": "normal" - } - }, - { - "type": "allreduce", - "args": { - "num_elements": 3145728, - "dtype": "bf16", - "world_size": 2 - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 1024, - "num_experts": 256, - "top_k": 8, - "hidden": 3072, - "dtype": "bf16", - "world_size": 2, - "mode": "low_latency" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 1024, - "num_experts": 256, - "top_k": 8, - "hidden": 3072, - "dtype": "bf16", - "world_size": 2, - "mode": "low_latency" - } - }, - { - "type": "allgather", - "args": { - "num_elements_per_rank": 1610612736, - "dtype": "bf16", - "world_size": 2 - } - }, - { - "type": "reduce_scatter", - "args": { - "num_elements": 3221225472, - "dtype": "bf16", - "world_size": 2 - } - }, - { - "type": "allgather", - "args": { - "num_elements_per_rank": 1572864, - "dtype": "bf16", - "world_size": 2 - } - }, - { - "type": "reduce_scatter", - "args": { - "num_elements": 3145728, - "dtype": "bf16", - "world_size": 2 - } - }, - { - "type": "allreduce", - "args": { - "num_elements": 50331648, - "dtype": "bf16", - "world_size": 2 - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 131072, - "num_experts": 256, - "top_k": 8, - "hidden": 3072, - "dtype": "bf16", - "world_size": 2, - "mode": "normal" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 131072, - "num_experts": 256, - "top_k": 8, - "hidden": 3072, - "dtype": "bf16", - "world_size": 2, - "mode": "normal" - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 128, - "num_experts": 256, - "top_k": 8, - "hidden": 3072, - "dtype": "bf16", - "world_size": 2, - "mode": "low_latency" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 128, - "num_experts": 256, - "top_k": 8, - "hidden": 3072, - "dtype": "bf16", - "world_size": 2, - "mode": "low_latency" - } - }, - { - "type": "allgather", - "args": { - "num_elements_per_rank": 402653184, - "dtype": "bf16", - "world_size": 2 - } - }, - { - "type": "reduce_scatter", - "args": { - "num_elements": 805306368, - "dtype": "bf16", - "world_size": 2 - } - }, - { - "type": "allgather", - "args": { - "num_elements_per_rank": 393216, - "dtype": "bf16", - "world_size": 2 - } - }, - { - "type": "reduce_scatter", - "args": { - "num_elements": 786432, - "dtype": "bf16", - "world_size": 2 - } - }, - { - "type": "allgather", - "args": { - "num_elements_per_rank": 805306368, - "dtype": "bf16", - "world_size": 2 - } - }, - { - "type": "reduce_scatter", - "args": { - "num_elements": 1610612736, - "dtype": "bf16", - "world_size": 2 - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 262144, - "num_experts": 256, - "top_k": 8, - "hidden": 3072, - "dtype": "bf16", - "world_size": 2, - "mode": "normal" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 262144, - "num_experts": 256, - "top_k": 8, - "hidden": 3072, - "dtype": "bf16", - "world_size": 2, - "mode": "normal" - } - }, - { - "type": "allgather", - "args": { - "num_elements_per_rank": 786432, - "dtype": "bf16", - "world_size": 2 - } - }, - { - "type": "reduce_scatter", - "args": { - "num_elements": 1572864, - "dtype": "bf16", - "world_size": 2 - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 256, - "num_experts": 256, - "top_k": 8, - "hidden": 3072, - "dtype": "bf16", - "world_size": 2, - "mode": "low_latency" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 256, - "num_experts": 256, - "top_k": 8, - "hidden": 3072, - "dtype": "bf16", - "world_size": 2, - "mode": "low_latency" - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 65536, - "num_experts": 256, - "top_k": 8, - "hidden": 3072, - "dtype": "bf16", - "world_size": 4, - "mode": "normal" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 65536, - "num_experts": 256, - "top_k": 8, - "hidden": 3072, - "dtype": "bf16", - "world_size": 4, - "mode": "normal" - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 64, - "num_experts": 256, - "top_k": 8, - "hidden": 3072, - "dtype": "bf16", - "world_size": 4, - "mode": "low_latency" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 64, - "num_experts": 256, - "top_k": 8, - "hidden": 3072, - "dtype": "bf16", - "world_size": 4, - "mode": "low_latency" - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 131072, - "num_experts": 256, - "top_k": 8, - "hidden": 3072, - "dtype": "bf16", - "world_size": 4, - "mode": "normal" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 131072, - "num_experts": 256, - "top_k": 8, - "hidden": 3072, - "dtype": "bf16", - "world_size": 4, - "mode": "normal" - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 128, - "num_experts": 256, - "top_k": 8, - "hidden": 3072, - "dtype": "bf16", - "world_size": 4, - "mode": "low_latency" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 128, - "num_experts": 256, - "top_k": 8, - "hidden": 3072, - "dtype": "bf16", - "world_size": 4, - "mode": "low_latency" - } - }, - { - "type": "allreduce", - "args": { - "num_elements": 12582912, - "dtype": "bf16", - "world_size": 8 - } - }, - { - "type": "allreduce", - "args": { - "num_elements": 12288, - "dtype": "bf16", - "world_size": 8 - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 2097152, - "num_experts": 256, - "top_k": 8, - "hidden": 3072, - "dtype": "bf16", - "world_size": 2, - "mode": "normal" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 2097152, - "num_experts": 256, - "top_k": 8, - "hidden": 3072, - "dtype": "bf16", - "world_size": 2, - "mode": "normal" - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 4194304, - "num_experts": 256, - "top_k": 8, - "hidden": 3072, - "dtype": "bf16", - "world_size": 2, - "mode": "normal" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 4194304, - "num_experts": 256, - "top_k": 8, - "hidden": 3072, - "dtype": "bf16", - "world_size": 2, - "mode": "normal" - } - }, - { - "type": "allgather", - "args": { - "num_elements_per_rank": 3221225472, - "dtype": "bf16", - "world_size": 2 - } - }, - { - "type": "reduce_scatter", - "args": { - "num_elements": 6442450944, - "dtype": "bf16", - "world_size": 2 - } - }, - { - "type": "allgather", - "args": { - "num_elements_per_rank": 3145728, - "dtype": "bf16", - "world_size": 2 - } - }, - { - "type": "reduce_scatter", - "args": { - "num_elements": 6291456, - "dtype": "bf16", - "world_size": 2 - } - }, - { - "type": "allgather", - "args": { - "num_elements_per_rank": 6442450944, - "dtype": "bf16", - "world_size": 2 - } - }, - { - "type": "reduce_scatter", - "args": { - "num_elements": 12884901888, - "dtype": "bf16", - "world_size": 2 - } - }, - { - "type": "allreduce", - "args": { - "num_elements": 47185920, - "dtype": "bf16", - "world_size": 8 - } - }, - { - "type": "allreduce", - "args": { - "num_elements": 46080, - "dtype": "bf16", - "world_size": 8 - } - }, - { - "type": "allreduce", - "args": { - "num_elements": 92160, - "dtype": "bf16", - "world_size": 8 - } - }, - { - "type": "allreduce", - "args": { - "num_elements": 188743680, - "dtype": "bf16", - "world_size": 8 - } - }, - { - "type": "allreduce", - "args": { - "num_elements": 184320, - "dtype": "bf16", - "world_size": 8 - } - }, - { - "type": "allreduce", - "args": { - "num_elements": 377487360, - "dtype": "bf16", - "world_size": 8 - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 4096, - "num_experts": 256, - "top_k": 8, - "hidden": 3072, - "dtype": "bf16", - "world_size": 4, - "mode": "normal" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 4096, - "num_experts": 256, - "top_k": 8, - "hidden": 3072, - "dtype": "bf16", - "world_size": 4, - "mode": "normal" - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 4, - "num_experts": 256, - "top_k": 8, - "hidden": 3072, - "dtype": "bf16", - "world_size": 4, - "mode": "low_latency" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 4, - "num_experts": 256, - "top_k": 8, - "hidden": 3072, - "dtype": "bf16", - "world_size": 4, - "mode": "low_latency" - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 8192, - "num_experts": 256, - "top_k": 8, - "hidden": 3072, - "dtype": "bf16", - "world_size": 4, - "mode": "normal" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 8192, - "num_experts": 256, - "top_k": 8, - "hidden": 3072, - "dtype": "bf16", - "world_size": 4, - "mode": "normal" - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 8, - "num_experts": 256, - "top_k": 8, - "hidden": 3072, - "dtype": "bf16", - "world_size": 4, - "mode": "low_latency" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 8, - "num_experts": 256, - "top_k": 8, - "hidden": 3072, - "dtype": "bf16", - "world_size": 4, - "mode": "low_latency" - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 16384, - "num_experts": 256, - "top_k": 8, - "hidden": 3072, - "dtype": "bf16", - "world_size": 4, - "mode": "normal" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 16384, - "num_experts": 256, - "top_k": 8, - "hidden": 3072, - "dtype": "bf16", - "world_size": 4, - "mode": "normal" - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 16, - "num_experts": 256, - "top_k": 8, - "hidden": 3072, - "dtype": "bf16", - "world_size": 4, - "mode": "low_latency" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 16, - "num_experts": 256, - "top_k": 8, - "hidden": 3072, - "dtype": "bf16", - "world_size": 4, - "mode": "low_latency" - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 32768, - "num_experts": 256, - "top_k": 8, - "hidden": 3072, - "dtype": "bf16", - "world_size": 4, - "mode": "normal" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 32768, - "num_experts": 256, - "top_k": 8, - "hidden": 3072, - "dtype": "bf16", - "world_size": 4, - "mode": "normal" - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 32, - "num_experts": 256, - "top_k": 8, - "hidden": 3072, - "dtype": "bf16", - "world_size": 4, - "mode": "low_latency" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 32, - "num_experts": 256, - "top_k": 8, - "hidden": 3072, - "dtype": "bf16", - "world_size": 4, - "mode": "low_latency" - } - }, - { - "type": "allreduce", - "args": { - "num_elements": 7340032, - "dtype": "bf16", - "world_size": 16 - } - }, - { - "type": "allreduce", - "args": { - "num_elements": 14336, - "dtype": "bf16", - "world_size": 16 - } - }, - { - "type": "allreduce", - "args": { - "num_elements": 14680064, - "dtype": "bf16", - "world_size": 16 - } - }, - { - "type": "allreduce", - "args": { - "num_elements": 28672, - "dtype": "bf16", - "world_size": 16 - } - }, - { - "type": "allreduce", - "args": { - "num_elements": 29360128, - "dtype": "bf16", - "world_size": 16 - } - }, - { - "type": "allreduce", - "args": { - "num_elements": 57344, - "dtype": "bf16", - "world_size": 16 - } - }, - { - "type": "allreduce", - "args": { - "num_elements": 58720256, - "dtype": "bf16", - "world_size": 16 - } - }, - { - "type": "allreduce", - "args": { - "num_elements": 114688, - "dtype": "bf16", - "world_size": 16 - } - }, - { - "type": "allreduce", - "args": { - "num_elements": 117440512, - "dtype": "bf16", - "world_size": 16 - } - }, - { - "type": "allreduce", - "args": { - "num_elements": 229376, - "dtype": "bf16", - "world_size": 16 - } - }, - { - "type": "allreduce", - "args": { - "num_elements": 234881024, - "dtype": "bf16", - "world_size": 16 - } - }, - { - "type": "allreduce", - "args": { - "num_elements": 458752, - "dtype": "bf16", - "world_size": 16 - } - }, - { - "type": "allreduce", - "args": { - "num_elements": 469762048, - "dtype": "bf16", - "world_size": 16 - } - }, - { - "type": "allreduce", - "args": { - "num_elements": 917504, - "dtype": "bf16", - "world_size": 16 - } - }, - { - "type": "allreduce", - "args": { - "num_elements": 939524096, - "dtype": "bf16", - "world_size": 16 - } - }, - { - "type": "allreduce", - "args": { - "num_elements": 1835008, - "dtype": "bf16", - "world_size": 16 - } - }, - { - "type": "allgather", - "args": { - "num_elements_per_rank": 458752, - "dtype": "bf16", - "world_size": 16 - } - }, - { - "type": "reduce_scatter", - "args": { - "num_elements": 7340032, - "dtype": "bf16", - "world_size": 16 - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 64, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 16, - "mode": "normal" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 64, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 16, - "mode": "normal" - } - }, - { - "type": "allgather", - "args": { - "num_elements_per_rank": 917504, - "dtype": "bf16", - "world_size": 16 - } - }, - { - "type": "reduce_scatter", - "args": { - "num_elements": 14680064, - "dtype": "bf16", - "world_size": 16 - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 128, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 16, - "mode": "normal" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 128, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 16, - "mode": "normal" - } - }, - { - "type": "allgather", - "args": { - "num_elements_per_rank": 1835008, - "dtype": "bf16", - "world_size": 16 - } - }, - { - "type": "reduce_scatter", - "args": { - "num_elements": 29360128, - "dtype": "bf16", - "world_size": 16 - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 256, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 16, - "mode": "normal" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 256, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 16, - "mode": "normal" - } - }, - { - "type": "allgather", - "args": { - "num_elements_per_rank": 3670016, - "dtype": "bf16", - "world_size": 16 - } - }, - { - "type": "reduce_scatter", - "args": { - "num_elements": 58720256, - "dtype": "bf16", - "world_size": 16 - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 512, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 16, - "mode": "normal" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 512, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 16, - "mode": "normal" - } - }, - { - "type": "allgather", - "args": { - "num_elements_per_rank": 7340032, - "dtype": "bf16", - "world_size": 16 - } - }, - { - "type": "reduce_scatter", - "args": { - "num_elements": 117440512, - "dtype": "bf16", - "world_size": 16 - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 1024, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 16, - "mode": "normal" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 1024, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 16, - "mode": "normal" - } - }, - { - "type": "allgather", - "args": { - "num_elements_per_rank": 14336, - "dtype": "bf16", - "world_size": 16 - } - }, - { - "type": "reduce_scatter", - "args": { - "num_elements": 229376, - "dtype": "bf16", - "world_size": 16 - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 2, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 16, - "mode": "low_latency" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 2, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 16, - "mode": "low_latency" - } - }, - { - "type": "allgather", - "args": { - "num_elements_per_rank": 14680064, - "dtype": "bf16", - "world_size": 16 - } - }, - { - "type": "reduce_scatter", - "args": { - "num_elements": 234881024, - "dtype": "bf16", - "world_size": 16 - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 2048, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 16, - "mode": "normal" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 2048, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 16, - "mode": "normal" - } - }, - { - "type": "allgather", - "args": { - "num_elements_per_rank": 28672, - "dtype": "bf16", - "world_size": 16 - } - }, - { - "type": "reduce_scatter", - "args": { - "num_elements": 458752, - "dtype": "bf16", - "world_size": 16 - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 4, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 16, - "mode": "low_latency" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 4, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 16, - "mode": "low_latency" - } - }, - { - "type": "allgather", - "args": { - "num_elements_per_rank": 29360128, - "dtype": "bf16", - "world_size": 16 - } - }, - { - "type": "reduce_scatter", - "args": { - "num_elements": 469762048, - "dtype": "bf16", - "world_size": 16 - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 4096, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 16, - "mode": "normal" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 4096, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 16, - "mode": "normal" - } - }, - { - "type": "allgather", - "args": { - "num_elements_per_rank": 57344, - "dtype": "bf16", - "world_size": 16 - } - }, - { - "type": "reduce_scatter", - "args": { - "num_elements": 917504, - "dtype": "bf16", - "world_size": 16 - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 8, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 16, - "mode": "low_latency" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 8, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 16, - "mode": "low_latency" - } - }, - { - "type": "allreduce", - "args": { - "num_elements": 1879048192, - "dtype": "bf16", - "world_size": 16 - } - }, - { - "type": "allreduce", - "args": { - "num_elements": 3758096384, - "dtype": "bf16", - "world_size": 16 - } - }, - { - "type": "allreduce", - "args": { - "num_elements": 7516192768, - "dtype": "bf16", - "world_size": 16 - } - }, - { - "type": "allgather", - "args": { - "num_elements_per_rank": 58720256, - "dtype": "bf16", - "world_size": 16 - } - }, - { - "type": "reduce_scatter", - "args": { - "num_elements": 939524096, - "dtype": "bf16", - "world_size": 16 - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 8192, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 16, - "mode": "normal" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 8192, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 16, - "mode": "normal" - } - }, - { - "type": "allgather", - "args": { - "num_elements_per_rank": 117440512, - "dtype": "bf16", - "world_size": 16 - } - }, - { - "type": "reduce_scatter", - "args": { - "num_elements": 1879048192, - "dtype": "bf16", - "world_size": 16 - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 16384, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 16, - "mode": "normal" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 16384, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 16, - "mode": "normal" - } - }, - { - "type": "allgather", - "args": { - "num_elements_per_rank": 234881024, - "dtype": "bf16", - "world_size": 16 - } - }, - { - "type": "reduce_scatter", - "args": { - "num_elements": 3758096384, - "dtype": "bf16", - "world_size": 16 - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 32768, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 16, - "mode": "normal" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 32768, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 16, - "mode": "normal" - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 4096, - "num_experts": 128, - "top_k": 4, - "hidden": 2880, - "dtype": "bf16", - "world_size": 2, - "mode": "normal" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 4096, - "num_experts": 128, - "top_k": 4, - "hidden": 2880, - "dtype": "bf16", - "world_size": 2, - "mode": "normal" - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 4, - "num_experts": 128, - "top_k": 4, - "hidden": 2880, - "dtype": "bf16", - "world_size": 2, - "mode": "low_latency" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 4, - "num_experts": 128, - "top_k": 4, - "hidden": 2880, - "dtype": "bf16", - "world_size": 2, - "mode": "low_latency" - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 8192, - "num_experts": 128, - "top_k": 4, - "hidden": 2880, - "dtype": "bf16", - "world_size": 2, - "mode": "normal" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 8192, - "num_experts": 128, - "top_k": 4, - "hidden": 2880, - "dtype": "bf16", - "world_size": 2, - "mode": "normal" - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 8, - "num_experts": 128, - "top_k": 4, - "hidden": 2880, - "dtype": "bf16", - "world_size": 2, - "mode": "low_latency" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 8, - "num_experts": 128, - "top_k": 4, - "hidden": 2880, - "dtype": "bf16", - "world_size": 2, - "mode": "low_latency" - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 16384, - "num_experts": 128, - "top_k": 4, - "hidden": 2880, - "dtype": "bf16", - "world_size": 2, - "mode": "normal" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 16384, - "num_experts": 128, - "top_k": 4, - "hidden": 2880, - "dtype": "bf16", - "world_size": 2, - "mode": "normal" - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 16, - "num_experts": 128, - "top_k": 4, - "hidden": 2880, - "dtype": "bf16", - "world_size": 2, - "mode": "low_latency" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 16, - "num_experts": 128, - "top_k": 4, - "hidden": 2880, - "dtype": "bf16", - "world_size": 2, - "mode": "low_latency" - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 32768, - "num_experts": 128, - "top_k": 4, - "hidden": 2880, - "dtype": "bf16", - "world_size": 2, - "mode": "normal" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 32768, - "num_experts": 128, - "top_k": 4, - "hidden": 2880, - "dtype": "bf16", - "world_size": 2, - "mode": "normal" - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 32, - "num_experts": 128, - "top_k": 4, - "hidden": 2880, - "dtype": "bf16", - "world_size": 2, - "mode": "low_latency" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 32, - "num_experts": 128, - "top_k": 4, - "hidden": 2880, - "dtype": "bf16", - "world_size": 2, - "mode": "low_latency" - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 65536, - "num_experts": 128, - "top_k": 4, - "hidden": 2880, - "dtype": "bf16", - "world_size": 2, - "mode": "normal" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 65536, - "num_experts": 128, - "top_k": 4, - "hidden": 2880, - "dtype": "bf16", - "world_size": 2, - "mode": "normal" - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 64, - "num_experts": 128, - "top_k": 4, - "hidden": 2880, - "dtype": "bf16", - "world_size": 2, - "mode": "low_latency" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 64, - "num_experts": 128, - "top_k": 4, - "hidden": 2880, - "dtype": "bf16", - "world_size": 2, - "mode": "low_latency" - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 4096, - "num_experts": 128, - "top_k": 4, - "hidden": 2880, - "dtype": "bf16", - "world_size": 4, - "mode": "normal" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 4096, - "num_experts": 128, - "top_k": 4, - "hidden": 2880, - "dtype": "bf16", - "world_size": 4, - "mode": "normal" - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 4, - "num_experts": 128, - "top_k": 4, - "hidden": 2880, - "dtype": "bf16", - "world_size": 4, - "mode": "low_latency" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 4, - "num_experts": 128, - "top_k": 4, - "hidden": 2880, - "dtype": "bf16", - "world_size": 4, - "mode": "low_latency" - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 8192, - "num_experts": 128, - "top_k": 4, - "hidden": 2880, - "dtype": "bf16", - "world_size": 4, - "mode": "normal" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 8192, - "num_experts": 128, - "top_k": 4, - "hidden": 2880, - "dtype": "bf16", - "world_size": 4, - "mode": "normal" - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 8, - "num_experts": 128, - "top_k": 4, - "hidden": 2880, - "dtype": "bf16", - "world_size": 4, - "mode": "low_latency" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 8, - "num_experts": 128, - "top_k": 4, - "hidden": 2880, - "dtype": "bf16", - "world_size": 4, - "mode": "low_latency" - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 32768, - "num_experts": 128, - "top_k": 4, - "hidden": 2880, - "dtype": "bf16", - "world_size": 4, - "mode": "normal" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 32768, - "num_experts": 128, - "top_k": 4, - "hidden": 2880, - "dtype": "bf16", - "world_size": 4, - "mode": "normal" - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 32, - "num_experts": 128, - "top_k": 4, - "hidden": 2880, - "dtype": "bf16", - "world_size": 4, - "mode": "low_latency" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 32, - "num_experts": 128, - "top_k": 4, - "hidden": 2880, - "dtype": "bf16", - "world_size": 4, - "mode": "low_latency" - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 4096, - "num_experts": 128, - "top_k": 4, - "hidden": 2880, - "dtype": "bf16", - "world_size": 8, - "mode": "normal" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 4096, - "num_experts": 128, - "top_k": 4, - "hidden": 2880, - "dtype": "bf16", - "world_size": 8, - "mode": "normal" - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 4, - "num_experts": 128, - "top_k": 4, - "hidden": 2880, - "dtype": "bf16", - "world_size": 8, - "mode": "low_latency" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 4, - "num_experts": 128, - "top_k": 4, - "hidden": 2880, - "dtype": "bf16", - "world_size": 8, - "mode": "low_latency" - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 8192, - "num_experts": 128, - "top_k": 4, - "hidden": 2880, - "dtype": "bf16", - "world_size": 8, - "mode": "normal" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 8192, - "num_experts": 128, - "top_k": 4, - "hidden": 2880, - "dtype": "bf16", - "world_size": 8, - "mode": "normal" - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 8, - "num_experts": 128, - "top_k": 4, - "hidden": 2880, - "dtype": "bf16", - "world_size": 8, - "mode": "low_latency" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 8, - "num_experts": 128, - "top_k": 4, - "hidden": 2880, - "dtype": "bf16", - "world_size": 8, - "mode": "low_latency" - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 262144, - "num_experts": 128, - "top_k": 4, - "hidden": 2880, - "dtype": "bf16", - "world_size": 2, - "mode": "normal" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 262144, - "num_experts": 128, - "top_k": 4, - "hidden": 2880, - "dtype": "bf16", - "world_size": 2, - "mode": "normal" - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 524288, - "num_experts": 128, - "top_k": 4, - "hidden": 2880, - "dtype": "bf16", - "world_size": 2, - "mode": "normal" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 524288, - "num_experts": 128, - "top_k": 4, - "hidden": 2880, - "dtype": "bf16", - "world_size": 2, - "mode": "normal" - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 65536, - "num_experts": 128, - "top_k": 4, - "hidden": 2880, - "dtype": "bf16", - "world_size": 4, - "mode": "normal" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 65536, - "num_experts": 128, - "top_k": 4, - "hidden": 2880, - "dtype": "bf16", - "world_size": 4, - "mode": "normal" - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 131072, - "num_experts": 128, - "top_k": 4, - "hidden": 2880, - "dtype": "bf16", - "world_size": 4, - "mode": "normal" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 131072, - "num_experts": 128, - "top_k": 4, - "hidden": 2880, - "dtype": "bf16", - "world_size": 4, - "mode": "normal" - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 262144, - "num_experts": 128, - "top_k": 4, - "hidden": 2880, - "dtype": "bf16", - "world_size": 4, - "mode": "normal" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 262144, - "num_experts": 128, - "top_k": 4, - "hidden": 2880, - "dtype": "bf16", - "world_size": 4, - "mode": "normal" - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 524288, - "num_experts": 128, - "top_k": 4, - "hidden": 2880, - "dtype": "bf16", - "world_size": 4, - "mode": "normal" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 524288, - "num_experts": 128, - "top_k": 4, - "hidden": 2880, - "dtype": "bf16", - "world_size": 4, - "mode": "normal" - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 64, - "num_experts": 128, - "top_k": 4, - "hidden": 2880, - "dtype": "bf16", - "world_size": 4, - "mode": "low_latency" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 64, - "num_experts": 128, - "top_k": 4, - "hidden": 2880, - "dtype": "bf16", - "world_size": 4, - "mode": "low_latency" - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 32768, - "num_experts": 128, - "top_k": 4, - "hidden": 2880, - "dtype": "bf16", - "world_size": 8, - "mode": "normal" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 32768, - "num_experts": 128, - "top_k": 4, - "hidden": 2880, - "dtype": "bf16", - "world_size": 8, - "mode": "normal" - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 65536, - "num_experts": 128, - "top_k": 4, - "hidden": 2880, - "dtype": "bf16", - "world_size": 8, - "mode": "normal" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 65536, - "num_experts": 128, - "top_k": 4, - "hidden": 2880, - "dtype": "bf16", - "world_size": 8, - "mode": "normal" - } - }, - { - "type": "allreduce", - "args": { - "num_elements": 754974720, - "dtype": "bf16", - "world_size": 8 - } - }, - { - "type": "allgather", - "args": { - "num_elements_per_rank": 41287680, - "dtype": "bf16", - "world_size": 32 - } - }, - { - "type": "reduce_scatter", - "args": { - "num_elements": 1321205760, - "dtype": "bf16", - "world_size": 32 - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 5760, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 32, - "mode": "normal" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 5760, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 32, - "mode": "normal" - } - }, - { - "type": "allgather", - "args": { - "num_elements_per_rank": 78848, - "dtype": "bf16", - "world_size": 32 - } - }, - { - "type": "reduce_scatter", - "args": { - "num_elements": 2523136, - "dtype": "bf16", - "world_size": 32 - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 11, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 32, - "mode": "low_latency" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 11, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 32, - "mode": "low_latency" - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 1024, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 4, - "mode": "normal" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 1024, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 4, - "mode": "normal" - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 3072, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 4, - "mode": "normal" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 3072, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 4, - "mode": "normal" - } - }, - { - "type": "allgather", - "args": { - "num_elements_per_rank": 44040192, - "dtype": "bf16", - "world_size": 4 - } - }, - { - "type": "reduce_scatter", - "args": { - "num_elements": 176160768, - "dtype": "bf16", - "world_size": 4 - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 6144, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 4, - "mode": "normal" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 6144, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 4, - "mode": "normal" - } - }, - { - "type": "allgather", - "args": { - "num_elements_per_rank": 172032, - "dtype": "bf16", - "world_size": 4 - } - }, - { - "type": "reduce_scatter", - "args": { - "num_elements": 688128, - "dtype": "bf16", - "world_size": 4 - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 12288, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 8, - "mode": "normal" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 12288, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 8, - "mode": "normal" - } - }, - { - "type": "allreduce", - "args": { - "num_elements": 172032, - "dtype": "bf16", - "world_size": 8 - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 24, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 8, - "mode": "low_latency" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 24, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 8, - "mode": "low_latency" - } - }, - { - "type": "allreduce", - "args": { - "num_elements": 176160768, - "dtype": "bf16", - "world_size": 8 - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 24576, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 8, - "mode": "normal" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 24576, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 8, - "mode": "normal" - } - }, - { - "type": "allreduce", - "args": { - "num_elements": 344064, - "dtype": "bf16", - "world_size": 8 - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 48, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 8, - "mode": "low_latency" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 48, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 8, - "mode": "low_latency" - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 96, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 8, - "mode": "low_latency" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 96, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 8, - "mode": "low_latency" - } - }, - { - "type": "allgather", - "args": { - "num_elements_per_rank": 7892369408, - "dtype": "bf16", - "world_size": 4 - } - }, - { - "type": "reduce_scatter", - "args": { - "num_elements": 31569477632, - "dtype": "bf16", - "world_size": 4 - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 1101056, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 4, - "mode": "normal" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 1101056, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 4, - "mode": "normal" - } - }, - { - "type": "allgather", - "args": { - "num_elements_per_rank": 15411200, - "dtype": "bf16", - "world_size": 4 - } - }, - { - "type": "reduce_scatter", - "args": { - "num_elements": 61644800, - "dtype": "bf16", - "world_size": 4 - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 2150, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 4, - "mode": "low_latency" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 2150, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 4, - "mode": "low_latency" - } - }, - { - "type": "allgather", - "args": { - "num_elements_per_rank": 1973092352, - "dtype": "bf16", - "world_size": 16 - } - }, - { - "type": "reduce_scatter", - "args": { - "num_elements": 31569477632, - "dtype": "bf16", - "world_size": 16 - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 275264, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 16, - "mode": "normal" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 275264, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 16, - "mode": "normal" - } - }, - { - "type": "allgather", - "args": { - "num_elements_per_rank": 3849216, - "dtype": "bf16", - "world_size": 16 - } - }, - { - "type": "reduce_scatter", - "args": { - "num_elements": 61587456, - "dtype": "bf16", - "world_size": 16 - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 537, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 16, - "mode": "low_latency" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 537, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 16, - "mode": "low_latency" - } - }, - { - "type": "allgather", - "args": { - "num_elements_per_rank": 4134273024, - "dtype": "bf16", - "world_size": 4 - } - }, - { - "type": "reduce_scatter", - "args": { - "num_elements": 16537092096, - "dtype": "bf16", - "world_size": 4 - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 576768, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 4, - "mode": "normal" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 576768, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 4, - "mode": "normal" - } - }, - { - "type": "allgather", - "args": { - "num_elements_per_rank": 8071168, - "dtype": "bf16", - "world_size": 4 - } - }, - { - "type": "reduce_scatter", - "args": { - "num_elements": 32284672, - "dtype": "bf16", - "world_size": 4 - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 1126, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 4, - "mode": "low_latency" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 1126, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 4, - "mode": "low_latency" - } - }, - { - "type": "allgather", - "args": { - "num_elements_per_rank": 516784128, - "dtype": "bf16", - "world_size": 32 - } - }, - { - "type": "reduce_scatter", - "args": { - "num_elements": 16537092096, - "dtype": "bf16", - "world_size": 32 - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 72096, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 32, - "mode": "normal" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 72096, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 32, - "mode": "normal" - } - }, - { - "type": "allgather", - "args": { - "num_elements_per_rank": 1003520, - "dtype": "bf16", - "world_size": 32 - } - }, - { - "type": "reduce_scatter", - "args": { - "num_elements": 32112640, - "dtype": "bf16", - "world_size": 32 - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 140, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 32, - "mode": "low_latency" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 140, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 32, - "mode": "low_latency" - } - }, - { - "type": "allgather", - "args": { - "num_elements_per_rank": 29598679040, - "dtype": "bf16", - "world_size": 4 - } - }, - { - "type": "reduce_scatter", - "args": { - "num_elements": 118394716160, - "dtype": "bf16", - "world_size": 4 - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 4129280, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 4, - "mode": "normal" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 4129280, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 4, - "mode": "normal" - } - }, - { - "type": "allgather", - "args": { - "num_elements_per_rank": 57809920, - "dtype": "bf16", - "world_size": 4 - } - }, - { - "type": "reduce_scatter", - "args": { - "num_elements": 231239680, - "dtype": "bf16", - "world_size": 4 - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 8065, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 4, - "mode": "low_latency" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 8065, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 4, - "mode": "low_latency" - } - }, - { - "type": "allgather", - "args": { - "num_elements_per_rank": 7705600, - "dtype": "bf16", - "world_size": 4 - } - }, - { - "type": "reduce_scatter", - "args": { - "num_elements": 30822400, - "dtype": "bf16", - "world_size": 4 - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 1075, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 4, - "mode": "low_latency" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 1075, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 4, - "mode": "low_latency" - } - }, - { - "type": "allgather", - "args": { - "num_elements_per_rank": 3946184704, - "dtype": "bf16", - "world_size": 8 - } - }, - { - "type": "reduce_scatter", - "args": { - "num_elements": 31569477632, - "dtype": "bf16", - "world_size": 8 - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 550528, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 8, - "mode": "normal" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 550528, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 8, - "mode": "normal" - } - }, - { - "type": "allgather", - "args": { - "num_elements_per_rank": 3849216, - "dtype": "bf16", - "world_size": 8 - } - }, - { - "type": "reduce_scatter", - "args": { - "num_elements": 30793728, - "dtype": "bf16", - "world_size": 8 - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 537, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 8, - "mode": "low_latency" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 537, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 8, - "mode": "low_latency" - } - }, - { - "type": "allgather", - "args": { - "num_elements_per_rank": 1222115328, - "dtype": "bf16", - "world_size": 4 - } - }, - { - "type": "reduce_scatter", - "args": { - "num_elements": 4888461312, - "dtype": "bf16", - "world_size": 4 - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 170496, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 4, - "mode": "normal" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 170496, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 4, - "mode": "normal" - } - }, - { - "type": "allgather", - "args": { - "num_elements_per_rank": 1189888, - "dtype": "bf16", - "world_size": 4 - } - }, - { - "type": "reduce_scatter", - "args": { - "num_elements": 4759552, - "dtype": "bf16", - "world_size": 4 - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 166, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 4, - "mode": "low_latency" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 166, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 4, - "mode": "low_latency" - } - }, - { - "type": "allgather", - "args": { - "num_elements_per_rank": 152764416, - "dtype": "bf16", - "world_size": 32 - } - }, - { - "type": "reduce_scatter", - "args": { - "num_elements": 4888461312, - "dtype": "bf16", - "world_size": 32 - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 21312, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 32, - "mode": "normal" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 21312, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 32, - "mode": "normal" - } - }, - { - "type": "allgather", - "args": { - "num_elements_per_rank": 143360, - "dtype": "bf16", - "world_size": 32 - } - }, - { - "type": "reduce_scatter", - "args": { - "num_elements": 4587520, - "dtype": "bf16", - "world_size": 32 - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 20, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 32, - "mode": "low_latency" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 20, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 32, - "mode": "low_latency" - } - }, - { - "type": "allgather", - "args": { - "num_elements_per_rank": 11274289152, - "dtype": "bf16", - "world_size": 4 - } - }, - { - "type": "reduce_scatter", - "args": { - "num_elements": 45097156608, - "dtype": "bf16", - "world_size": 4 - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 1572864, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 4, - "mode": "normal" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 1572864, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 4, - "mode": "normal" - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 1536, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 4, - "mode": "low_latency" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 1536, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 4, - "mode": "low_latency" - } - }, - { - "type": "allgather", - "args": { - "num_elements_per_rank": 176160768, - "dtype": "bf16", - "world_size": 4 - } - }, - { - "type": "reduce_scatter", - "args": { - "num_elements": 704643072, - "dtype": "bf16", - "world_size": 4 - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 24576, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 4, - "mode": "normal" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 24576, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 4, - "mode": "normal" - } - }, - { - "type": "allgather", - "args": { - "num_elements_per_rank": 352321536, - "dtype": "bf16", - "world_size": 4 - } - }, - { - "type": "reduce_scatter", - "args": { - "num_elements": 1409286144, - "dtype": "bf16", - "world_size": 4 - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 49152, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 4, - "mode": "normal" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 49152, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 4, - "mode": "normal" - } - }, - { - "type": "allgather", - "args": { - "num_elements_per_rank": 344064, - "dtype": "bf16", - "world_size": 4 - } - }, - { - "type": "reduce_scatter", - "args": { - "num_elements": 1376256, - "dtype": "bf16", - "world_size": 4 - } - }, - { - "type": "allreduce", - "args": { - "num_elements": 1409286144, - "dtype": "bf16", - "world_size": 8 - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 196608, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 8, - "mode": "normal" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 196608, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 8, - "mode": "normal" - } - }, - { - "type": "allreduce", - "args": { - "num_elements": 1376256, - "dtype": "bf16", - "world_size": 8 - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 192, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 8, - "mode": "low_latency" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 192, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 8, - "mode": "low_latency" - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 1280, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 4, - "mode": "normal" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 1280, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 4, - "mode": "normal" - } - }, - { - "type": "allreduce", - "args": { - "num_elements": 36700160, - "dtype": "bf16", - "world_size": 8 - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 5120, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 8, - "mode": "normal" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 5120, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 8, - "mode": "normal" - } - }, - { - "type": "allreduce", - "args": { - "num_elements": 35840, - "dtype": "bf16", - "world_size": 8 - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 5, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 8, - "mode": "low_latency" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 5, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 8, - "mode": "low_latency" - } - }, - { - "type": "allgather", - "args": { - "num_elements_per_rank": 1921024, - "dtype": "bf16", - "world_size": 16 - } - }, - { - "type": "reduce_scatter", - "args": { - "num_elements": 30736384, - "dtype": "bf16", - "world_size": 16 - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 268, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 16, - "mode": "low_latency" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 268, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 16, - "mode": "low_latency" - } - }, - { - "type": "allgather", - "args": { - "num_elements_per_rank": 4035584, - "dtype": "bf16", - "world_size": 4 - } - }, - { - "type": "reduce_scatter", - "args": { - "num_elements": 16142336, - "dtype": "bf16", - "world_size": 4 - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 563, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 4, - "mode": "low_latency" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 563, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 4, - "mode": "low_latency" - } - }, - { - "type": "allgather", - "args": { - "num_elements_per_rank": 501760, - "dtype": "bf16", - "world_size": 32 - } - }, - { - "type": "reduce_scatter", - "args": { - "num_elements": 16056320, - "dtype": "bf16", - "world_size": 32 - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 70, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 32, - "mode": "low_latency" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 70, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 32, - "mode": "low_latency" - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 393216, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 8, - "mode": "normal" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 393216, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 8, - "mode": "normal" - } - }, - { - "type": "allgather", - "args": { - "num_elements_per_rank": 2642411520, - "dtype": "bf16", - "world_size": 4 - } - }, - { - "type": "reduce_scatter", - "args": { - "num_elements": 10569646080, - "dtype": "bf16", - "world_size": 4 - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 368640, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 4, - "mode": "normal" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 368640, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 4, - "mode": "normal" - } - }, - { - "type": "allgather", - "args": { - "num_elements_per_rank": 330301440, - "dtype": "bf16", - "world_size": 32 - } - }, - { - "type": "reduce_scatter", - "args": { - "num_elements": 10569646080, - "dtype": "bf16", - "world_size": 32 - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 46080, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 32, - "mode": "normal" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 46080, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 32, - "mode": "normal" - } - }, - { - "type": "allgather", - "args": { - "num_elements_per_rank": 18041798656, - "dtype": "bf16", - "world_size": 4 - } - }, - { - "type": "reduce_scatter", - "args": { - "num_elements": 72167194624, - "dtype": "bf16", - "world_size": 4 - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 2516992, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 4, - "mode": "normal" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 2516992, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 4, - "mode": "normal" - } - }, - { - "type": "allgather", - "args": { - "num_elements_per_rank": 4401152, - "dtype": "bf16", - "world_size": 4 - } - }, - { - "type": "reduce_scatter", - "args": { - "num_elements": 17604608, - "dtype": "bf16", - "world_size": 4 - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 614, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 4, - "mode": "low_latency" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 614, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 4, - "mode": "low_latency" - } - }, - { - "type": "allgather", - "args": { - "num_elements_per_rank": 4510449664, - "dtype": "bf16", - "world_size": 16 - } - }, - { - "type": "reduce_scatter", - "args": { - "num_elements": 72167194624, - "dtype": "bf16", - "world_size": 16 - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 629248, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 16, - "mode": "normal" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 629248, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 16, - "mode": "normal" - } - }, - { - "type": "allgather", - "args": { - "num_elements_per_rank": 9776922624, - "dtype": "bf16", - "world_size": 4 - } - }, - { - "type": "reduce_scatter", - "args": { - "num_elements": 39107690496, - "dtype": "bf16", - "world_size": 4 - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 1363968, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 4, - "mode": "normal" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 1363968, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 4, - "mode": "normal" - } - }, - { - "type": "allgather", - "args": { - "num_elements_per_rank": 2386944, - "dtype": "bf16", - "world_size": 4 - } - }, - { - "type": "reduce_scatter", - "args": { - "num_elements": 9547776, - "dtype": "bf16", - "world_size": 4 - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 333, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 4, - "mode": "low_latency" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 333, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 4, - "mode": "low_latency" - } - }, - { - "type": "allgather", - "args": { - "num_elements_per_rank": 1222115328, - "dtype": "bf16", - "world_size": 32 - } - }, - { - "type": "reduce_scatter", - "args": { - "num_elements": 39107690496, - "dtype": "bf16", - "world_size": 32 - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 170496, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 32, - "mode": "normal" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 170496, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 32, - "mode": "normal" - } - }, - { - "type": "allgather", - "args": { - "num_elements_per_rank": 293888, - "dtype": "bf16", - "world_size": 32 - } - }, - { - "type": "reduce_scatter", - "args": { - "num_elements": 9404416, - "dtype": "bf16", - "world_size": 32 - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 41, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 32, - "mode": "low_latency" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 41, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 32, - "mode": "low_latency" - } - }, - { - "type": "allgather", - "args": { - "num_elements_per_rank": 63138955264, - "dtype": "bf16", - "world_size": 4 - } - }, - { - "type": "reduce_scatter", - "args": { - "num_elements": 252555821056, - "dtype": "bf16", - "world_size": 4 - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 8808448, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 4, - "mode": "normal" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 8808448, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 4, - "mode": "normal" - } - }, - { - "type": "allgather", - "args": { - "num_elements_per_rank": 15784738816, - "dtype": "bf16", - "world_size": 16 - } - }, - { - "type": "reduce_scatter", - "args": { - "num_elements": 252555821056, - "dtype": "bf16", - "world_size": 16 - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 2202112, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 16, - "mode": "normal" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 2202112, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 16, - "mode": "normal" - } - }, - { - "type": "allgather", - "args": { - "num_elements_per_rank": 645922816, - "dtype": "bf16", - "world_size": 4 - } - }, - { - "type": "reduce_scatter", - "args": { - "num_elements": 2583691264, - "dtype": "bf16", - "world_size": 4 - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 90112, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 4, - "mode": "normal" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 90112, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 4, - "mode": "normal" - } - }, - { - "type": "allgather", - "args": { - "num_elements_per_rank": 1115684864, - "dtype": "bf16", - "world_size": 4 - } - }, - { - "type": "reduce_scatter", - "args": { - "num_elements": 4462739456, - "dtype": "bf16", - "world_size": 4 - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 155648, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 4, - "mode": "normal" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 155648, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 4, - "mode": "normal" - } - }, - { - "type": "allgather", - "args": { - "num_elements_per_rank": 136192, - "dtype": "bf16", - "world_size": 4 - } - }, - { - "type": "reduce_scatter", - "args": { - "num_elements": 544768, - "dtype": "bf16", - "world_size": 4 - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 19, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 4, - "mode": "low_latency" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 19, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 4, - "mode": "low_latency" - } - }, - { - "type": "allreduce", - "args": { - "num_elements": 2583691264, - "dtype": "bf16", - "world_size": 8 - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 360448, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 8, - "mode": "normal" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 360448, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 8, - "mode": "normal" - } - }, - { - "type": "allreduce", - "args": { - "num_elements": 315392, - "dtype": "bf16", - "world_size": 8 - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 44, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 8, - "mode": "low_latency" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 44, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 8, - "mode": "low_latency" - } - }, - { - "type": "allreduce", - "args": { - "num_elements": 4462739456, - "dtype": "bf16", - "world_size": 8 - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 622592, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 8, - "mode": "normal" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 622592, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 8, - "mode": "normal" - } - }, - { - "type": "allreduce", - "args": { - "num_elements": 544768, - "dtype": "bf16", - "world_size": 8 - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 76, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 8, - "mode": "low_latency" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 76, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 8, - "mode": "low_latency" - } - }, - { - "type": "allgather", - "args": { - "num_elements_per_rank": 73400320, - "dtype": "bf16", - "world_size": 4 - } - }, - { - "type": "reduce_scatter", - "args": { - "num_elements": 293601280, - "dtype": "bf16", - "world_size": 4 - } - }, - { - "type": "allreduce", - "args": { - "num_elements": 293601280, - "dtype": "bf16", - "world_size": 8 - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 40960, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 8, - "mode": "normal" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 40960, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 8, - "mode": "normal" - } - }, - { - "type": "allgather", - "args": { - "num_elements_per_rank": 4888461312, - "dtype": "bf16", - "world_size": 4 - } - }, - { - "type": "reduce_scatter", - "args": { - "num_elements": 19553845248, - "dtype": "bf16", - "world_size": 4 - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 681984, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 4, - "mode": "normal" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 681984, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 4, - "mode": "normal" - } - }, - { - "type": "allgather", - "args": { - "num_elements_per_rank": 594944, - "dtype": "bf16", - "world_size": 4 - } - }, - { - "type": "reduce_scatter", - "args": { - "num_elements": 2379776, - "dtype": "bf16", - "world_size": 4 - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 83, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 4, - "mode": "low_latency" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 83, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 4, - "mode": "low_latency" - } - }, - { - "type": "allgather", - "args": { - "num_elements_per_rank": 611057664, - "dtype": "bf16", - "world_size": 32 - } - }, - { - "type": "reduce_scatter", - "args": { - "num_elements": 19553845248, - "dtype": "bf16", - "world_size": 32 - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 85248, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 32, - "mode": "normal" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 85248, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 32, - "mode": "normal" - } - }, - { - "type": "allgather", - "args": { - "num_elements_per_rank": 71680, - "dtype": "bf16", - "world_size": 32 - } - }, - { - "type": "reduce_scatter", - "args": { - "num_elements": 2293760, - "dtype": "bf16", - "world_size": 32 - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 10, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 32, - "mode": "low_latency" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 10, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 32, - "mode": "low_latency" - } - }, - { - "type": "allgather", - "args": { - "num_elements_per_rank": 2200576, - "dtype": "bf16", - "world_size": 4 - } - }, - { - "type": "reduce_scatter", - "args": { - "num_elements": 8802304, - "dtype": "bf16", - "world_size": 4 - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 307, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 4, - "mode": "low_latency" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 307, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 4, - "mode": "low_latency" - } - }, - { - "type": "allgather", - "args": { - "num_elements_per_rank": 2255224832, - "dtype": "bf16", - "world_size": 32 - } - }, - { - "type": "reduce_scatter", - "args": { - "num_elements": 72167194624, - "dtype": "bf16", - "world_size": 32 - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 314624, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 32, - "mode": "normal" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 314624, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 32, - "mode": "normal" - } - }, - { - "type": "allgather", - "args": { - "num_elements_per_rank": 272384, - "dtype": "bf16", - "world_size": 32 - } - }, - { - "type": "reduce_scatter", - "args": { - "num_elements": 8716288, - "dtype": "bf16", - "world_size": 32 - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 38, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 32, - "mode": "low_latency" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 38, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 32, - "mode": "low_latency" - } - }, - { - "type": "allgather", - "args": { - "num_elements_per_rank": 33074184192, - "dtype": "bf16", - "world_size": 4 - } - }, - { - "type": "reduce_scatter", - "args": { - "num_elements": 132296736768, - "dtype": "bf16", - "world_size": 4 - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 4614144, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 4, - "mode": "normal" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 4614144, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 4, - "mode": "normal" - } - }, - { - "type": "allgather", - "args": { - "num_elements_per_rank": 8268546048, - "dtype": "bf16", - "world_size": 16 - } - }, - { - "type": "reduce_scatter", - "args": { - "num_elements": 132296736768, - "dtype": "bf16", - "world_size": 16 - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 1153536, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 16, - "mode": "normal" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 1153536, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 16, - "mode": "normal" - } - }, - { - "type": "allgather", - "args": { - "num_elements_per_rank": 1003520, - "dtype": "bf16", - "world_size": 16 - } - }, - { - "type": "reduce_scatter", - "args": { - "num_elements": 16056320, - "dtype": "bf16", - "world_size": 16 - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 140, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 16, - "mode": "low_latency" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 140, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 16, - "mode": "low_latency" - } - }, - { - "type": "allgather", - "args": { - "num_elements_per_rank": 60129542144, - "dtype": "bf16", - "world_size": 4 - } - }, - { - "type": "reduce_scatter", - "args": { - "num_elements": 240518168576, - "dtype": "bf16", - "world_size": 4 - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 8388608, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 4, - "mode": "normal" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 8388608, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 4, - "mode": "normal" - } - }, - { - "type": "allgather", - "args": { - "num_elements_per_rank": 15032385536, - "dtype": "bf16", - "world_size": 16 - } - }, - { - "type": "reduce_scatter", - "args": { - "num_elements": 240518168576, - "dtype": "bf16", - "world_size": 16 - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 2097152, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 16, - "mode": "normal" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 2097152, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 16, - "mode": "normal" - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 256, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 16, - "mode": "low_latency" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 256, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 16, - "mode": "low_latency" - } - }, - { - "type": "allgather", - "args": { - "num_elements_per_rank": 7705600, - "dtype": "bf16", - "world_size": 8 - } - }, - { - "type": "reduce_scatter", - "args": { - "num_elements": 61644800, - "dtype": "bf16", - "world_size": 8 - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 1075, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 8, - "mode": "low_latency" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 1075, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 8, - "mode": "low_latency" - } - }, - { - "type": "allgather", - "args": { - "num_elements_per_rank": 1973551104, - "dtype": "bf16", - "world_size": 8 - } - }, - { - "type": "reduce_scatter", - "args": { - "num_elements": 15788408832, - "dtype": "bf16", - "world_size": 8 - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 275328, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 8, - "mode": "normal" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 275328, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 8, - "mode": "normal" - } - }, - { - "type": "allgather", - "args": { - "num_elements_per_rank": 1127612416, - "dtype": "bf16", - "world_size": 8 - } - }, - { - "type": "reduce_scatter", - "args": { - "num_elements": 9020899328, - "dtype": "bf16", - "world_size": 8 - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 157312, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 8, - "mode": "normal" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 157312, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 8, - "mode": "normal" - } - }, - { - "type": "allgather", - "args": { - "num_elements_per_rank": 2200576, - "dtype": "bf16", - "world_size": 8 - } - }, - { - "type": "reduce_scatter", - "args": { - "num_elements": 17604608, - "dtype": "bf16", - "world_size": 8 - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 307, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 8, - "mode": "low_latency" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 307, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 8, - "mode": "low_latency" - } - }, - { - "type": "allgather", - "args": { - "num_elements_per_rank": 564264960, - "dtype": "bf16", - "world_size": 8 - } - }, - { - "type": "reduce_scatter", - "args": { - "num_elements": 4514119680, - "dtype": "bf16", - "world_size": 8 - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 78720, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 8, - "mode": "normal" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 78720, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 8, - "mode": "normal" - } - }, - { - "type": "allgather", - "args": { - "num_elements_per_rank": 1096704, - "dtype": "bf16", - "world_size": 8 - } - }, - { - "type": "reduce_scatter", - "args": { - "num_elements": 8773632, - "dtype": "bf16", - "world_size": 8 - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 153, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 8, - "mode": "low_latency" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 153, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 8, - "mode": "low_latency" - } - }, - { - "type": "allgather", - "args": { - "num_elements_per_rank": 141066240, - "dtype": "bf16", - "world_size": 32 - } - }, - { - "type": "reduce_scatter", - "args": { - "num_elements": 4514119680, - "dtype": "bf16", - "world_size": 32 - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 19680, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 32, - "mode": "normal" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 19680, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 32, - "mode": "normal" - } - }, - { - "type": "allgather", - "args": { - "num_elements_per_rank": 33030144, - "dtype": "bf16", - "world_size": 8 - } - }, - { - "type": "reduce_scatter", - "args": { - "num_elements": 264241152, - "dtype": "bf16", - "world_size": 8 - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 4608, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 8, - "mode": "normal" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 4608, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 8, - "mode": "normal" - } - }, - { - "type": "allgather", - "args": { - "num_elements_per_rank": 64512, - "dtype": "bf16", - "world_size": 8 - } - }, - { - "type": "reduce_scatter", - "args": { - "num_elements": 516096, - "dtype": "bf16", - "world_size": 8 - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 9, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 8, - "mode": "low_latency" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 9, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 8, - "mode": "low_latency" - } - }, - { - "type": "allreduce", - "args": { - "num_elements": 264241152, - "dtype": "bf16", - "world_size": 8 - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 36864, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 8, - "mode": "normal" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 36864, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 8, - "mode": "normal" - } - }, - { - "type": "allreduce", - "args": { - "num_elements": 516096, - "dtype": "bf16", - "world_size": 8 - } - }, - { - "type": "allgather", - "args": { - "num_elements_per_rank": 16515072, - "dtype": "bf16", - "world_size": 8 - } - }, - { - "type": "reduce_scatter", - "args": { - "num_elements": 132120576, - "dtype": "bf16", - "world_size": 8 - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 2304, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 8, - "mode": "normal" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 2304, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 8, - "mode": "normal" - } - }, - { - "type": "allreduce", - "args": { - "num_elements": 132120576, - "dtype": "bf16", - "world_size": 8 - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 18432, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 8, - "mode": "normal" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 18432, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 8, - "mode": "normal" - } - }, - { - "type": "allreduce", - "args": { - "num_elements": 258048, - "dtype": "bf16", - "world_size": 8 - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 36, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 8, - "mode": "low_latency" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 36, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 8, - "mode": "low_latency" - } - }, - { - "type": "allgather", - "args": { - "num_elements_per_rank": 8257536, - "dtype": "bf16", - "world_size": 8 - } - }, - { - "type": "reduce_scatter", - "args": { - "num_elements": 66060288, - "dtype": "bf16", - "world_size": 8 - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 1152, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 8, - "mode": "normal" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 1152, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 8, - "mode": "normal" - } - }, - { - "type": "allreduce", - "args": { - "num_elements": 66060288, - "dtype": "bf16", - "world_size": 8 - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 9216, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 8, - "mode": "normal" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 9216, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 8, - "mode": "normal" - } - }, - { - "type": "allreduce", - "args": { - "num_elements": 129024, - "dtype": "bf16", - "world_size": 8 - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 18, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 8, - "mode": "low_latency" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 18, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 8, - "mode": "low_latency" - } - }, - { - "type": "allgather", - "args": { - "num_elements_per_rank": 5637144576, - "dtype": "bf16", - "world_size": 8 - } - }, - { - "type": "reduce_scatter", - "args": { - "num_elements": 45097156608, - "dtype": "bf16", - "world_size": 8 - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 786432, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 8, - "mode": "normal" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 786432, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 8, - "mode": "normal" - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 768, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 8, - "mode": "low_latency" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 768, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 8, - "mode": "low_latency" - } - }, - { - "type": "allgather", - "args": { - "num_elements_per_rank": 1921024, - "dtype": "bf16", - "world_size": 8 - } - }, - { - "type": "reduce_scatter", - "args": { - "num_elements": 15368192, - "dtype": "bf16", - "world_size": 8 - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 268, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 8, - "mode": "low_latency" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 268, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 8, - "mode": "low_latency" - } - }, - { - "type": "allgather", - "args": { - "num_elements_per_rank": 986775552, - "dtype": "bf16", - "world_size": 16 - } - }, - { - "type": "reduce_scatter", - "args": { - "num_elements": 15788408832, - "dtype": "bf16", - "world_size": 16 - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 137664, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 16, - "mode": "normal" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 137664, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 16, - "mode": "normal" - } - }, - { - "type": "allgather", - "args": { - "num_elements_per_rank": 960512, - "dtype": "bf16", - "world_size": 16 - } - }, - { - "type": "reduce_scatter", - "args": { - "num_elements": 15368192, - "dtype": "bf16", - "world_size": 16 - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 134, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 16, - "mode": "low_latency" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 134, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 16, - "mode": "low_latency" - } - }, - { - "type": "allgather", - "args": { - "num_elements_per_rank": 1034027008, - "dtype": "bf16", - "world_size": 8 - } - }, - { - "type": "reduce_scatter", - "args": { - "num_elements": 8272216064, - "dtype": "bf16", - "world_size": 8 - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 144256, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 8, - "mode": "normal" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 144256, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 8, - "mode": "normal" - } - }, - { - "type": "allgather", - "args": { - "num_elements_per_rank": 1003520, - "dtype": "bf16", - "world_size": 8 - } - }, - { - "type": "reduce_scatter", - "args": { - "num_elements": 8028160, - "dtype": "bf16", - "world_size": 8 - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 140, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 8, - "mode": "low_latency" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 140, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 8, - "mode": "low_latency" - } - }, - { - "type": "allgather", - "args": { - "num_elements_per_rank": 258506752, - "dtype": "bf16", - "world_size": 32 - } - }, - { - "type": "reduce_scatter", - "args": { - "num_elements": 8272216064, - "dtype": "bf16", - "world_size": 32 - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 36064, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 32, - "mode": "normal" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 36064, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 32, - "mode": "normal" - } - }, - { - "type": "allgather", - "args": { - "num_elements_per_rank": 250880, - "dtype": "bf16", - "world_size": 32 - } - }, - { - "type": "reduce_scatter", - "args": { - "num_elements": 8028160, - "dtype": "bf16", - "world_size": 32 - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 35, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 32, - "mode": "low_latency" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 35, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 32, - "mode": "low_latency" - } - }, - { - "type": "allgather", - "args": { - "num_elements_per_rank": 58720256, - "dtype": "bf16", - "world_size": 32 - } - }, - { - "type": "reduce_scatter", - "args": { - "num_elements": 1879048192, - "dtype": "bf16", - "world_size": 32 - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 8192, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 32, - "mode": "normal" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 8192, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 32, - "mode": "normal" - } - }, - { - "type": "allgather", - "args": { - "num_elements_per_rank": 57344, - "dtype": "bf16", - "world_size": 32 - } - }, - { - "type": "reduce_scatter", - "args": { - "num_elements": 1835008, - "dtype": "bf16", - "world_size": 32 - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 8, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 32, - "mode": "low_latency" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 8, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 32, - "mode": "low_latency" - } - }, - { - "type": "allgather", - "args": { - "num_elements_per_rank": 24772608, - "dtype": "bf16", - "world_size": 8 - } - }, - { - "type": "reduce_scatter", - "args": { - "num_elements": 198180864, - "dtype": "bf16", - "world_size": 8 - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 3456, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 8, - "mode": "normal" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 3456, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 8, - "mode": "normal" - } - }, - { - "type": "allgather", - "args": { - "num_elements_per_rank": 21504, - "dtype": "bf16", - "world_size": 8 - } - }, - { - "type": "reduce_scatter", - "args": { - "num_elements": 172032, - "dtype": "bf16", - "world_size": 8 - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 3, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 8, - "mode": "low_latency" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 3, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 8, - "mode": "low_latency" - } - }, - { - "type": "allreduce", - "args": { - "num_elements": 198180864, - "dtype": "bf16", - "world_size": 8 - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 27648, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 8, - "mode": "normal" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 27648, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 8, - "mode": "normal" - } - }, - { - "type": "allreduce", - "args": { - "num_elements": 193536, - "dtype": "bf16", - "world_size": 8 - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 27, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 8, - "mode": "low_latency" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 27, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 8, - "mode": "low_latency" - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 384, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 8, - "mode": "normal" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 384, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 8, - "mode": "normal" - } - }, - { - "type": "allreduce", - "args": { - "num_elements": 22020096, - "dtype": "bf16", - "world_size": 8 - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 3072, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 8, - "mode": "normal" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 3072, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 8, - "mode": "normal" - } - }, - { - "type": "allreduce", - "args": { - "num_elements": 21504, - "dtype": "bf16", - "world_size": 8 - } - }, - { - "type": "allgather", - "args": { - "num_elements_per_rank": 4888461312, - "dtype": "bf16", - "world_size": 8 - } - }, - { - "type": "reduce_scatter", - "args": { - "num_elements": 39107690496, - "dtype": "bf16", - "world_size": 8 - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 681984, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 8, - "mode": "normal" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 681984, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 8, - "mode": "normal" - } - }, - { - "type": "allgather", - "args": { - "num_elements_per_rank": 1189888, - "dtype": "bf16", - "world_size": 8 - } - }, - { - "type": "reduce_scatter", - "args": { - "num_elements": 9519104, - "dtype": "bf16", - "world_size": 8 - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 166, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 8, - "mode": "low_latency" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 166, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 8, - "mode": "low_latency" - } - }, - { - "type": "allgather", - "args": { - "num_elements_per_rank": 2444230656, - "dtype": "bf16", - "world_size": 16 - } - }, - { - "type": "reduce_scatter", - "args": { - "num_elements": 39107690496, - "dtype": "bf16", - "world_size": 16 - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 340992, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 16, - "mode": "normal" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 340992, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 16, - "mode": "normal" - } - }, - { - "type": "allgather", - "args": { - "num_elements_per_rank": 594944, - "dtype": "bf16", - "world_size": 16 - } - }, - { - "type": "reduce_scatter", - "args": { - "num_elements": 9519104, - "dtype": "bf16", - "world_size": 16 - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 83, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 16, - "mode": "low_latency" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 83, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 16, - "mode": "low_latency" - } - }, - { - "type": "allgather", - "args": { - "num_elements_per_rank": 2444230656, - "dtype": "bf16", - "world_size": 8 - } - }, - { - "type": "reduce_scatter", - "args": { - "num_elements": 19553845248, - "dtype": "bf16", - "world_size": 8 - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 340992, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 8, - "mode": "normal" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 340992, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 8, - "mode": "normal" - } - }, - { - "type": "allgather", - "args": { - "num_elements_per_rank": 594944, - "dtype": "bf16", - "world_size": 8 - } - }, - { - "type": "reduce_scatter", - "args": { - "num_elements": 4759552, - "dtype": "bf16", - "world_size": 8 - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 83, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 8, - "mode": "low_latency" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 83, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 8, - "mode": "low_latency" - } - }, - { - "type": "allgather", - "args": { - "num_elements_per_rank": 1222115328, - "dtype": "bf16", - "world_size": 16 - } - }, - { - "type": "reduce_scatter", - "args": { - "num_elements": 19553845248, - "dtype": "bf16", - "world_size": 16 - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 170496, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 16, - "mode": "normal" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 170496, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 16, - "mode": "normal" - } - }, - { - "type": "allgather", - "args": { - "num_elements_per_rank": 293888, - "dtype": "bf16", - "world_size": 16 - } - }, - { - "type": "reduce_scatter", - "args": { - "num_elements": 4702208, - "dtype": "bf16", - "world_size": 16 - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 41, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 16, - "mode": "low_latency" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 41, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 16, - "mode": "low_latency" - } - }, - { - "type": "allgather", - "args": { - "num_elements_per_rank": 165150720, - "dtype": "bf16", - "world_size": 32 - } - }, - { - "type": "reduce_scatter", - "args": { - "num_elements": 5284823040, - "dtype": "bf16", - "world_size": 32 - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 23040, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 32, - "mode": "normal" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 23040, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 32, - "mode": "normal" - } - }, - { - "type": "allgather", - "args": { - "num_elements_per_rank": 35840, - "dtype": "bf16", - "world_size": 32 - } - }, - { - "type": "reduce_scatter", - "args": { - "num_elements": 1146880, - "dtype": "bf16", - "world_size": 32 - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 5, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 32, - "mode": "low_latency" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 5, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 32, - "mode": "low_latency" - } - }, - { - "type": "allgather", - "args": { - "num_elements_per_rank": 110100480, - "dtype": "bf16", - "world_size": 8 - } - }, - { - "type": "reduce_scatter", - "args": { - "num_elements": 880803840, - "dtype": "bf16", - "world_size": 8 - } - }, - { - "type": "allgather", - "args": { - "num_elements_per_rank": 44040192, - "dtype": "bf16", - "world_size": 8 - } - }, - { - "type": "reduce_scatter", - "args": { - "num_elements": 352321536, - "dtype": "bf16", - "world_size": 8 - } - }, - { - "type": "allgather", - "args": { - "num_elements_per_rank": 9020899328, - "dtype": "bf16", - "world_size": 8 - } - }, - { - "type": "reduce_scatter", - "args": { - "num_elements": 72167194624, - "dtype": "bf16", - "world_size": 8 - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 1258496, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 8, - "mode": "normal" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 1258496, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 8, - "mode": "normal" - } - }, - { - "type": "allgather", - "args": { - "num_elements_per_rank": 4514119680, - "dtype": "bf16", - "world_size": 8 - } - }, - { - "type": "reduce_scatter", - "args": { - "num_elements": 36112957440, - "dtype": "bf16", - "world_size": 8 - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 629760, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 8, - "mode": "normal" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 629760, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 8, - "mode": "normal" - } - }, - { - "type": "allgather", - "args": { - "num_elements_per_rank": 544768, - "dtype": "bf16", - "world_size": 8 - } - }, - { - "type": "reduce_scatter", - "args": { - "num_elements": 4358144, - "dtype": "bf16", - "world_size": 8 - } - }, - { - "type": "allgather", - "args": { - "num_elements_per_rank": 2257059840, - "dtype": "bf16", - "world_size": 16 - } - }, - { - "type": "reduce_scatter", - "args": { - "num_elements": 36112957440, - "dtype": "bf16", - "world_size": 16 - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 314880, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 16, - "mode": "normal" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 314880, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 16, - "mode": "normal" - } - }, - { - "type": "allgather", - "args": { - "num_elements_per_rank": 272384, - "dtype": "bf16", - "world_size": 16 - } - }, - { - "type": "reduce_scatter", - "args": { - "num_elements": 4358144, - "dtype": "bf16", - "world_size": 16 - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 38, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 16, - "mode": "low_latency" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 38, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 16, - "mode": "low_latency" - } - }, - { - "type": "allgather", - "args": { - "num_elements_per_rank": 293888, - "dtype": "bf16", - "world_size": 8 - } - }, - { - "type": "reduce_scatter", - "args": { - "num_elements": 2351104, - "dtype": "bf16", - "world_size": 8 - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 41, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 8, - "mode": "low_latency" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 41, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 8, - "mode": "low_latency" - } - }, - { - "type": "allgather", - "args": { - "num_elements_per_rank": 462422016, - "dtype": "bf16", - "world_size": 8 - } - }, - { - "type": "reduce_scatter", - "args": { - "num_elements": 3699376128, - "dtype": "bf16", - "world_size": 8 - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 64512, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 8, - "mode": "normal" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 64512, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 8, - "mode": "normal" - } - }, - { - "type": "allgather", - "args": { - "num_elements_per_rank": 50176, - "dtype": "bf16", - "world_size": 8 - } - }, - { - "type": "reduce_scatter", - "args": { - "num_elements": 401408, - "dtype": "bf16", - "world_size": 8 - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 7, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 8, - "mode": "low_latency" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 7, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 8, - "mode": "low_latency" - } - }, - { - "type": "allgather", - "args": { - "num_elements_per_rank": 132120576, - "dtype": "bf16", - "world_size": 8 - } - }, - { - "type": "reduce_scatter", - "args": { - "num_elements": 1056964608, - "dtype": "bf16", - "world_size": 8 - } - }, - { - "type": "allreduce", - "args": { - "num_elements": 1056964608, - "dtype": "bf16", - "world_size": 8 - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 147456, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 8, - "mode": "normal" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 147456, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 8, - "mode": "normal" - } - }, - { - "type": "allgather", - "args": { - "num_elements_per_rank": 156584960, - "dtype": "bf16", - "world_size": 48 - } - }, - { - "type": "reduce_scatter", - "args": { - "num_elements": 7516078080, - "dtype": "bf16", - "world_size": 48 - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 21845, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 48, - "mode": "normal" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 21845, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 48, - "mode": "normal" - } - }, - { - "type": "allgather", - "args": { - "num_elements_per_rank": 150528, - "dtype": "bf16", - "world_size": 48 - } - }, - { - "type": "reduce_scatter", - "args": { - "num_elements": 7225344, - "dtype": "bf16", - "world_size": 48 - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 21, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 48, - "mode": "low_latency" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 21, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 48, - "mode": "low_latency" - } - }, - { - "type": "allgather", - "args": { - "num_elements_per_rank": 313169920, - "dtype": "bf16", - "world_size": 48 - } - }, - { - "type": "reduce_scatter", - "args": { - "num_elements": 15032156160, - "dtype": "bf16", - "world_size": 48 - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 43690, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 48, - "mode": "normal" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 43690, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 48, - "mode": "normal" - } - }, - { - "type": "allgather", - "args": { - "num_elements_per_rank": 301056, - "dtype": "bf16", - "world_size": 48 - } - }, - { - "type": "reduce_scatter", - "args": { - "num_elements": 14450688, - "dtype": "bf16", - "world_size": 48 - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 42, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 48, - "mode": "low_latency" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 42, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 48, - "mode": "low_latency" - } - }, - { - "type": "allgather", - "args": { - "num_elements_per_rank": 626347008, - "dtype": "bf16", - "world_size": 48 - } - }, - { - "type": "reduce_scatter", - "args": { - "num_elements": 30064656384, - "dtype": "bf16", - "world_size": 48 - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 87381, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 48, - "mode": "normal" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 87381, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 48, - "mode": "normal" - } - }, - { - "type": "allgather", - "args": { - "num_elements_per_rank": 609280, - "dtype": "bf16", - "world_size": 48 - } - }, - { - "type": "reduce_scatter", - "args": { - "num_elements": 29245440, - "dtype": "bf16", - "world_size": 48 - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 85, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 48, - "mode": "low_latency" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 85, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 48, - "mode": "low_latency" - } - }, - { - "type": "allgather", - "args": { - "num_elements_per_rank": 234881024, - "dtype": "bf16", - "world_size": 32 - } - }, - { - "type": "reduce_scatter", - "args": { - "num_elements": 7516192768, - "dtype": "bf16", - "world_size": 32 - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 32768, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 32, - "mode": "normal" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 32768, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 32, - "mode": "normal" - } - }, - { - "type": "allgather", - "args": { - "num_elements_per_rank": 229376, - "dtype": "bf16", - "world_size": 32 - } - }, - { - "type": "reduce_scatter", - "args": { - "num_elements": 7340032, - "dtype": "bf16", - "world_size": 32 - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 32, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 32, - "mode": "low_latency" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 32, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 32, - "mode": "low_latency" - } - }, - { - "type": "allgather", - "args": { - "num_elements_per_rank": 469762048, - "dtype": "bf16", - "world_size": 32 - } - }, - { - "type": "reduce_scatter", - "args": { - "num_elements": 15032385536, - "dtype": "bf16", - "world_size": 32 - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 65536, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 32, - "mode": "normal" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 65536, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 32, - "mode": "normal" - } - }, - { - "type": "allgather", - "args": { - "num_elements_per_rank": 458752, - "dtype": "bf16", - "world_size": 32 - } - }, - { - "type": "reduce_scatter", - "args": { - "num_elements": 14680064, - "dtype": "bf16", - "world_size": 32 - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 64, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 32, - "mode": "low_latency" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 64, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 32, - "mode": "low_latency" - } - }, - { - "type": "allgather", - "args": { - "num_elements_per_rank": 939524096, - "dtype": "bf16", - "world_size": 32 - } - }, - { - "type": "reduce_scatter", - "args": { - "num_elements": 30064771072, - "dtype": "bf16", - "world_size": 32 - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 131072, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 32, - "mode": "normal" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 131072, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 32, - "mode": "normal" - } - }, - { - "type": "allgather", - "args": { - "num_elements_per_rank": 917504, - "dtype": "bf16", - "world_size": 32 - } - }, - { - "type": "reduce_scatter", - "args": { - "num_elements": 29360128, - "dtype": "bf16", - "world_size": 32 - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 128, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 32, - "mode": "low_latency" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 128, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 32, - "mode": "low_latency" - } - }, - { - "type": "allgather", - "args": { - "num_elements_per_rank": 1409286144, - "dtype": "bf16", - "world_size": 32 - } - }, - { - "type": "reduce_scatter", - "args": { - "num_elements": 45097156608, - "dtype": "bf16", - "world_size": 32 - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 196608, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 32, - "mode": "normal" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 196608, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 32, - "mode": "normal" - } - }, - { - "type": "allgather", - "args": { - "num_elements_per_rank": 1376256, - "dtype": "bf16", - "world_size": 32 - } - }, - { - "type": "reduce_scatter", - "args": { - "num_elements": 44040192, - "dtype": "bf16", - "world_size": 32 - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 192, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 32, - "mode": "low_latency" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 192, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 32, - "mode": "low_latency" - } - }, - { - "type": "allgather", - "args": { - "num_elements_per_rank": 45097156608, - "dtype": "bf16", - "world_size": 8 - } - }, - { - "type": "reduce_scatter", - "args": { - "num_elements": 360777252864, - "dtype": "bf16", - "world_size": 8 - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 6291456, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 8, - "mode": "normal" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 6291456, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 8, - "mode": "normal" - } - }, - { - "type": "allgather", - "args": { - "num_elements_per_rank": 114688, - "dtype": "bf16", - "world_size": 32 - } - }, - { - "type": "reduce_scatter", - "args": { - "num_elements": 3670016, - "dtype": "bf16", - "world_size": 32 - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 16, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 32, - "mode": "low_latency" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 16, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 32, - "mode": "low_latency" - } - }, - { - "type": "allgather", - "args": { - "num_elements_per_rank": 1879048192, - "dtype": "bf16", - "world_size": 32 - } - }, - { - "type": "reduce_scatter", - "args": { - "num_elements": 60129542144, - "dtype": "bf16", - "world_size": 32 - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 262144, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 32, - "mode": "normal" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 262144, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 32, - "mode": "normal" - } - }, - { - "type": "allgather", - "args": { - "num_elements_per_rank": 3758096384, - "dtype": "bf16", - "world_size": 32 - } - }, - { - "type": "reduce_scatter", - "args": { - "num_elements": 120259084288, - "dtype": "bf16", - "world_size": 32 - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 524288, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 32, - "mode": "normal" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 524288, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 32, - "mode": "normal" - } - }, - { - "type": "allgather", - "args": { - "num_elements_per_rank": 11274289152, - "dtype": "bf16", - "world_size": 32 - } - }, - { - "type": "reduce_scatter", - "args": { - "num_elements": 360777252864, - "dtype": "bf16", - "world_size": 32 - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 1572864, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 32, - "mode": "normal" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 1572864, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 32, - "mode": "normal" - } - }, - { - "type": "allgather", - "args": { - "num_elements_per_rank": 5010790400, - "dtype": "bf16", - "world_size": 24 - } - }, - { - "type": "reduce_scatter", - "args": { - "num_elements": 120258969600, - "dtype": "bf16", - "world_size": 24 - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 699050, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 24, - "mode": "normal" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 699050, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 24, - "mode": "normal" - } - }, - { - "type": "allgather", - "args": { - "num_elements_per_rank": 609280, - "dtype": "bf16", - "world_size": 24 - } - }, - { - "type": "reduce_scatter", - "args": { - "num_elements": 14622720, - "dtype": "bf16", - "world_size": 24 - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 85, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 24, - "mode": "low_latency" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 85, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 24, - "mode": "low_latency" - } - }, - { - "type": "allgather", - "args": { - "num_elements_per_rank": 10021587968, - "dtype": "bf16", - "world_size": 24 - } - }, - { - "type": "reduce_scatter", - "args": { - "num_elements": 240518111232, - "dtype": "bf16", - "world_size": 24 - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 1398101, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 24, - "mode": "normal" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 1398101, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 24, - "mode": "normal" - } - }, - { - "type": "allgather", - "args": { - "num_elements_per_rank": 1218560, - "dtype": "bf16", - "world_size": 24 - } - }, - { - "type": "reduce_scatter", - "args": { - "num_elements": 29245440, - "dtype": "bf16", - "world_size": 24 - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 170, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 24, - "mode": "low_latency" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 170, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 24, - "mode": "low_latency" - } - }, - { - "type": "allgather", - "args": { - "num_elements_per_rank": 15032385536, - "dtype": "bf16", - "world_size": 24 - } - }, - { - "type": "reduce_scatter", - "args": { - "num_elements": 360777252864, - "dtype": "bf16", - "world_size": 24 - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 2097152, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 24, - "mode": "normal" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 2097152, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 24, - "mode": "normal" - } - }, - { - "type": "allgather", - "args": { - "num_elements_per_rank": 1835008, - "dtype": "bf16", - "world_size": 24 - } - }, - { - "type": "reduce_scatter", - "args": { - "num_elements": 44040192, - "dtype": "bf16", - "world_size": 24 - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 256, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 24, - "mode": "low_latency" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 256, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 24, - "mode": "low_latency" - } - }, - { - "type": "allgather", - "args": { - "num_elements_per_rank": 6576668672, - "dtype": "bf16", - "world_size": 8 - } - }, - { - "type": "reduce_scatter", - "args": { - "num_elements": 52613349376, - "dtype": "bf16", - "world_size": 8 - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 917504, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 8, - "mode": "normal" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 917504, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 8, - "mode": "normal" - } - }, - { - "type": "allgather", - "args": { - "num_elements_per_rank": 6422528, - "dtype": "bf16", - "world_size": 8 - } - }, - { - "type": "reduce_scatter", - "args": { - "num_elements": 51380224, - "dtype": "bf16", - "world_size": 8 - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 896, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 8, - "mode": "low_latency" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 896, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 8, - "mode": "low_latency" - } - }, - { - "type": "allgather", - "args": { - "num_elements_per_rank": 7046430720, - "dtype": "bf16", - "world_size": 8 - } - }, - { - "type": "reduce_scatter", - "args": { - "num_elements": 56371445760, - "dtype": "bf16", - "world_size": 8 - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 983040, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 8, - "mode": "normal" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 983040, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 8, - "mode": "normal" - } - }, - { - "type": "allgather", - "args": { - "num_elements_per_rank": 6881280, - "dtype": "bf16", - "world_size": 8 - } - }, - { - "type": "reduce_scatter", - "args": { - "num_elements": 55050240, - "dtype": "bf16", - "world_size": 8 - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 960, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 8, - "mode": "low_latency" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 960, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 8, - "mode": "low_latency" - } - }, - { - "type": "allgather", - "args": { - "num_elements_per_rank": 28672, - "dtype": "bf16", - "world_size": 32 - } - }, - { - "type": "reduce_scatter", - "args": { - "num_elements": 917504, - "dtype": "bf16", - "world_size": 32 - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 4, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 32, - "mode": "low_latency" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 4, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 32, - "mode": "low_latency" - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 512, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 4, - "mode": "low_latency" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 512, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 4, - "mode": "low_latency" - } - }, - { - "type": "allgather", - "args": { - "num_elements_per_rank": 15032385536, - "dtype": "bf16", - "world_size": 4 - } - }, - { - "type": "reduce_scatter", - "args": { - "num_elements": 60129542144, - "dtype": "bf16", - "world_size": 4 - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 2048, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 4, - "mode": "low_latency" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 2048, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 4, - "mode": "low_latency" - } - }, - { - "type": "allgather", - "args": { - "num_elements_per_rank": 117440512, - "dtype": "bf16", - "world_size": 32 - } - }, - { - "type": "reduce_scatter", - "args": { - "num_elements": 3758096384, - "dtype": "bf16", - "world_size": 32 - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 16384, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 32, - "mode": "normal" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 16384, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 32, - "mode": "normal" - } - }, - { - "type": "allgather", - "args": { - "num_elements_per_rank": 1835008, - "dtype": "bf16", - "world_size": 32 - } - }, - { - "type": "reduce_scatter", - "args": { - "num_elements": 58720256, - "dtype": "bf16", - "world_size": 32 - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 256, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 32, - "mode": "low_latency" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 256, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 32, - "mode": "low_latency" - } - }, - { - "type": "allreduce", - "args": { - "num_elements": 30064771072, - "dtype": "bf16", - "world_size": 4 - } - }, - { - "type": "allreduce", - "args": { - "num_elements": 120259084288, - "dtype": "bf16", - "world_size": 4 - } - }, - { - "type": "allreduce", - "args": { - "num_elements": 240518168576, - "dtype": "bf16", - "world_size": 4 - } - }, - { - "type": "allgather", - "args": { - "num_elements_per_rank": 71680, - "dtype": "bf16", - "world_size": 48 - } - }, - { - "type": "reduce_scatter", - "args": { - "num_elements": 3440640, - "dtype": "bf16", - "world_size": 48 - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 10, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 48, - "mode": "low_latency" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 10, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 48, - "mode": "low_latency" - } - }, - { - "type": "allgather", - "args": { - "num_elements_per_rank": 2505395200, - "dtype": "bf16", - "world_size": 48 - } - }, - { - "type": "reduce_scatter", - "args": { - "num_elements": 120258969600, - "dtype": "bf16", - "world_size": 48 - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 349525, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 48, - "mode": "normal" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 349525, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 48, - "mode": "normal" - } - }, - { - "type": "allgather", - "args": { - "num_elements_per_rank": 5010790400, - "dtype": "bf16", - "world_size": 48 - } - }, - { - "type": "reduce_scatter", - "args": { - "num_elements": 240517939200, - "dtype": "bf16", - "world_size": 48 - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 699050, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 48, - "mode": "normal" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 699050, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 48, - "mode": "normal" - } - }, - { - "type": "allgather", - "args": { - "num_elements_per_rank": 11839471616, - "dtype": "bf16", - "world_size": 2 - } - }, - { - "type": "reduce_scatter", - "args": { - "num_elements": 23678943232, - "dtype": "bf16", - "world_size": 2 - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 1651712, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 2, - "mode": "normal" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 1651712, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 2, - "mode": "normal" - } - }, - { - "type": "allgather", - "args": { - "num_elements_per_rank": 23123968, - "dtype": "bf16", - "world_size": 2 - } - }, - { - "type": "reduce_scatter", - "args": { - "num_elements": 46247936, - "dtype": "bf16", - "world_size": 2 - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 3226, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 2, - "mode": "low_latency" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 3226, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 2, - "mode": "low_latency" - } - }, - { - "type": "allgather", - "args": { - "num_elements_per_rank": 5919735808, - "dtype": "bf16", - "world_size": 4 - } - }, - { - "type": "reduce_scatter", - "args": { - "num_elements": 23678943232, - "dtype": "bf16", - "world_size": 4 - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 825856, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 4, - "mode": "normal" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 825856, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 4, - "mode": "normal" - } - }, - { - "type": "allgather", - "args": { - "num_elements_per_rank": 11561984, - "dtype": "bf16", - "world_size": 4 - } - }, - { - "type": "reduce_scatter", - "args": { - "num_elements": 46247936, - "dtype": "bf16", - "world_size": 4 - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 1613, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 4, - "mode": "low_latency" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 1613, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 4, - "mode": "low_latency" - } - }, - { - "type": "allgather", - "args": { - "num_elements_per_rank": 1222115328, - "dtype": "bf16", - "world_size": 2 - } - }, - { - "type": "reduce_scatter", - "args": { - "num_elements": 2444230656, - "dtype": "bf16", - "world_size": 2 - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 170496, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 2, - "mode": "normal" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 170496, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 2, - "mode": "normal" - } - }, - { - "type": "allgather", - "args": { - "num_elements_per_rank": 2386944, - "dtype": "bf16", - "world_size": 2 - } - }, - { - "type": "reduce_scatter", - "args": { - "num_elements": 4773888, - "dtype": "bf16", - "world_size": 2 - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 333, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 2, - "mode": "low_latency" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 333, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 2, - "mode": "low_latency" - } - }, - { - "type": "allgather", - "args": { - "num_elements_per_rank": 76382208, - "dtype": "bf16", - "world_size": 32 - } - }, - { - "type": "reduce_scatter", - "args": { - "num_elements": 2444230656, - "dtype": "bf16", - "world_size": 32 - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 10656, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 32, - "mode": "normal" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 10656, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 32, - "mode": "normal" - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 4096, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 2, - "mode": "normal" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 4096, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 2, - "mode": "normal" - } - }, - { - "type": "allgather", - "args": { - "num_elements_per_rank": 44040192, - "dtype": "bf16", - "world_size": 2 - } - }, - { - "type": "reduce_scatter", - "args": { - "num_elements": 88080384, - "dtype": "bf16", - "world_size": 2 - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 6144, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 2, - "mode": "normal" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 6144, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 2, - "mode": "normal" - } - }, - { - "type": "allgather", - "args": { - "num_elements_per_rank": 88080384, - "dtype": "bf16", - "world_size": 2 - } - }, - { - "type": "reduce_scatter", - "args": { - "num_elements": 176160768, - "dtype": "bf16", - "world_size": 2 - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 12288, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 2, - "mode": "normal" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 12288, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 2, - "mode": "normal" - } - }, - { - "type": "allgather", - "args": { - "num_elements_per_rank": 172032, - "dtype": "bf16", - "world_size": 2 - } - }, - { - "type": "reduce_scatter", - "args": { - "num_elements": 344064, - "dtype": "bf16", - "world_size": 2 - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 24, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 2, - "mode": "low_latency" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 24, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 2, - "mode": "low_latency" - } - }, - { - "type": "allgather", - "args": { - "num_elements_per_rank": 176160768, - "dtype": "bf16", - "world_size": 2 - } - }, - { - "type": "reduce_scatter", - "args": { - "num_elements": 352321536, - "dtype": "bf16", - "world_size": 2 - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 24576, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 2, - "mode": "normal" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 24576, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 2, - "mode": "normal" - } - }, - { - "type": "allgather", - "args": { - "num_elements_per_rank": 344064, - "dtype": "bf16", - "world_size": 2 - } - }, - { - "type": "reduce_scatter", - "args": { - "num_elements": 688128, - "dtype": "bf16", - "world_size": 2 - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 48, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 2, - "mode": "low_latency" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 48, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 2, - "mode": "low_latency" - } - }, - { - "type": "allgather", - "args": { - "num_elements_per_rank": 8268546048, - "dtype": "bf16", - "world_size": 2 - } - }, - { - "type": "reduce_scatter", - "args": { - "num_elements": 16537092096, - "dtype": "bf16", - "world_size": 2 - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 1153536, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 2, - "mode": "normal" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 1153536, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 2, - "mode": "normal" - } - }, - { - "type": "allgather", - "args": { - "num_elements_per_rank": 16149504, - "dtype": "bf16", - "world_size": 2 - } - }, - { - "type": "reduce_scatter", - "args": { - "num_elements": 32299008, - "dtype": "bf16", - "world_size": 2 - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 2253, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 2, - "mode": "low_latency" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 2253, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 2, - "mode": "low_latency" - } - }, - { - "type": "allgather", - "args": { - "num_elements_per_rank": 1033568256, - "dtype": "bf16", - "world_size": 16 - } - }, - { - "type": "reduce_scatter", - "args": { - "num_elements": 16537092096, - "dtype": "bf16", - "world_size": 16 - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 144192, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 16, - "mode": "normal" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 144192, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 16, - "mode": "normal" - } - }, - { - "type": "allgather", - "args": { - "num_elements_per_rank": 2014208, - "dtype": "bf16", - "world_size": 16 - } - }, - { - "type": "reduce_scatter", - "args": { - "num_elements": 32227328, - "dtype": "bf16", - "world_size": 16 - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 281, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 16, - "mode": "low_latency" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 281, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 16, - "mode": "low_latency" - } - }, - { - "type": "allgather", - "args": { - "num_elements_per_rank": 4510449664, - "dtype": "bf16", - "world_size": 2 - } - }, - { - "type": "reduce_scatter", - "args": { - "num_elements": 9020899328, - "dtype": "bf16", - "world_size": 2 - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 629248, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 2, - "mode": "normal" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 629248, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 2, - "mode": "normal" - } - }, - { - "type": "allgather", - "args": { - "num_elements_per_rank": 8809472, - "dtype": "bf16", - "world_size": 2 - } - }, - { - "type": "reduce_scatter", - "args": { - "num_elements": 17618944, - "dtype": "bf16", - "world_size": 2 - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 1229, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 2, - "mode": "low_latency" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 1229, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 2, - "mode": "low_latency" - } - }, - { - "type": "allgather", - "args": { - "num_elements_per_rank": 281903104, - "dtype": "bf16", - "world_size": 32 - } - }, - { - "type": "reduce_scatter", - "args": { - "num_elements": 9020899328, - "dtype": "bf16", - "world_size": 32 - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 39328, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 32, - "mode": "normal" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 39328, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 32, - "mode": "normal" - } - }, - { - "type": "allgather", - "args": { - "num_elements_per_rank": 544768, - "dtype": "bf16", - "world_size": 32 - } - }, - { - "type": "reduce_scatter", - "args": { - "num_elements": 17432576, - "dtype": "bf16", - "world_size": 32 - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 76, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 32, - "mode": "low_latency" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 76, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 32, - "mode": "low_latency" - } - }, - { - "type": "allgather", - "args": { - "num_elements_per_rank": 704643072, - "dtype": "bf16", - "world_size": 2 - } - }, - { - "type": "reduce_scatter", - "args": { - "num_elements": 1409286144, - "dtype": "bf16", - "world_size": 2 - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 98304, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 2, - "mode": "normal" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 98304, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 2, - "mode": "normal" - } - }, - { - "type": "allgather", - "args": { - "num_elements_per_rank": 688128, - "dtype": "bf16", - "world_size": 2 - } - }, - { - "type": "reduce_scatter", - "args": { - "num_elements": 1376256, - "dtype": "bf16", - "world_size": 2 - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 96, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 2, - "mode": "low_latency" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 96, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 2, - "mode": "low_latency" - } - }, - { - "type": "allgather", - "args": { - "num_elements_per_rank": 4401152, - "dtype": "bf16", - "world_size": 2 - } - }, - { - "type": "reduce_scatter", - "args": { - "num_elements": 8802304, - "dtype": "bf16", - "world_size": 2 - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 614, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 2, - "mode": "low_latency" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 614, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 2, - "mode": "low_latency" - } - }, - { - "type": "allgather", - "args": { - "num_elements_per_rank": 15784738816, - "dtype": "bf16", - "world_size": 2 - } - }, - { - "type": "reduce_scatter", - "args": { - "num_elements": 31569477632, - "dtype": "bf16", - "world_size": 2 - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 2202112, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 2, - "mode": "normal" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 2202112, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 2, - "mode": "normal" - } - }, - { - "type": "allgather", - "args": { - "num_elements_per_rank": 15411200, - "dtype": "bf16", - "world_size": 2 - } - }, - { - "type": "reduce_scatter", - "args": { - "num_elements": 30822400, - "dtype": "bf16", - "world_size": 2 - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 2150, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 2, - "mode": "low_latency" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 2150, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 2, - "mode": "low_latency" - } - }, - { - "type": "allgather", - "args": { - "num_elements_per_rank": 8071168, - "dtype": "bf16", - "world_size": 2 - } - }, - { - "type": "reduce_scatter", - "args": { - "num_elements": 16142336, - "dtype": "bf16", - "world_size": 2 - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 1126, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 2, - "mode": "low_latency" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 1126, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 2, - "mode": "low_latency" - } - }, - { - "type": "allgather", - "args": { - "num_elements_per_rank": 968884224, - "dtype": "bf16", - "world_size": 2 - } - }, - { - "type": "reduce_scatter", - "args": { - "num_elements": 1937768448, - "dtype": "bf16", - "world_size": 2 - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 135168, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 2, - "mode": "normal" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 135168, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 2, - "mode": "normal" - } - }, - { - "type": "allgather", - "args": { - "num_elements_per_rank": 236544, - "dtype": "bf16", - "world_size": 2 - } - }, - { - "type": "reduce_scatter", - "args": { - "num_elements": 473088, - "dtype": "bf16", - "world_size": 2 - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 33, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 2, - "mode": "low_latency" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 33, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 2, - "mode": "low_latency" - } - }, - { - "type": "allreduce", - "args": { - "num_elements": 1937768448, - "dtype": "bf16", - "world_size": 8 - } - }, - { - "type": "allreduce", - "args": { - "num_elements": 473088, - "dtype": "bf16", - "world_size": 8 - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 66, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 8, - "mode": "low_latency" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 66, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 8, - "mode": "low_latency" - } - }, - { - "type": "allgather", - "args": { - "num_elements_per_rank": 146800640, - "dtype": "bf16", - "world_size": 2 - } - }, - { - "type": "reduce_scatter", - "args": { - "num_elements": 293601280, - "dtype": "bf16", - "world_size": 2 - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 20480, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 2, - "mode": "normal" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 20480, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 2, - "mode": "normal" - } - }, - { - "type": "allgather", - "args": { - "num_elements_per_rank": 5284823040, - "dtype": "bf16", - "world_size": 2 - } - }, - { - "type": "reduce_scatter", - "args": { - "num_elements": 10569646080, - "dtype": "bf16", - "world_size": 2 - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 737280, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 2, - "mode": "normal" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 737280, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 2, - "mode": "normal" - } - }, - { - "type": "allgather", - "args": { - "num_elements_per_rank": 1290240, - "dtype": "bf16", - "world_size": 2 - } - }, - { - "type": "reduce_scatter", - "args": { - "num_elements": 2580480, - "dtype": "bf16", - "world_size": 2 - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 180, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 2, - "mode": "low_latency" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 180, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 2, - "mode": "low_latency" - } - }, - { - "type": "allgather", - "args": { - "num_elements_per_rank": 9042919424, - "dtype": "bf16", - "world_size": 2 - } - }, - { - "type": "reduce_scatter", - "args": { - "num_elements": 18085838848, - "dtype": "bf16", - "world_size": 2 - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 1261568, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 2, - "mode": "normal" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 1261568, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 2, - "mode": "normal" - } - }, - { - "type": "allgather", - "args": { - "num_elements_per_rank": 2207744, - "dtype": "bf16", - "world_size": 2 - } - }, - { - "type": "reduce_scatter", - "args": { - "num_elements": 4415488, - "dtype": "bf16", - "world_size": 2 - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 308, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 2, - "mode": "low_latency" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 308, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 2, - "mode": "low_latency" - } - }, - { - "type": "allgather", - "args": { - "num_elements_per_rank": 565182464, - "dtype": "bf16", - "world_size": 32 - } - }, - { - "type": "reduce_scatter", - "args": { - "num_elements": 18085838848, - "dtype": "bf16", - "world_size": 32 - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 78848, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 32, - "mode": "normal" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 78848, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 32, - "mode": "normal" - } - }, - { - "type": "allgather", - "args": { - "num_elements_per_rank": 136192, - "dtype": "bf16", - "world_size": 32 - } - }, - { - "type": "reduce_scatter", - "args": { - "num_elements": 4358144, - "dtype": "bf16", - "world_size": 32 - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 19, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 32, - "mode": "low_latency" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 19, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 32, - "mode": "low_latency" - } - }, - { - "type": "allgather", - "args": { - "num_elements_per_rank": 66148368384, - "dtype": "bf16", - "world_size": 2 - } - }, - { - "type": "reduce_scatter", - "args": { - "num_elements": 132296736768, - "dtype": "bf16", - "world_size": 2 - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 9228288, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 2, - "mode": "normal" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 9228288, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 2, - "mode": "normal" - } - }, - { - "type": "allgather", - "args": { - "num_elements_per_rank": 16537092096, - "dtype": "bf16", - "world_size": 8 - } - }, - { - "type": "reduce_scatter", - "args": { - "num_elements": 132296736768, - "dtype": "bf16", - "world_size": 8 - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 2307072, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 8, - "mode": "normal" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 2307072, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 8, - "mode": "normal" - } - }, - { - "type": "allgather", - "args": { - "num_elements_per_rank": 4035584, - "dtype": "bf16", - "world_size": 8 - } - }, - { - "type": "reduce_scatter", - "args": { - "num_elements": 32284672, - "dtype": "bf16", - "world_size": 8 - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 563, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 8, - "mode": "low_latency" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 563, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 8, - "mode": "low_latency" - } - }, - { - "type": "allgather", - "args": { - "num_elements_per_rank": 19553845248, - "dtype": "bf16", - "world_size": 2 - } - }, - { - "type": "reduce_scatter", - "args": { - "num_elements": 39107690496, - "dtype": "bf16", - "world_size": 2 - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 2727936, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 2, - "mode": "normal" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 2727936, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 2, - "mode": "normal" - } - }, - { - "type": "allgather", - "args": { - "num_elements_per_rank": 4773888, - "dtype": "bf16", - "world_size": 2 - } - }, - { - "type": "reduce_scatter", - "args": { - "num_elements": 9547776, - "dtype": "bf16", - "world_size": 2 - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 666, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 2, - "mode": "low_latency" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 666, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 2, - "mode": "low_latency" - } - }, - { - "type": "allgather", - "args": { - "num_elements_per_rank": 33088864256, - "dtype": "bf16", - "world_size": 2 - } - }, - { - "type": "reduce_scatter", - "args": { - "num_elements": 66177728512, - "dtype": "bf16", - "world_size": 2 - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 4616192, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 2, - "mode": "normal" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 4616192, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 2, - "mode": "normal" - } - }, - { - "type": "allgather", - "args": { - "num_elements_per_rank": 8078336, - "dtype": "bf16", - "world_size": 2 - } - }, - { - "type": "reduce_scatter", - "args": { - "num_elements": 16156672, - "dtype": "bf16", - "world_size": 2 - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 1127, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 2, - "mode": "low_latency" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 1127, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 2, - "mode": "low_latency" - } - }, - { - "type": "allgather", - "args": { - "num_elements_per_rank": 4136108032, - "dtype": "bf16", - "world_size": 16 - } - }, - { - "type": "reduce_scatter", - "args": { - "num_elements": 66177728512, - "dtype": "bf16", - "world_size": 16 - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 577024, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 16, - "mode": "normal" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 577024, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 16, - "mode": "normal" - } - }, - { - "type": "allgather", - "args": { - "num_elements_per_rank": 2113929216, - "dtype": "bf16", - "world_size": 2 - } - }, - { - "type": "reduce_scatter", - "args": { - "num_elements": 4227858432, - "dtype": "bf16", - "world_size": 2 - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 294912, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 2, - "mode": "normal" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 294912, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 2, - "mode": "normal" - } - }, - { - "type": "allgather", - "args": { - "num_elements_per_rank": 258048, - "dtype": "bf16", - "world_size": 2 - } - }, - { - "type": "reduce_scatter", - "args": { - "num_elements": 516096, - "dtype": "bf16", - "world_size": 2 - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 36, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 2, - "mode": "low_latency" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 36, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 2, - "mode": "low_latency" - } - }, - { - "type": "allreduce", - "args": { - "num_elements": 4227858432, - "dtype": "bf16", - "world_size": 8 - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 589824, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 8, - "mode": "normal" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 589824, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 8, - "mode": "normal" - } - }, - { - "type": "allgather", - "args": { - "num_elements_per_rank": 880803840, - "dtype": "bf16", - "world_size": 2 - } - }, - { - "type": "reduce_scatter", - "args": { - "num_elements": 1761607680, - "dtype": "bf16", - "world_size": 2 - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 122880, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 2, - "mode": "normal" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 122880, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 2, - "mode": "normal" - } - }, - { - "type": "allgather", - "args": { - "num_elements_per_rank": 107520, - "dtype": "bf16", - "world_size": 2 - } - }, - { - "type": "reduce_scatter", - "args": { - "num_elements": 215040, - "dtype": "bf16", - "world_size": 2 - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 15, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 2, - "mode": "low_latency" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 15, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 2, - "mode": "low_latency" - } - }, - { - "type": "allreduce", - "args": { - "num_elements": 293601280, - "dtype": "bf16", - "world_size": 4 - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 40960, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 4, - "mode": "normal" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 40960, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 4, - "mode": "normal" - } - }, - { - "type": "allreduce", - "args": { - "num_elements": 880803840, - "dtype": "bf16", - "world_size": 4 - } - }, - { - "type": "allreduce", - "args": { - "num_elements": 1761607680, - "dtype": "bf16", - "world_size": 4 - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 245760, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 4, - "mode": "normal" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 245760, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 4, - "mode": "normal" - } - }, - { - "type": "allreduce", - "args": { - "num_elements": 215040, - "dtype": "bf16", - "world_size": 4 - } - }, - { - "type": "allgather", - "args": { - "num_elements_per_rank": 36083597312, - "dtype": "bf16", - "world_size": 2 - } - }, - { - "type": "reduce_scatter", - "args": { - "num_elements": 72167194624, - "dtype": "bf16", - "world_size": 2 - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 5033984, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 2, - "mode": "normal" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 5033984, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 2, - "mode": "normal" - } - }, - { - "type": "allgather", - "args": { - "num_elements_per_rank": 94774493184, - "dtype": "bf16", - "world_size": 2 - } - }, - { - "type": "reduce_scatter", - "args": { - "num_elements": 189548986368, - "dtype": "bf16", - "world_size": 2 - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 13221888, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 2, - "mode": "normal" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 13221888, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 2, - "mode": "normal" - } - }, - { - "type": "allgather", - "args": { - "num_elements_per_rank": 11569152, - "dtype": "bf16", - "world_size": 2 - } - }, - { - "type": "reduce_scatter", - "args": { - "num_elements": 23138304, - "dtype": "bf16", - "world_size": 2 - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 1614, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 2, - "mode": "low_latency" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 1614, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 2, - "mode": "low_latency" - } - }, - { - "type": "allgather", - "args": { - "num_elements_per_rank": 47387246592, - "dtype": "bf16", - "world_size": 4 - } - }, - { - "type": "reduce_scatter", - "args": { - "num_elements": 189548986368, - "dtype": "bf16", - "world_size": 4 - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 6610944, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 4, - "mode": "normal" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 6610944, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 4, - "mode": "normal" - } - }, - { - "type": "allgather", - "args": { - "num_elements_per_rank": 5784576, - "dtype": "bf16", - "world_size": 4 - } - }, - { - "type": "reduce_scatter", - "args": { - "num_elements": 23138304, - "dtype": "bf16", - "world_size": 4 - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 807, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 4, - "mode": "low_latency" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 807, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 4, - "mode": "low_latency" - } - }, - { - "type": "allgather", - "args": { - "num_elements_per_rank": 78288896, - "dtype": "bf16", - "world_size": 48 - } - }, - { - "type": "reduce_scatter", - "args": { - "num_elements": 3757867008, - "dtype": "bf16", - "world_size": 48 - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 10922, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 48, - "mode": "normal" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 10922, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 48, - "mode": "normal" - } - }, - { - "type": "allgather", - "args": { - "num_elements_per_rank": 1252694016, - "dtype": "bf16", - "world_size": 48 - } - }, - { - "type": "reduce_scatter", - "args": { - "num_elements": 60129312768, - "dtype": "bf16", - "world_size": 48 - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 174762, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 48, - "mode": "normal" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 174762, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 48, - "mode": "normal" - } - }, - { - "type": "allgather", - "args": { - "num_elements_per_rank": 1218560, - "dtype": "bf16", - "world_size": 48 - } - }, - { - "type": "reduce_scatter", - "args": { - "num_elements": 58490880, - "dtype": "bf16", - "world_size": 48 - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 170, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 48, - "mode": "low_latency" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 170, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 48, - "mode": "low_latency" - } - }, - { - "type": "allgather", - "args": { - "num_elements_per_rank": 1034944512, - "dtype": "bf16", - "world_size": 4 - } - }, - { - "type": "reduce_scatter", - "args": { - "num_elements": 4139778048, - "dtype": "bf16", - "world_size": 4 - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 144384, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 4, - "mode": "normal" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 144384, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 4, - "mode": "normal" - } - }, - { - "type": "allgather", - "args": { - "num_elements_per_rank": 2021376, - "dtype": "bf16", - "world_size": 4 - } - }, - { - "type": "reduce_scatter", - "args": { - "num_elements": 8085504, - "dtype": "bf16", - "world_size": 4 - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 282, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 4, - "mode": "low_latency" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 282, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 4, - "mode": "low_latency" - } - }, - { - "type": "allgather", - "args": { - "num_elements_per_rank": 129368064, - "dtype": "bf16", - "world_size": 32 - } - }, - { - "type": "reduce_scatter", - "args": { - "num_elements": 4139778048, - "dtype": "bf16", - "world_size": 32 - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 18048, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 32, - "mode": "normal" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 18048, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 32, - "mode": "normal" - } - }, - { - "type": "allgather", - "args": { - "num_elements_per_rank": 305528832, - "dtype": "bf16", - "world_size": 16 - } - }, - { - "type": "reduce_scatter", - "args": { - "num_elements": 4888461312, - "dtype": "bf16", - "world_size": 16 - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 42624, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 16, - "mode": "normal" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 42624, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 16, - "mode": "normal" - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 4096, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 4, - "mode": "low_latency" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 4096, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 4, - "mode": "low_latency" - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 2048, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 8, - "mode": "low_latency" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 2048, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 8, - "mode": "low_latency" - } - }, - { - "type": "allgather", - "args": { - "num_elements_per_rank": 154140672, - "dtype": "bf16", - "world_size": 4 - } - }, - { - "type": "reduce_scatter", - "args": { - "num_elements": 616562688, - "dtype": "bf16", - "world_size": 4 - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 21504, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 4, - "mode": "normal" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 21504, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 4, - "mode": "normal" - } - }, - { - "type": "allgather", - "args": { - "num_elements_per_rank": 150528, - "dtype": "bf16", - "world_size": 4 - } - }, - { - "type": "reduce_scatter", - "args": { - "num_elements": 602112, - "dtype": "bf16", - "world_size": 4 - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 21, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 4, - "mode": "low_latency" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 21, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 4, - "mode": "low_latency" - } - }, - { - "type": "allreduce", - "args": { - "num_elements": 616562688, - "dtype": "bf16", - "world_size": 8 - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 86016, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 8, - "mode": "normal" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 86016, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 8, - "mode": "normal" - } - }, - { - "type": "allreduce", - "args": { - "num_elements": 602112, - "dtype": "bf16", - "world_size": 8 - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 84, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 8, - "mode": "low_latency" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 84, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 8, - "mode": "low_latency" - } - }, - { - "type": "allgather", - "args": { - "num_elements_per_rank": 2255224832, - "dtype": "bf16", - "world_size": 4 - } - }, - { - "type": "reduce_scatter", - "args": { - "num_elements": 9020899328, - "dtype": "bf16", - "world_size": 4 - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 314624, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 4, - "mode": "normal" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 314624, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 4, - "mode": "normal" - } - }, - { - "type": "allgather", - "args": { - "num_elements_per_rank": 15784738816, - "dtype": "bf16", - "world_size": 4 - } - }, - { - "type": "reduce_scatter", - "args": { - "num_elements": 63138955264, - "dtype": "bf16", - "world_size": 4 - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 2202112, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 4, - "mode": "normal" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 2202112, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 4, - "mode": "normal" - } - }, - { - "type": "allgather", - "args": { - "num_elements_per_rank": 7892369408, - "dtype": "bf16", - "world_size": 8 - } - }, - { - "type": "reduce_scatter", - "args": { - "num_elements": 63138955264, - "dtype": "bf16", - "world_size": 8 - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 1101056, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 8, - "mode": "normal" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 1101056, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 8, - "mode": "normal" - } - }, - { - "type": "allgather", - "args": { - "num_elements_per_rank": 22548578304, - "dtype": "bf16", - "world_size": 4 - } - }, - { - "type": "reduce_scatter", - "args": { - "num_elements": 90194313216, - "dtype": "bf16", - "world_size": 4 - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 3145728, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 4, - "mode": "normal" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 3145728, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 4, - "mode": "normal" - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 3072, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 4, - "mode": "low_latency" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 3072, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 4, - "mode": "low_latency" - } - }, - { - "type": "allgather", - "args": { - "num_elements_per_rank": 11274289152, - "dtype": "bf16", - "world_size": 8 - } - }, - { - "type": "reduce_scatter", - "args": { - "num_elements": 90194313216, - "dtype": "bf16", - "world_size": 8 - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 1572864, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 8, - "mode": "normal" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 1572864, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 8, - "mode": "normal" - } - }, - { - "type": "allgather", - "args": { - "num_elements_per_rank": 11010048, - "dtype": "bf16", - "world_size": 8 - } - }, - { - "type": "reduce_scatter", - "args": { - "num_elements": 88080384, - "dtype": "bf16", - "world_size": 8 - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 1536, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 8, - "mode": "low_latency" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 1536, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 8, - "mode": "low_latency" - } - }, - { - "type": "allgather", - "args": { - "num_elements_per_rank": 528482304, - "dtype": "bf16", - "world_size": 4 - } - }, - { - "type": "reduce_scatter", - "args": { - "num_elements": 2113929216, - "dtype": "bf16", - "world_size": 4 - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 73728, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 4, - "mode": "normal" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 73728, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 4, - "mode": "normal" - } - }, - { - "type": "allgather", - "args": { - "num_elements_per_rank": 64512, - "dtype": "bf16", - "world_size": 4 - } - }, - { - "type": "reduce_scatter", - "args": { - "num_elements": 258048, - "dtype": "bf16", - "world_size": 4 - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 9, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 4, - "mode": "low_latency" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 9, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 4, - "mode": "low_latency" - } - }, - { - "type": "allreduce", - "args": { - "num_elements": 2113929216, - "dtype": "bf16", - "world_size": 8 - } - }, - { - "type": "allgather", - "args": { - "num_elements_per_rank": 31576817664, - "dtype": "bf16", - "world_size": 4 - } - }, - { - "type": "reduce_scatter", - "args": { - "num_elements": 126307270656, - "dtype": "bf16", - "world_size": 4 - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 4405248, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 4, - "mode": "normal" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 4405248, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 4, - "mode": "normal" - } - }, - { - "type": "allgather", - "args": { - "num_elements_per_rank": 3849216, - "dtype": "bf16", - "world_size": 4 - } - }, - { - "type": "reduce_scatter", - "args": { - "num_elements": 15396864, - "dtype": "bf16", - "world_size": 4 - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 537, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 4, - "mode": "low_latency" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 537, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 4, - "mode": "low_latency" - } - }, - { - "type": "allgather", - "args": { - "num_elements_per_rank": 15788408832, - "dtype": "bf16", - "world_size": 8 - } - }, - { - "type": "reduce_scatter", - "args": { - "num_elements": 126307270656, - "dtype": "bf16", - "world_size": 8 - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 2202624, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 8, - "mode": "normal" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 2202624, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 8, - "mode": "normal" - } - }, - { - "type": "allreduce", - "args": { - "num_elements": 2949120, - "dtype": "bf16", - "world_size": 4 - } - }, - { - "type": "allreduce", - "args": { - "num_elements": 2880, - "dtype": "bf16", - "world_size": 4 - } - }, - { - "type": "allreduce", - "args": { - "num_elements": 5898240, - "dtype": "bf16", - "world_size": 4 - } - }, - { - "type": "allreduce", - "args": { - "num_elements": 5760, - "dtype": "bf16", - "world_size": 4 - } - }, - { - "type": "allreduce", - "args": { - "num_elements": 368640, - "dtype": "bf16", - "world_size": 4 - } - }, - { - "type": "allgather", - "args": { - "num_elements_per_rank": 754974720, - "dtype": "bf16", - "world_size": 2 - } - }, - { - "type": "reduce_scatter", - "args": { - "num_elements": 1509949440, - "dtype": "bf16", - "world_size": 2 - } - }, - { - "type": "allgather", - "args": { - "num_elements_per_rank": 737280, - "dtype": "bf16", - "world_size": 2 - } - }, - { - "type": "reduce_scatter", - "args": { - "num_elements": 1474560, - "dtype": "bf16", - "world_size": 2 - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 256, - "num_experts": 128, - "top_k": 4, - "hidden": 2880, - "dtype": "bf16", - "world_size": 2, - "mode": "low_latency" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 256, - "num_experts": 128, - "top_k": 4, - "hidden": 2880, - "dtype": "bf16", - "world_size": 2, - "mode": "low_latency" - } - }, - { - "type": "allgather", - "args": { - "num_elements_per_rank": 1509949440, - "dtype": "bf16", - "world_size": 2 - } - }, - { - "type": "reduce_scatter", - "args": { - "num_elements": 3019898880, - "dtype": "bf16", - "world_size": 2 - } - }, - { - "type": "allgather", - "args": { - "num_elements_per_rank": 1474560, - "dtype": "bf16", - "world_size": 2 - } - }, - { - "type": "reduce_scatter", - "args": { - "num_elements": 2949120, - "dtype": "bf16", - "world_size": 2 - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 512, - "num_experts": 128, - "top_k": 4, - "hidden": 2880, - "dtype": "bf16", - "world_size": 2, - "mode": "low_latency" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 512, - "num_experts": 128, - "top_k": 4, - "hidden": 2880, - "dtype": "bf16", - "world_size": 2, - "mode": "low_latency" - } - }, - { - "type": "allgather", - "args": { - "num_elements_per_rank": 3019898880, - "dtype": "bf16", - "world_size": 2 - } - }, - { - "type": "reduce_scatter", - "args": { - "num_elements": 6039797760, - "dtype": "bf16", - "world_size": 2 - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 1048576, - "num_experts": 128, - "top_k": 4, - "hidden": 2880, - "dtype": "bf16", - "world_size": 2, - "mode": "normal" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 1048576, - "num_experts": 128, - "top_k": 4, - "hidden": 2880, - "dtype": "bf16", - "world_size": 2, - "mode": "normal" - } - }, - { - "type": "allgather", - "args": { - "num_elements_per_rank": 2949120, - "dtype": "bf16", - "world_size": 2 - } - }, - { - "type": "reduce_scatter", - "args": { - "num_elements": 5898240, - "dtype": "bf16", - "world_size": 2 - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 1024, - "num_experts": 128, - "top_k": 4, - "hidden": 2880, - "dtype": "bf16", - "world_size": 2, - "mode": "low_latency" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 1024, - "num_experts": 128, - "top_k": 4, - "hidden": 2880, - "dtype": "bf16", - "world_size": 2, - "mode": "low_latency" - } - }, - { - "type": "allgather", - "args": { - "num_elements_per_rank": 3774873600, - "dtype": "bf16", - "world_size": 2 - } - }, - { - "type": "reduce_scatter", - "args": { - "num_elements": 7549747200, - "dtype": "bf16", - "world_size": 2 - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 1310720, - "num_experts": 128, - "top_k": 4, - "hidden": 2880, - "dtype": "bf16", - "world_size": 2, - "mode": "normal" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 1310720, - "num_experts": 128, - "top_k": 4, - "hidden": 2880, - "dtype": "bf16", - "world_size": 2, - "mode": "normal" - } - }, - { - "type": "allgather", - "args": { - "num_elements_per_rank": 3686400, - "dtype": "bf16", - "world_size": 2 - } - }, - { - "type": "reduce_scatter", - "args": { - "num_elements": 7372800, - "dtype": "bf16", - "world_size": 2 - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 1280, - "num_experts": 128, - "top_k": 4, - "hidden": 2880, - "dtype": "bf16", - "world_size": 2, - "mode": "low_latency" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 1280, - "num_experts": 128, - "top_k": 4, - "hidden": 2880, - "dtype": "bf16", - "world_size": 2, - "mode": "low_latency" - } - }, - { - "type": "allgather", - "args": { - "num_elements_per_rank": 188743680, - "dtype": "bf16", - "world_size": 4 - } - }, - { - "type": "reduce_scatter", - "args": { - "num_elements": 754974720, - "dtype": "bf16", - "world_size": 4 - } - }, - { - "type": "allgather", - "args": { - "num_elements_per_rank": 184320, - "dtype": "bf16", - "world_size": 4 - } - }, - { - "type": "reduce_scatter", - "args": { - "num_elements": 737280, - "dtype": "bf16", - "world_size": 4 - } - }, - { - "type": "allgather", - "args": { - "num_elements_per_rank": 754974720, - "dtype": "bf16", - "world_size": 4 - } - }, - { - "type": "reduce_scatter", - "args": { - "num_elements": 3019898880, - "dtype": "bf16", - "world_size": 4 - } - }, - { - "type": "allgather", - "args": { - "num_elements_per_rank": 737280, - "dtype": "bf16", - "world_size": 4 - } - }, - { - "type": "reduce_scatter", - "args": { - "num_elements": 2949120, - "dtype": "bf16", - "world_size": 4 - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 256, - "num_experts": 128, - "top_k": 4, - "hidden": 2880, - "dtype": "bf16", - "world_size": 4, - "mode": "low_latency" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 256, - "num_experts": 128, - "top_k": 4, - "hidden": 2880, - "dtype": "bf16", - "world_size": 4, - "mode": "low_latency" - } - }, - { - "type": "allgather", - "args": { - "num_elements_per_rank": 1132462080, - "dtype": "bf16", - "world_size": 4 - } - }, - { - "type": "reduce_scatter", - "args": { - "num_elements": 4529848320, - "dtype": "bf16", - "world_size": 4 - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 393216, - "num_experts": 128, - "top_k": 4, - "hidden": 2880, - "dtype": "bf16", - "world_size": 4, - "mode": "normal" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 393216, - "num_experts": 128, - "top_k": 4, - "hidden": 2880, - "dtype": "bf16", - "world_size": 4, - "mode": "normal" - } - }, - { - "type": "allgather", - "args": { - "num_elements_per_rank": 1105920, - "dtype": "bf16", - "world_size": 4 - } - }, - { - "type": "reduce_scatter", - "args": { - "num_elements": 4423680, - "dtype": "bf16", - "world_size": 4 - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 384, - "num_experts": 128, - "top_k": 4, - "hidden": 2880, - "dtype": "bf16", - "world_size": 4, - "mode": "low_latency" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 384, - "num_experts": 128, - "top_k": 4, - "hidden": 2880, - "dtype": "bf16", - "world_size": 4, - "mode": "low_latency" - } - }, - { - "type": "allgather", - "args": { - "num_elements_per_rank": 2264924160, - "dtype": "bf16", - "world_size": 4 - } - }, - { - "type": "reduce_scatter", - "args": { - "num_elements": 9059696640, - "dtype": "bf16", - "world_size": 4 - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 786432, - "num_experts": 128, - "top_k": 4, - "hidden": 2880, - "dtype": "bf16", - "world_size": 4, - "mode": "normal" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 786432, - "num_experts": 128, - "top_k": 4, - "hidden": 2880, - "dtype": "bf16", - "world_size": 4, - "mode": "normal" - } - }, - { - "type": "allgather", - "args": { - "num_elements_per_rank": 2211840, - "dtype": "bf16", - "world_size": 4 - } - }, - { - "type": "reduce_scatter", - "args": { - "num_elements": 8847360, - "dtype": "bf16", - "world_size": 4 - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 768, - "num_experts": 128, - "top_k": 4, - "hidden": 2880, - "dtype": "bf16", - "world_size": 4, - "mode": "low_latency" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 768, - "num_experts": 128, - "top_k": 4, - "hidden": 2880, - "dtype": "bf16", - "world_size": 4, - "mode": "low_latency" - } - }, - { - "type": "allreduce", - "args": { - "num_elements": 2880, - "dtype": "bf16", - "world_size": 8 - } - }, - { - "type": "allreduce", - "args": { - "num_elements": 3019898880, - "dtype": "bf16", - "world_size": 4 - } - }, - { - "type": "allreduce", - "args": { - "num_elements": 12079595520, - "dtype": "bf16", - "world_size": 4 - } - }, - { - "type": "allreduce", - "args": { - "num_elements": 1474560, - "dtype": "bf16", - "world_size": 4 - } - }, - { - "type": "allreduce", - "args": { - "num_elements": 9059696640, - "dtype": "bf16", - "world_size": 2 - } - }, - { - "type": "allreduce", - "args": { - "num_elements": 1105920, - "dtype": "bf16", - "world_size": 2 - } - }, - { - "type": "allgather", - "args": { - "num_elements_per_rank": 184320, - "dtype": "bf16", - "world_size": 2 - } - }, - { - "type": "reduce_scatter", - "args": { - "num_elements": 368640, - "dtype": "bf16", - "world_size": 2 - } - }, - { - "type": "allgather", - "args": { - "num_elements_per_rank": 6039797760, - "dtype": "bf16", - "world_size": 2 - } - }, - { - "type": "reduce_scatter", - "args": { - "num_elements": 12079595520, - "dtype": "bf16", - "world_size": 2 - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 2097152, - "num_experts": 128, - "top_k": 4, - "hidden": 2880, - "dtype": "bf16", - "world_size": 2, - "mode": "normal" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 2097152, - "num_experts": 128, - "top_k": 4, - "hidden": 2880, - "dtype": "bf16", - "world_size": 2, - "mode": "normal" - } - }, - { - "type": "allgather", - "args": { - "num_elements_per_rank": 1879048192, - "dtype": "bf16", - "world_size": 4 - } - }, - { - "type": "reduce_scatter", - "args": { - "num_elements": 7516192768, - "dtype": "bf16", - "world_size": 4 - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 4194304, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 4, - "mode": "normal" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 4194304, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 4, - "mode": "normal" - } - }, - { - "type": "allgather", - "args": { - "num_elements_per_rank": 143360, - "dtype": "bf16", - "world_size": 8 - } - }, - { - "type": "reduce_scatter", - "args": { - "num_elements": 1146880, - "dtype": "bf16", - "world_size": 8 - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 163840, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 8, - "mode": "normal" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 163840, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 8, - "mode": "normal" - } - }, - { - "type": "allgather", - "args": { - "num_elements_per_rank": 286720, - "dtype": "bf16", - "world_size": 8 - } - }, - { - "type": "reduce_scatter", - "args": { - "num_elements": 2293760, - "dtype": "bf16", - "world_size": 8 - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 1024, - "num_experts": 384, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 4, - "mode": "normal" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 1024, - "num_experts": 384, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 4, - "mode": "normal" - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 1, - "num_experts": 384, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 4, - "mode": "low_latency" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 1, - "num_experts": 384, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 4, - "mode": "low_latency" - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 49152, - "num_experts": 384, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 4, - "mode": "normal" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 49152, - "num_experts": 384, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 4, - "mode": "normal" - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 48, - "num_experts": 384, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 4, - "mode": "low_latency" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 48, - "num_experts": 384, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 4, - "mode": "low_latency" - } - }, - { - "type": "allgather", - "args": { - "num_elements_per_rank": 660602880, - "dtype": "bf16", - "world_size": 4 - } - }, - { - "type": "reduce_scatter", - "args": { - "num_elements": 2642411520, - "dtype": "bf16", - "world_size": 4 - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 92160, - "num_experts": 384, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 4, - "mode": "normal" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 92160, - "num_experts": 384, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 4, - "mode": "normal" - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 90, - "num_experts": 384, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 4, - "mode": "low_latency" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 90, - "num_experts": 384, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 4, - "mode": "low_latency" - } - }, - { - "type": "allgather", - "args": { - "num_elements_per_rank": 1225785344, - "dtype": "bf16", - "world_size": 4 - } - }, - { - "type": "reduce_scatter", - "args": { - "num_elements": 4903141376, - "dtype": "bf16", - "world_size": 4 - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 171008, - "num_experts": 384, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 4, - "mode": "normal" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 171008, - "num_experts": 384, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 4, - "mode": "normal" - } - }, - { - "type": "allgather", - "args": { - "num_elements_per_rank": 1197056, - "dtype": "bf16", - "world_size": 4 - } - }, - { - "type": "reduce_scatter", - "args": { - "num_elements": 4788224, - "dtype": "bf16", - "world_size": 4 - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 167, - "num_experts": 384, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 4, - "mode": "low_latency" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 167, - "num_experts": 384, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 4, - "mode": "low_latency" - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 4096, - "num_experts": 384, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 8, - "mode": "normal" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 4096, - "num_experts": 384, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 8, - "mode": "normal" - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 4, - "num_experts": 384, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 8, - "mode": "low_latency" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 4, - "num_experts": 384, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 8, - "mode": "low_latency" - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 196608, - "num_experts": 384, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 8, - "mode": "normal" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 196608, - "num_experts": 384, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 8, - "mode": "normal" - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 192, - "num_experts": 384, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 8, - "mode": "low_latency" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 192, - "num_experts": 384, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 8, - "mode": "low_latency" - } - }, - { - "type": "allreduce", - "args": { - "num_elements": 2642411520, - "dtype": "bf16", - "world_size": 8 - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 368640, - "num_experts": 384, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 8, - "mode": "normal" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 368640, - "num_experts": 384, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 8, - "mode": "normal" - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 360, - "num_experts": 384, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 8, - "mode": "low_latency" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 360, - "num_experts": 384, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 8, - "mode": "low_latency" - } - }, - { - "type": "allreduce", - "args": { - "num_elements": 4903141376, - "dtype": "bf16", - "world_size": 8 - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 684032, - "num_experts": 384, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 8, - "mode": "normal" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 684032, - "num_experts": 384, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 8, - "mode": "normal" - } - }, - { - "type": "allreduce", - "args": { - "num_elements": 4788224, - "dtype": "bf16", - "world_size": 8 - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 668, - "num_experts": 384, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 8, - "mode": "low_latency" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 668, - "num_experts": 384, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 8, - "mode": "low_latency" - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 1280, - "num_experts": 384, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 4, - "mode": "normal" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 1280, - "num_experts": 384, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 4, - "mode": "normal" - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 3840, - "num_experts": 384, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 4, - "mode": "normal" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 3840, - "num_experts": 384, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 4, - "mode": "normal" - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 3, - "num_experts": 384, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 4, - "mode": "low_latency" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 3, - "num_experts": 384, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 4, - "mode": "low_latency" - } - }, - { - "type": "allgather", - "args": { - "num_elements_per_rank": 55050240, - "dtype": "bf16", - "world_size": 4 - } - }, - { - "type": "reduce_scatter", - "args": { - "num_elements": 220200960, - "dtype": "bf16", - "world_size": 4 - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 7680, - "num_experts": 384, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 4, - "mode": "normal" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 7680, - "num_experts": 384, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 4, - "mode": "normal" - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 7, - "num_experts": 384, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 4, - "mode": "low_latency" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 7, - "num_experts": 384, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 4, - "mode": "low_latency" - } - }, - { - "type": "allgather", - "args": { - "num_elements_per_rank": 100925440, - "dtype": "bf16", - "world_size": 4 - } - }, - { - "type": "reduce_scatter", - "args": { - "num_elements": 403701760, - "dtype": "bf16", - "world_size": 4 - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 14080, - "num_experts": 384, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 4, - "mode": "normal" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 14080, - "num_experts": 384, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 4, - "mode": "normal" - } - }, - { - "type": "allgather", - "args": { - "num_elements_per_rank": 93184, - "dtype": "bf16", - "world_size": 4 - } - }, - { - "type": "reduce_scatter", - "args": { - "num_elements": 372736, - "dtype": "bf16", - "world_size": 4 - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 13, - "num_experts": 384, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 4, - "mode": "low_latency" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 13, - "num_experts": 384, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 4, - "mode": "low_latency" - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 5120, - "num_experts": 384, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 4, - "mode": "normal" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 5120, - "num_experts": 384, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 4, - "mode": "normal" - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 5, - "num_experts": 384, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 4, - "mode": "low_latency" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 5, - "num_experts": 384, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 4, - "mode": "low_latency" - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 15360, - "num_experts": 384, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 4, - "mode": "normal" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 15360, - "num_experts": 384, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 4, - "mode": "normal" - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 15, - "num_experts": 384, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 4, - "mode": "low_latency" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 15, - "num_experts": 384, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 4, - "mode": "low_latency" - } - }, - { - "type": "allreduce", - "args": { - "num_elements": 220200960, - "dtype": "bf16", - "world_size": 4 - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 30720, - "num_experts": 384, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 4, - "mode": "normal" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 30720, - "num_experts": 384, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 4, - "mode": "normal" - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 30, - "num_experts": 384, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 4, - "mode": "low_latency" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 30, - "num_experts": 384, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 4, - "mode": "low_latency" - } - }, - { - "type": "allreduce", - "args": { - "num_elements": 403701760, - "dtype": "bf16", - "world_size": 4 - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 56320, - "num_experts": 384, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 4, - "mode": "normal" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 56320, - "num_experts": 384, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 4, - "mode": "normal" - } - }, - { - "type": "allreduce", - "args": { - "num_elements": 394240, - "dtype": "bf16", - "world_size": 4 - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 55, - "num_experts": 384, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 4, - "mode": "low_latency" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 55, - "num_experts": 384, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 4, - "mode": "low_latency" - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 170496, - "num_experts": 384, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 4, - "mode": "normal" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 170496, - "num_experts": 384, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 4, - "mode": "normal" - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 166, - "num_experts": 384, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 4, - "mode": "low_latency" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 166, - "num_experts": 384, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 4, - "mode": "low_latency" - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 42624, - "num_experts": 384, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 16, - "mode": "normal" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 42624, - "num_experts": 384, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 16, - "mode": "normal" - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 41, - "num_experts": 384, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 16, - "mode": "low_latency" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 41, - "num_experts": 384, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 16, - "mode": "low_latency" - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 576768, - "num_experts": 384, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 4, - "mode": "normal" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 576768, - "num_experts": 384, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 4, - "mode": "normal" - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 563, - "num_experts": 384, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 4, - "mode": "low_latency" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 563, - "num_experts": 384, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 4, - "mode": "low_latency" - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 72096, - "num_experts": 384, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 32, - "mode": "normal" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 72096, - "num_experts": 384, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 32, - "mode": "normal" - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 70, - "num_experts": 384, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 32, - "mode": "low_latency" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 70, - "num_experts": 384, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 32, - "mode": "low_latency" - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 1101056, - "num_experts": 384, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 4, - "mode": "normal" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 1101056, - "num_experts": 384, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 4, - "mode": "normal" - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 1075, - "num_experts": 384, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 4, - "mode": "low_latency" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 1075, - "num_experts": 384, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 4, - "mode": "low_latency" - } - }, - { - "type": "allgather", - "args": { - "num_elements_per_rank": 11839471616, - "dtype": "bf16", - "world_size": 4 - } - }, - { - "type": "reduce_scatter", - "args": { - "num_elements": 47357886464, - "dtype": "bf16", - "world_size": 4 - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 1651712, - "num_experts": 384, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 4, - "mode": "normal" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 1651712, - "num_experts": 384, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 4, - "mode": "normal" - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 1613, - "num_experts": 384, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 4, - "mode": "low_latency" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 1613, - "num_experts": 384, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 4, - "mode": "low_latency" - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 550528, - "num_experts": 384, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 8, - "mode": "normal" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 550528, - "num_experts": 384, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 8, - "mode": "normal" - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 537, - "num_experts": 384, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 8, - "mode": "low_latency" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 537, - "num_experts": 384, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 8, - "mode": "low_latency" - } - }, - { - "type": "allgather", - "args": { - "num_elements_per_rank": 5919735808, - "dtype": "bf16", - "world_size": 8 - } - }, - { - "type": "reduce_scatter", - "args": { - "num_elements": 47357886464, - "dtype": "bf16", - "world_size": 8 - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 825856, - "num_experts": 384, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 8, - "mode": "normal" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 825856, - "num_experts": 384, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 8, - "mode": "normal" - } - }, - { - "type": "allgather", - "args": { - "num_elements_per_rank": 5777408, - "dtype": "bf16", - "world_size": 8 - } - }, - { - "type": "reduce_scatter", - "args": { - "num_elements": 46219264, - "dtype": "bf16", - "world_size": 8 - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 806, - "num_experts": 384, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 8, - "mode": "low_latency" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 806, - "num_experts": 384, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 8, - "mode": "low_latency" - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 275264, - "num_experts": 384, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 16, - "mode": "normal" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 275264, - "num_experts": 384, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 16, - "mode": "normal" - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 268, - "num_experts": 384, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 16, - "mode": "low_latency" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 268, - "num_experts": 384, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 16, - "mode": "low_latency" - } - }, - { - "type": "allgather", - "args": { - "num_elements_per_rank": 986546176, - "dtype": "bf16", - "world_size": 32 - } - }, - { - "type": "reduce_scatter", - "args": { - "num_elements": 31569477632, - "dtype": "bf16", - "world_size": 32 - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 137632, - "num_experts": 384, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 32, - "mode": "normal" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 137632, - "num_experts": 384, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 32, - "mode": "normal" - } - }, - { - "type": "allgather", - "args": { - "num_elements_per_rank": 960512, - "dtype": "bf16", - "world_size": 32 - } - }, - { - "type": "reduce_scatter", - "args": { - "num_elements": 30736384, - "dtype": "bf16", - "world_size": 32 - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 134, - "num_experts": 384, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 32, - "mode": "low_latency" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 134, - "num_experts": 384, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 32, - "mode": "low_latency" - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 8192, - "num_experts": 384, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 4, - "mode": "normal" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 8192, - "num_experts": 384, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 4, - "mode": "normal" - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 32768, - "num_experts": 384, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 8, - "mode": "normal" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 32768, - "num_experts": 384, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 8, - "mode": "normal" - } - }, - { - "type": "allgather", - "args": { - "num_elements_per_rank": 2290089984, - "dtype": "bf16", - "world_size": 4 - } - }, - { - "type": "reduce_scatter", - "args": { - "num_elements": 9160359936, - "dtype": "bf16", - "world_size": 4 - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 319488, - "num_experts": 384, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 4, - "mode": "normal" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 319488, - "num_experts": 384, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 4, - "mode": "normal" - } - }, - { - "type": "allgather", - "args": { - "num_elements_per_rank": 279552, - "dtype": "bf16", - "world_size": 4 - } - }, - { - "type": "reduce_scatter", - "args": { - "num_elements": 1118208, - "dtype": "bf16", - "world_size": 4 - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 39, - "num_experts": 384, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 4, - "mode": "low_latency" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 39, - "num_experts": 384, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 4, - "mode": "low_latency" - } - }, - { - "type": "allreduce", - "args": { - "num_elements": 9160359936, - "dtype": "bf16", - "world_size": 4 - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 1277952, - "num_experts": 384, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 4, - "mode": "normal" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 1277952, - "num_experts": 384, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 4, - "mode": "normal" - } - }, - { - "type": "allreduce", - "args": { - "num_elements": 1118208, - "dtype": "bf16", - "world_size": 4 - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 156, - "num_experts": 384, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 4, - "mode": "low_latency" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 156, - "num_experts": 384, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 4, - "mode": "low_latency" - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 10240, - "num_experts": 384, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 4, - "mode": "normal" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 10240, - "num_experts": 384, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 4, - "mode": "normal" - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 61440, - "num_experts": 384, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 4, - "mode": "normal" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 61440, - "num_experts": 384, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 4, - "mode": "normal" - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 122880, - "num_experts": 384, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 4, - "mode": "normal" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 122880, - "num_experts": 384, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 4, - "mode": "normal" - } - }, - { - "type": "allgather", - "args": { - "num_elements_per_rank": 1541406720, - "dtype": "bf16", - "world_size": 4 - } - }, - { - "type": "reduce_scatter", - "args": { - "num_elements": 6165626880, - "dtype": "bf16", - "world_size": 4 - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 215040, - "num_experts": 384, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 4, - "mode": "normal" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 215040, - "num_experts": 384, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 4, - "mode": "normal" - } - }, - { - "type": "allgather", - "args": { - "num_elements_per_rank": 186368, - "dtype": "bf16", - "world_size": 4 - } - }, - { - "type": "reduce_scatter", - "args": { - "num_elements": 745472, - "dtype": "bf16", - "world_size": 4 - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 26, - "num_experts": 384, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 4, - "mode": "low_latency" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 26, - "num_experts": 384, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 4, - "mode": "low_latency" - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 40960, - "num_experts": 384, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 4, - "mode": "normal" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 40960, - "num_experts": 384, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 4, - "mode": "normal" - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 245760, - "num_experts": 384, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 4, - "mode": "normal" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 245760, - "num_experts": 384, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 4, - "mode": "normal" - } - }, - { - "type": "allreduce", - "args": { - "num_elements": 3523215360, - "dtype": "bf16", - "world_size": 4 - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 491520, - "num_experts": 384, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 4, - "mode": "normal" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 491520, - "num_experts": 384, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 4, - "mode": "normal" - } - }, - { - "type": "allreduce", - "args": { - "num_elements": 430080, - "dtype": "bf16", - "world_size": 4 - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 60, - "num_experts": 384, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 4, - "mode": "low_latency" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 60, - "num_experts": 384, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 4, - "mode": "low_latency" - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 860160, - "num_experts": 384, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 4, - "mode": "normal" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 860160, - "num_experts": 384, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 4, - "mode": "normal" - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 105, - "num_experts": 384, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 4, - "mode": "low_latency" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 105, - "num_experts": 384, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 4, - "mode": "low_latency" - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 681984, - "num_experts": 384, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 4, - "mode": "normal" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 681984, - "num_experts": 384, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 4, - "mode": "normal" - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 83, - "num_experts": 384, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 4, - "mode": "low_latency" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 83, - "num_experts": 384, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 4, - "mode": "low_latency" - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 170496, - "num_experts": 384, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 16, - "mode": "normal" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 170496, - "num_experts": 384, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 16, - "mode": "normal" - } - }, - { - "type": "allgather", - "args": { - "num_elements_per_rank": 143360, - "dtype": "bf16", - "world_size": 16 - } - }, - { - "type": "reduce_scatter", - "args": { - "num_elements": 2293760, - "dtype": "bf16", - "world_size": 16 - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 20, - "num_experts": 384, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 16, - "mode": "low_latency" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 20, - "num_experts": 384, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 16, - "mode": "low_latency" - } - }, - { - "type": "allgather", - "args": { - "num_elements_per_rank": 9028239360, - "dtype": "bf16", - "world_size": 4 - } - }, - { - "type": "reduce_scatter", - "args": { - "num_elements": 36112957440, - "dtype": "bf16", - "world_size": 4 - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 1259520, - "num_experts": 384, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 4, - "mode": "normal" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 1259520, - "num_experts": 384, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 4, - "mode": "normal" - } - }, - { - "type": "allgather", - "args": { - "num_elements_per_rank": 1096704, - "dtype": "bf16", - "world_size": 4 - } - }, - { - "type": "reduce_scatter", - "args": { - "num_elements": 4386816, - "dtype": "bf16", - "world_size": 4 - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 153, - "num_experts": 384, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 4, - "mode": "low_latency" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 153, - "num_experts": 384, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 4, - "mode": "low_latency" - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 314880, - "num_experts": 384, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 16, - "mode": "normal" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 314880, - "num_experts": 384, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 16, - "mode": "normal" - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 38, - "num_experts": 384, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 16, - "mode": "low_latency" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 38, - "num_experts": 384, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 16, - "mode": "low_latency" - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 4405248, - "num_experts": 384, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 4, - "mode": "normal" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 4405248, - "num_experts": 384, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 4, - "mode": "normal" - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 537, - "num_experts": 384, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 4, - "mode": "low_latency" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 537, - "num_experts": 384, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 4, - "mode": "low_latency" - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 2202624, - "num_experts": 384, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 8, - "mode": "normal" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 2202624, - "num_experts": 384, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 8, - "mode": "normal" - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 268, - "num_experts": 384, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 8, - "mode": "low_latency" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 268, - "num_experts": 384, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 8, - "mode": "low_latency" - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 4614144, - "num_experts": 384, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 4, - "mode": "normal" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 4614144, - "num_experts": 384, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 4, - "mode": "normal" - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 1153536, - "num_experts": 384, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 16, - "mode": "normal" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 1153536, - "num_experts": 384, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 16, - "mode": "normal" - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 140, - "num_experts": 384, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 16, - "mode": "low_latency" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 140, - "num_experts": 384, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 16, - "mode": "low_latency" - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 65536, - "num_experts": 384, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 4, - "mode": "normal" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 65536, - "num_experts": 384, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 4, - "mode": "normal" - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 64, - "num_experts": 384, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 4, - "mode": "low_latency" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 64, - "num_experts": 384, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 4, - "mode": "low_latency" - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 131072, - "num_experts": 384, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 4, - "mode": "normal" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 131072, - "num_experts": 384, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 4, - "mode": "normal" - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 128, - "num_experts": 384, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 4, - "mode": "low_latency" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 128, - "num_experts": 384, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 4, - "mode": "low_latency" - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 262144, - "num_experts": 384, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 4, - "mode": "normal" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 262144, - "num_experts": 384, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 4, - "mode": "normal" - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 256, - "num_experts": 384, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 4, - "mode": "low_latency" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 256, - "num_experts": 384, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 4, - "mode": "low_latency" - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 524288, - "num_experts": 384, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 4, - "mode": "normal" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 524288, - "num_experts": 384, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 4, - "mode": "normal" - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 512, - "num_experts": 384, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 4, - "mode": "low_latency" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 512, - "num_experts": 384, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 4, - "mode": "low_latency" - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 786432, - "num_experts": 384, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 4, - "mode": "normal" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 786432, - "num_experts": 384, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 4, - "mode": "normal" - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 768, - "num_experts": 384, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 4, - "mode": "low_latency" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 768, - "num_experts": 384, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 4, - "mode": "low_latency" - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 1048576, - "num_experts": 384, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 4, - "mode": "normal" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 1048576, - "num_experts": 384, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 4, - "mode": "normal" - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 1024, - "num_experts": 384, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 4, - "mode": "low_latency" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 1024, - "num_experts": 384, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 4, - "mode": "low_latency" - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 16384, - "num_experts": 384, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 16, - "mode": "normal" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 16384, - "num_experts": 384, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 16, - "mode": "normal" - } - }, - { - "type": "allgather", - "args": { - "num_elements_per_rank": 114688, - "dtype": "bf16", - "world_size": 16 - } - }, - { - "type": "reduce_scatter", - "args": { - "num_elements": 1835008, - "dtype": "bf16", - "world_size": 16 - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 16, - "num_experts": 384, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 16, - "mode": "low_latency" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 16, - "num_experts": 384, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 16, - "mode": "low_latency" - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 32768, - "num_experts": 384, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 16, - "mode": "normal" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 32768, - "num_experts": 384, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 16, - "mode": "normal" - } - }, - { - "type": "allgather", - "args": { - "num_elements_per_rank": 229376, - "dtype": "bf16", - "world_size": 16 - } - }, - { - "type": "reduce_scatter", - "args": { - "num_elements": 3670016, - "dtype": "bf16", - "world_size": 16 - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 32, - "num_experts": 384, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 16, - "mode": "low_latency" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 32, - "num_experts": 384, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 16, - "mode": "low_latency" - } - }, - { - "type": "allgather", - "args": { - "num_elements_per_rank": 469762048, - "dtype": "bf16", - "world_size": 16 - } - }, - { - "type": "reduce_scatter", - "args": { - "num_elements": 7516192768, - "dtype": "bf16", - "world_size": 16 - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 65536, - "num_experts": 384, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 16, - "mode": "normal" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 65536, - "num_experts": 384, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 16, - "mode": "normal" - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 64, - "num_experts": 384, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 16, - "mode": "low_latency" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 64, - "num_experts": 384, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 16, - "mode": "low_latency" - } - }, - { - "type": "allgather", - "args": { - "num_elements_per_rank": 939524096, - "dtype": "bf16", - "world_size": 16 - } - }, - { - "type": "reduce_scatter", - "args": { - "num_elements": 15032385536, - "dtype": "bf16", - "world_size": 16 - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 131072, - "num_experts": 384, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 16, - "mode": "normal" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 131072, - "num_experts": 384, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 16, - "mode": "normal" - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 128, - "num_experts": 384, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 16, - "mode": "low_latency" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 128, - "num_experts": 384, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 16, - "mode": "low_latency" - } - }, - { - "type": "allgather", - "args": { - "num_elements_per_rank": 1409286144, - "dtype": "bf16", - "world_size": 16 - } - }, - { - "type": "reduce_scatter", - "args": { - "num_elements": 22548578304, - "dtype": "bf16", - "world_size": 16 - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 196608, - "num_experts": 384, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 16, - "mode": "normal" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 196608, - "num_experts": 384, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 16, - "mode": "normal" - } - }, - { - "type": "allgather", - "args": { - "num_elements_per_rank": 1376256, - "dtype": "bf16", - "world_size": 16 - } - }, - { - "type": "reduce_scatter", - "args": { - "num_elements": 22020096, - "dtype": "bf16", - "world_size": 16 - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 192, - "num_experts": 384, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 16, - "mode": "low_latency" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 192, - "num_experts": 384, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 16, - "mode": "low_latency" - } - }, - { - "type": "allgather", - "args": { - "num_elements_per_rank": 1879048192, - "dtype": "bf16", - "world_size": 16 - } - }, - { - "type": "reduce_scatter", - "args": { - "num_elements": 30064771072, - "dtype": "bf16", - "world_size": 16 - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 262144, - "num_experts": 384, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 16, - "mode": "normal" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 262144, - "num_experts": 384, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 16, - "mode": "normal" - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 256, - "num_experts": 384, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 16, - "mode": "low_latency" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 256, - "num_experts": 384, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 16, - "mode": "low_latency" - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 2048, - "num_experts": 384, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 4, - "mode": "normal" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 2048, - "num_experts": 384, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 4, - "mode": "normal" - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 2, - "num_experts": 384, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 4, - "mode": "low_latency" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 2, - "num_experts": 384, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 4, - "mode": "low_latency" - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 4096, - "num_experts": 384, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 4, - "mode": "normal" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 4096, - "num_experts": 384, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 4, - "mode": "normal" - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 4, - "num_experts": 384, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 4, - "mode": "low_latency" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 4, - "num_experts": 384, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 4, - "mode": "low_latency" - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 8, - "num_experts": 384, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 4, - "mode": "low_latency" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 8, - "num_experts": 384, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 4, - "mode": "low_latency" - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 16384, - "num_experts": 384, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 4, - "mode": "normal" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 16384, - "num_experts": 384, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 4, - "mode": "normal" - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 16, - "num_experts": 384, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 4, - "mode": "low_latency" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 16, - "num_experts": 384, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 4, - "mode": "low_latency" - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 32768, - "num_experts": 384, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 4, - "mode": "normal" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 32768, - "num_experts": 384, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 4, - "mode": "normal" - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 32, - "num_experts": 384, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 4, - "mode": "low_latency" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 32, - "num_experts": 384, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 4, - "mode": "low_latency" - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 2097152, - "num_experts": 384, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 4, - "mode": "normal" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 2097152, - "num_experts": 384, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 4, - "mode": "normal" - } - }, - { - "type": "allgather", - "args": { - "num_elements_per_rank": 3758096384, - "dtype": "bf16", - "world_size": 16 - } - }, - { - "type": "reduce_scatter", - "args": { - "num_elements": 60129542144, - "dtype": "bf16", - "world_size": 16 - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 524288, - "num_experts": 384, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 16, - "mode": "normal" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 524288, - "num_experts": 384, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 16, - "mode": "normal" - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 4194304, - "num_experts": 384, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 4, - "mode": "normal" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 4194304, - "num_experts": 384, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 4, - "mode": "normal" - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 2097152, - "num_experts": 384, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 8, - "mode": "normal" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 2097152, - "num_experts": 384, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 8, - "mode": "normal" - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 256, - "num_experts": 384, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 8, - "mode": "low_latency" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 256, - "num_experts": 384, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 8, - "mode": "low_latency" - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 6291456, - "num_experts": 384, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 4, - "mode": "normal" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 6291456, - "num_experts": 384, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 4, - "mode": "normal" - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 8388608, - "num_experts": 384, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 4, - "mode": "normal" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 8388608, - "num_experts": 384, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 4, - "mode": "normal" - } - }, - { - "type": "allgather", - "args": { - "num_elements_per_rank": 11274289152, - "dtype": "bf16", - "world_size": 16 - } - }, - { - "type": "reduce_scatter", - "args": { - "num_elements": 180388626432, - "dtype": "bf16", - "world_size": 16 - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 1572864, - "num_experts": 384, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 16, - "mode": "normal" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 1572864, - "num_experts": 384, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 16, - "mode": "normal" - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 2097152, - "num_experts": 384, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 16, - "mode": "normal" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 2097152, - "num_experts": 384, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 16, - "mode": "normal" - } - }, - { - "type": "allgather", - "args": { - "num_elements_per_rank": 60129542144, - "dtype": "bf16", - "world_size": 8 - } - }, - { - "type": "reduce_scatter", - "args": { - "num_elements": 481036337152, - "dtype": "bf16", - "world_size": 8 - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 8388608, - "num_experts": 384, - "top_k": 6, - "hidden": 7168, - "dtype": "bf16", - "world_size": 8, - "mode": "normal" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 8388608, - "num_experts": 384, - "top_k": 6, - "hidden": 7168, - "dtype": "bf16", - "world_size": 8, - "mode": "normal" - } - }, - { - "type": "allgather", - "args": { - "num_elements_per_rank": 90612695040, - "dtype": "bf16", - "world_size": 8 - } - }, - { - "type": "reduce_scatter", - "args": { - "num_elements": 724901560320, - "dtype": "bf16", - "world_size": 8 - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 12641280, - "num_experts": 384, - "top_k": 6, - "hidden": 7168, - "dtype": "bf16", - "world_size": 8, - "mode": "normal" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 12641280, - "num_experts": 384, - "top_k": 6, - "hidden": 7168, - "dtype": "bf16", - "world_size": 8, - "mode": "normal" - } - }, - { - "type": "allgather", - "args": { - "num_elements_per_rank": 11060224, - "dtype": "bf16", - "world_size": 8 - } - }, - { - "type": "reduce_scatter", - "args": { - "num_elements": 88481792, - "dtype": "bf16", - "world_size": 8 - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 1543, - "num_experts": 384, - "top_k": 6, - "hidden": 7168, - "dtype": "bf16", - "world_size": 8, - "mode": "low_latency" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 1543, - "num_experts": 384, - "top_k": 6, - "hidden": 7168, - "dtype": "bf16", - "world_size": 8, - "mode": "low_latency" - } - }, - { - "type": "allgather", - "args": { - "num_elements_per_rank": 2818572288, - "dtype": "bf16", - "world_size": 4 - } - }, - { - "type": "reduce_scatter", - "args": { - "num_elements": 11274289152, - "dtype": "bf16", - "world_size": 4 - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 393216, - "num_experts": 384, - "top_k": 6, - "hidden": 7168, - "dtype": "bf16", - "world_size": 4, - "mode": "normal" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 393216, - "num_experts": 384, - "top_k": 6, - "hidden": 7168, - "dtype": "bf16", - "world_size": 4, - "mode": "normal" - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 48, - "num_experts": 384, - "top_k": 6, - "hidden": 7168, - "dtype": "bf16", - "world_size": 4, - "mode": "low_latency" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 48, - "num_experts": 384, - "top_k": 6, - "hidden": 7168, - "dtype": "bf16", - "world_size": 4, - "mode": "low_latency" - } - }, - { - "type": "allreduce", - "args": { - "num_elements": 11274289152, - "dtype": "bf16", - "world_size": 4 - } - }, - { - "type": "allreduce", - "args": { - "num_elements": 1376256, - "dtype": "bf16", - "world_size": 4 - } - }, - { - "type": "allreduce", - "args": { - "num_elements": 1056964608, - "dtype": "bf16", - "world_size": 4 - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 147456, - "num_experts": 384, - "top_k": 6, - "hidden": 7168, - "dtype": "bf16", - "world_size": 4, - "mode": "normal" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 147456, - "num_experts": 384, - "top_k": 6, - "hidden": 7168, - "dtype": "bf16", - "world_size": 4, - "mode": "normal" - } - }, - { - "type": "allreduce", - "args": { - "num_elements": 129024, - "dtype": "bf16", - "world_size": 4 - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 18, - "num_experts": 384, - "top_k": 6, - "hidden": 7168, - "dtype": "bf16", - "world_size": 4, - "mode": "low_latency" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 18, - "num_experts": 384, - "top_k": 6, - "hidden": 7168, - "dtype": "bf16", - "world_size": 4, - "mode": "low_latency" - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 8388608, - "num_experts": 384, - "top_k": 6, - "hidden": 7168, - "dtype": "bf16", - "world_size": 4, - "mode": "normal" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 8388608, - "num_experts": 384, - "top_k": 6, - "hidden": 7168, - "dtype": "bf16", - "world_size": 4, - "mode": "normal" - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 1024, - "num_experts": 384, - "top_k": 6, - "hidden": 7168, - "dtype": "bf16", - "world_size": 4, - "mode": "low_latency" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 1024, - "num_experts": 384, - "top_k": 6, - "hidden": 7168, - "dtype": "bf16", - "world_size": 4, - "mode": "low_latency" - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 6291456, - "num_experts": 384, - "top_k": 6, - "hidden": 7168, - "dtype": "bf16", - "world_size": 4, - "mode": "normal" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 6291456, - "num_experts": 384, - "top_k": 6, - "hidden": 7168, - "dtype": "bf16", - "world_size": 4, - "mode": "normal" - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 768, - "num_experts": 384, - "top_k": 6, - "hidden": 7168, - "dtype": "bf16", - "world_size": 4, - "mode": "low_latency" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 768, - "num_experts": 384, - "top_k": 6, - "hidden": 7168, - "dtype": "bf16", - "world_size": 4, - "mode": "low_latency" - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 1572864, - "num_experts": 384, - "top_k": 6, - "hidden": 7168, - "dtype": "bf16", - "world_size": 16, - "mode": "normal" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 1572864, - "num_experts": 384, - "top_k": 6, - "hidden": 7168, - "dtype": "bf16", - "world_size": 16, - "mode": "normal" - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 192, - "num_experts": 384, - "top_k": 6, - "hidden": 7168, - "dtype": "bf16", - "world_size": 16, - "mode": "low_latency" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 192, - "num_experts": 384, - "top_k": 6, - "hidden": 7168, - "dtype": "bf16", - "world_size": 16, - "mode": "low_latency" - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 524288, - "num_experts": 384, - "top_k": 6, - "hidden": 7168, - "dtype": "bf16", - "world_size": 16, - "mode": "normal" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 524288, - "num_experts": 384, - "top_k": 6, - "hidden": 7168, - "dtype": "bf16", - "world_size": 16, - "mode": "normal" - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 64, - "num_experts": 384, - "top_k": 6, - "hidden": 7168, - "dtype": "bf16", - "world_size": 16, - "mode": "low_latency" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 64, - "num_experts": 384, - "top_k": 6, - "hidden": 7168, - "dtype": "bf16", - "world_size": 16, - "mode": "low_latency" - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 2097152, - "num_experts": 384, - "top_k": 6, - "hidden": 7168, - "dtype": "bf16", - "world_size": 16, - "mode": "normal" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 2097152, - "num_experts": 384, - "top_k": 6, - "hidden": 7168, - "dtype": "bf16", - "world_size": 16, - "mode": "normal" - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 256, - "num_experts": 384, - "top_k": 6, - "hidden": 7168, - "dtype": "bf16", - "world_size": 16, - "mode": "low_latency" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 256, - "num_experts": 384, - "top_k": 6, - "hidden": 7168, - "dtype": "bf16", - "world_size": 16, - "mode": "low_latency" - } - }, - { - "type": "allgather", - "args": { - "num_elements_per_rank": 120259084288, - "dtype": "bf16", - "world_size": 4 - } - }, - { - "type": "reduce_scatter", - "args": { - "num_elements": 481036337152, - "dtype": "bf16", - "world_size": 4 - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 16777216, - "num_experts": 384, - "top_k": 6, - "hidden": 7168, - "dtype": "bf16", - "world_size": 4, - "mode": "normal" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 16777216, - "num_experts": 384, - "top_k": 6, - "hidden": 7168, - "dtype": "bf16", - "world_size": 4, - "mode": "normal" - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 2048, - "num_experts": 384, - "top_k": 6, - "hidden": 7168, - "dtype": "bf16", - "world_size": 4, - "mode": "low_latency" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 2048, - "num_experts": 384, - "top_k": 6, - "hidden": 7168, - "dtype": "bf16", - "world_size": 4, - "mode": "low_latency" - } - }, - { - "type": "allgather", - "args": { - "num_elements_per_rank": 30064771072, - "dtype": "bf16", - "world_size": 16 - } - }, - { - "type": "reduce_scatter", - "args": { - "num_elements": 481036337152, - "dtype": "bf16", - "world_size": 16 - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 4194304, - "num_experts": 384, - "top_k": 6, - "hidden": 7168, - "dtype": "bf16", - "world_size": 16, - "mode": "normal" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 4194304, - "num_experts": 384, - "top_k": 6, - "hidden": 7168, - "dtype": "bf16", - "world_size": 16, - "mode": "normal" - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 512, - "num_experts": 384, - "top_k": 6, - "hidden": 7168, - "dtype": "bf16", - "world_size": 16, - "mode": "low_latency" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 512, - "num_experts": 384, - "top_k": 6, - "hidden": 7168, - "dtype": "bf16", - "world_size": 16, - "mode": "low_latency" - } - }, - { - "type": "allgather", - "args": { - "num_elements_per_rank": 315680096256, - "dtype": "bf16", - "world_size": 4 - } - }, - { - "type": "reduce_scatter", - "args": { - "num_elements": 1262720385024, - "dtype": "bf16", - "world_size": 4 - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 44040192, - "num_experts": 384, - "top_k": 6, - "hidden": 7168, - "dtype": "bf16", - "world_size": 4, - "mode": "normal" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 44040192, - "num_experts": 384, - "top_k": 6, - "hidden": 7168, - "dtype": "bf16", - "world_size": 4, - "mode": "normal" - } - }, - { - "type": "allgather", - "args": { - "num_elements_per_rank": 38535168, - "dtype": "bf16", - "world_size": 4 - } - }, - { - "type": "reduce_scatter", - "args": { - "num_elements": 154140672, - "dtype": "bf16", - "world_size": 4 - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 5376, - "num_experts": 384, - "top_k": 6, - "hidden": 7168, - "dtype": "bf16", - "world_size": 4, - "mode": "low_latency" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 5376, - "num_experts": 384, - "top_k": 6, - "hidden": 7168, - "dtype": "bf16", - "world_size": 4, - "mode": "low_latency" - } - }, - { - "type": "allgather", - "args": { - "num_elements_per_rank": 105226698752, - "dtype": "bf16", - "world_size": 12 - } - }, - { - "type": "reduce_scatter", - "args": { - "num_elements": 1262720385024, - "dtype": "bf16", - "world_size": 12 - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 14680064, - "num_experts": 384, - "top_k": 6, - "hidden": 7168, - "dtype": "bf16", - "world_size": 12, - "mode": "normal" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 14680064, - "num_experts": 384, - "top_k": 6, - "hidden": 7168, - "dtype": "bf16", - "world_size": 12, - "mode": "normal" - } - }, - { - "type": "allgather", - "args": { - "num_elements_per_rank": 12845056, - "dtype": "bf16", - "world_size": 12 - } - }, - { - "type": "reduce_scatter", - "args": { - "num_elements": 154140672, - "dtype": "bf16", - "world_size": 12 - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 1792, - "num_experts": 384, - "top_k": 6, - "hidden": 7168, - "dtype": "bf16", - "world_size": 12, - "mode": "low_latency" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 1792, - "num_experts": 384, - "top_k": 6, - "hidden": 7168, - "dtype": "bf16", - "world_size": 12, - "mode": "low_latency" - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 131072, - "num_experts": 384, - "top_k": 6, - "hidden": 7168, - "dtype": "bf16", - "world_size": 16, - "mode": "normal" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 131072, - "num_experts": 384, - "top_k": 6, - "hidden": 7168, - "dtype": "bf16", - "world_size": 16, - "mode": "normal" - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 32, - "num_experts": 384, - "top_k": 6, - "hidden": 7168, - "dtype": "bf16", - "world_size": 16, - "mode": "low_latency" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 32, - "num_experts": 384, - "top_k": 6, - "hidden": 7168, - "dtype": "bf16", - "world_size": 16, - "mode": "low_latency" - } - }, - { - "type": "allreduce", - "args": { - "num_elements": 8589934592, - "dtype": "bf16", - "world_size": 8 - } - }, - { - "type": "allreduce", - "args": { - "num_elements": 2097152, - "dtype": "bf16", - "world_size": 8 - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 512, - "num_experts": 512, - "top_k": 10, - "hidden": 4096, - "dtype": "bf16", - "world_size": 8, - "mode": "low_latency" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 512, - "num_experts": 512, - "top_k": 10, - "hidden": 4096, - "dtype": "bf16", - "world_size": 8, - "mode": "low_latency" - } - }, - { - "type": "allreduce", - "args": { - "num_elements": 2097152, - "dtype": "bf16", - "world_size": 2 - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 512, - "num_experts": 512, - "top_k": 10, - "hidden": 4096, - "dtype": "bf16", - "world_size": 2, - "mode": "low_latency" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 512, - "num_experts": 512, - "top_k": 10, - "hidden": 4096, - "dtype": "bf16", - "world_size": 2, - "mode": "low_latency" - } - }, - { - "type": "allreduce", - "args": { - "num_elements": 8589934592, - "dtype": "bf16", - "world_size": 2 - } - }, - { - "type": "allreduce", - "args": { - "num_elements": 25165824, - "dtype": "bf16", - "world_size": 2 - } - }, - { - "type": "allreduce", - "args": { - "num_elements": 6291456, - "dtype": "bf16", - "world_size": 2 - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 2048, - "num_experts": 256, - "top_k": 8, - "hidden": 3072, - "dtype": "bf16", - "world_size": 2, - "mode": "normal" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 2048, - "num_experts": 256, - "top_k": 8, - "hidden": 3072, - "dtype": "bf16", - "world_size": 2, - "mode": "normal" - } - }, - { - "type": "allreduce", - "args": { - "num_elements": 6144, - "dtype": "bf16", - "world_size": 2 - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 2, - "num_experts": 256, - "top_k": 8, - "hidden": 3072, - "dtype": "bf16", - "world_size": 2, - "mode": "low_latency" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 2, - "num_experts": 256, - "top_k": 8, - "hidden": 3072, - "dtype": "bf16", - "world_size": 2, - "mode": "low_latency" - } - }, - { - "type": "allreduce", - "args": { - "num_elements": 12582912, - "dtype": "bf16", - "world_size": 2 - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 4096, - "num_experts": 256, - "top_k": 8, - "hidden": 3072, - "dtype": "bf16", - "world_size": 2, - "mode": "normal" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 4096, - "num_experts": 256, - "top_k": 8, - "hidden": 3072, - "dtype": "bf16", - "world_size": 2, - "mode": "normal" - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 4, - "num_experts": 256, - "top_k": 8, - "hidden": 3072, - "dtype": "bf16", - "world_size": 2, - "mode": "low_latency" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 4, - "num_experts": 256, - "top_k": 8, - "hidden": 3072, - "dtype": "bf16", - "world_size": 2, - "mode": "low_latency" - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 8192, - "num_experts": 256, - "top_k": 8, - "hidden": 3072, - "dtype": "bf16", - "world_size": 2, - "mode": "normal" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 8192, - "num_experts": 256, - "top_k": 8, - "hidden": 3072, - "dtype": "bf16", - "world_size": 2, - "mode": "normal" - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 8, - "num_experts": 256, - "top_k": 8, - "hidden": 3072, - "dtype": "bf16", - "world_size": 2, - "mode": "low_latency" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 8, - "num_experts": 256, - "top_k": 8, - "hidden": 3072, - "dtype": "bf16", - "world_size": 2, - "mode": "low_latency" - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 16384, - "num_experts": 256, - "top_k": 8, - "hidden": 3072, - "dtype": "bf16", - "world_size": 2, - "mode": "normal" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 16384, - "num_experts": 256, - "top_k": 8, - "hidden": 3072, - "dtype": "bf16", - "world_size": 2, - "mode": "normal" - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 16, - "num_experts": 256, - "top_k": 8, - "hidden": 3072, - "dtype": "bf16", - "world_size": 2, - "mode": "low_latency" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 16, - "num_experts": 256, - "top_k": 8, - "hidden": 3072, - "dtype": "bf16", - "world_size": 2, - "mode": "low_latency" - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 32768, - "num_experts": 256, - "top_k": 8, - "hidden": 3072, - "dtype": "bf16", - "world_size": 2, - "mode": "normal" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 32768, - "num_experts": 256, - "top_k": 8, - "hidden": 3072, - "dtype": "bf16", - "world_size": 2, - "mode": "normal" - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 32, - "num_experts": 256, - "top_k": 8, - "hidden": 3072, - "dtype": "bf16", - "world_size": 2, - "mode": "low_latency" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 32, - "num_experts": 256, - "top_k": 8, - "hidden": 3072, - "dtype": "bf16", - "world_size": 2, - "mode": "low_latency" - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 65536, - "num_experts": 256, - "top_k": 8, - "hidden": 3072, - "dtype": "bf16", - "world_size": 2, - "mode": "normal" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 65536, - "num_experts": 256, - "top_k": 8, - "hidden": 3072, - "dtype": "bf16", - "world_size": 2, - "mode": "normal" - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 64, - "num_experts": 256, - "top_k": 8, - "hidden": 3072, - "dtype": "bf16", - "world_size": 2, - "mode": "low_latency" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 64, - "num_experts": 256, - "top_k": 8, - "hidden": 3072, - "dtype": "bf16", - "world_size": 2, - "mode": "low_latency" - } - }, - { - "type": "allreduce", - "args": { - "num_elements": 6291456, - "dtype": "bf16", - "world_size": 8 - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 2048, - "num_experts": 256, - "top_k": 8, - "hidden": 3072, - "dtype": "bf16", - "world_size": 8, - "mode": "normal" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 2048, - "num_experts": 256, - "top_k": 8, - "hidden": 3072, - "dtype": "bf16", - "world_size": 8, - "mode": "normal" - } - }, - { - "type": "allreduce", - "args": { - "num_elements": 6144, - "dtype": "bf16", - "world_size": 8 - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 2, - "num_experts": 256, - "top_k": 8, - "hidden": 3072, - "dtype": "bf16", - "world_size": 8, - "mode": "low_latency" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 2, - "num_experts": 256, - "top_k": 8, - "hidden": 3072, - "dtype": "bf16", - "world_size": 8, - "mode": "low_latency" - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 1048576, - "num_experts": 256, - "top_k": 8, - "hidden": 3072, - "dtype": "bf16", - "world_size": 4, - "mode": "normal" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 1048576, - "num_experts": 256, - "top_k": 8, - "hidden": 3072, - "dtype": "bf16", - "world_size": 4, - "mode": "normal" - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 2097152, - "num_experts": 256, - "top_k": 8, - "hidden": 3072, - "dtype": "bf16", - "world_size": 4, - "mode": "normal" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 2097152, - "num_experts": 256, - "top_k": 8, - "hidden": 3072, - "dtype": "bf16", - "world_size": 4, - "mode": "normal" - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 4194304, - "num_experts": 256, - "top_k": 8, - "hidden": 3072, - "dtype": "bf16", - "world_size": 4, - "mode": "normal" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 4194304, - "num_experts": 256, - "top_k": 8, - "hidden": 3072, - "dtype": "bf16", - "world_size": 4, - "mode": "normal" - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 16384, - "num_experts": 256, - "top_k": 8, - "hidden": 3072, - "dtype": "bf16", - "world_size": 8, - "mode": "normal" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 16384, - "num_experts": 256, - "top_k": 8, - "hidden": 3072, - "dtype": "bf16", - "world_size": 8, - "mode": "normal" - } - }, - { - "type": "allreduce", - "args": { - "num_elements": 25769803776, - "dtype": "bf16", - "world_size": 2 - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 4096, - "num_experts": 256, - "top_k": 8, - "hidden": 3072, - "dtype": "bf16", - "world_size": 8, - "mode": "normal" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 4096, - "num_experts": 256, - "top_k": 8, - "hidden": 3072, - "dtype": "bf16", - "world_size": 8, - "mode": "normal" - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 4, - "num_experts": 256, - "top_k": 8, - "hidden": 3072, - "dtype": "bf16", - "world_size": 8, - "mode": "low_latency" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 4, - "num_experts": 256, - "top_k": 8, - "hidden": 3072, - "dtype": "bf16", - "world_size": 8, - "mode": "low_latency" - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 8192, - "num_experts": 256, - "top_k": 8, - "hidden": 3072, - "dtype": "bf16", - "world_size": 8, - "mode": "normal" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 8192, - "num_experts": 256, - "top_k": 8, - "hidden": 3072, - "dtype": "bf16", - "world_size": 8, - "mode": "normal" - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 8, - "num_experts": 256, - "top_k": 8, - "hidden": 3072, - "dtype": "bf16", - "world_size": 8, - "mode": "low_latency" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 8, - "num_experts": 256, - "top_k": 8, - "hidden": 3072, - "dtype": "bf16", - "world_size": 8, - "mode": "low_latency" - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 16, - "num_experts": 256, - "top_k": 8, - "hidden": 3072, - "dtype": "bf16", - "world_size": 8, - "mode": "low_latency" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 16, - "num_experts": 256, - "top_k": 8, - "hidden": 3072, - "dtype": "bf16", - "world_size": 8, - "mode": "low_latency" - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 32768, - "num_experts": 256, - "top_k": 8, - "hidden": 3072, - "dtype": "bf16", - "world_size": 8, - "mode": "normal" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 32768, - "num_experts": 256, - "top_k": 8, - "hidden": 3072, - "dtype": "bf16", - "world_size": 8, - "mode": "normal" - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 32, - "num_experts": 256, - "top_k": 8, - "hidden": 3072, - "dtype": "bf16", - "world_size": 8, - "mode": "low_latency" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 32, - "num_experts": 256, - "top_k": 8, - "hidden": 3072, - "dtype": "bf16", - "world_size": 8, - "mode": "low_latency" - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 65536, - "num_experts": 256, - "top_k": 8, - "hidden": 3072, - "dtype": "bf16", - "world_size": 8, - "mode": "normal" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 65536, - "num_experts": 256, - "top_k": 8, - "hidden": 3072, - "dtype": "bf16", - "world_size": 8, - "mode": "normal" - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 64, - "num_experts": 256, - "top_k": 8, - "hidden": 3072, - "dtype": "bf16", - "world_size": 8, - "mode": "low_latency" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 64, - "num_experts": 256, - "top_k": 8, - "hidden": 3072, - "dtype": "bf16", - "world_size": 8, - "mode": "low_latency" - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 131072, - "num_experts": 256, - "top_k": 8, - "hidden": 3072, - "dtype": "bf16", - "world_size": 8, - "mode": "normal" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 131072, - "num_experts": 256, - "top_k": 8, - "hidden": 3072, - "dtype": "bf16", - "world_size": 8, - "mode": "normal" - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 128, - "num_experts": 256, - "top_k": 8, - "hidden": 3072, - "dtype": "bf16", - "world_size": 8, - "mode": "low_latency" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 128, - "num_experts": 256, - "top_k": 8, - "hidden": 3072, - "dtype": "bf16", - "world_size": 8, - "mode": "low_latency" - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 262144, - "num_experts": 256, - "top_k": 8, - "hidden": 3072, - "dtype": "bf16", - "world_size": 8, - "mode": "normal" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 262144, - "num_experts": 256, - "top_k": 8, - "hidden": 3072, - "dtype": "bf16", - "world_size": 8, - "mode": "normal" - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 256, - "num_experts": 256, - "top_k": 8, - "hidden": 3072, - "dtype": "bf16", - "world_size": 8, - "mode": "low_latency" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 256, - "num_experts": 256, - "top_k": 8, - "hidden": 3072, - "dtype": "bf16", - "world_size": 8, - "mode": "low_latency" - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 524288, - "num_experts": 256, - "top_k": 8, - "hidden": 3072, - "dtype": "bf16", - "world_size": 8, - "mode": "normal" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 524288, - "num_experts": 256, - "top_k": 8, - "hidden": 3072, - "dtype": "bf16", - "world_size": 8, - "mode": "normal" - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 512, - "num_experts": 256, - "top_k": 8, - "hidden": 3072, - "dtype": "bf16", - "world_size": 8, - "mode": "low_latency" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 512, - "num_experts": 256, - "top_k": 8, - "hidden": 3072, - "dtype": "bf16", - "world_size": 8, - "mode": "low_latency" - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 1048576, - "num_experts": 256, - "top_k": 8, - "hidden": 3072, - "dtype": "bf16", - "world_size": 8, - "mode": "normal" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 1048576, - "num_experts": 256, - "top_k": 8, - "hidden": 3072, - "dtype": "bf16", - "world_size": 8, - "mode": "normal" - } - }, - { - "type": "dispatch", - "args": { - "num_tokens": 2097152, - "num_experts": 256, - "top_k": 8, - "hidden": 3072, - "dtype": "bf16", - "world_size": 8, - "mode": "normal" - } - }, - { - "type": "combine", - "args": { - "num_tokens": 2097152, - "num_experts": 256, - "top_k": 8, - "hidden": 3072, - "dtype": "bf16", - "world_size": 8, - "mode": "normal" - } - }, - { - "type": "allreduce", - "args": { - "num_elements": 2949120, - "dtype": "bf16", - "world_size": 8 - } - }, - { - "type": "allreduce", - "args": { - "num_elements": 5898240, - "dtype": "bf16", - "world_size": 8 - } - }, - { - "type": "allreduce", - "args": { - "num_elements": 5760, - "dtype": "bf16", - "world_size": 8 - } - }, - { - "type": "allreduce", - "args": { - "num_elements": 11274289152, - "dtype": "bf16", - "world_size": 8 - } - }, - { - "type": "allreduce", - "args": { - "num_elements": 11010048, - "dtype": "bf16", - "world_size": 8 - } - }, - { - "type": "allgather", - "args": { - "num_elements_per_rank": 1409286144, - "dtype": "bf16", - "world_size": 8 - } - }, - { - "type": "reduce_scatter", - "args": { - "num_elements": 11274289152, - "dtype": "bf16", - "world_size": 8 - } - }, - { - "type": "allgather", - "args": { - "num_elements_per_rank": 1376256, - "dtype": "bf16", - "world_size": 8 - } - }, - { - "type": "reduce_scatter", - "args": { - "num_elements": 11010048, - "dtype": "bf16", - "world_size": 8 - } - } -] \ No newline at end of file diff --git a/operatorx/testlists/gemm.json b/operatorx/testlists/gemm.json index a6bfe97dc7..5bd9c0b249 100644 --- a/operatorx/testlists/gemm.json +++ b/operatorx/testlists/gemm.json @@ -2,56542 +2,117829 @@ { "type": "gemm", "args": { - "m": 8192, - "n": 8192, - "k": 8192, - "dtype_a": "bf16", - "dtype_b": "bf16", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 8192, - "n": 8192, - "k": 8192, - "dtype_a": "fp16", - "dtype_b": "fp16", - "dtype_out": "fp16" - } + "m": 1, + "n": 64, + "k": 2880, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/gpt-oss-120b-w-mxfp4-a-fp8/k_proj", + "amd/gpt-oss-120b-w-mxfp4-a-fp8/v_proj", + "openai/gpt-oss-120b/k_proj", + "openai/gpt-oss-120b/v_proj" + ] }, { "type": "gemm", "args": { - "m": 8192, - "n": 8192, - "k": 8192, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 1, + "n": 128, + "k": 2880, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/gpt-oss-120b-w-mxfp4-a-fp8/k_proj", + "amd/gpt-oss-120b-w-mxfp4-a-fp8/v_proj", + "openai/gpt-oss-120b/k_proj", + "openai/gpt-oss-120b/v_proj" + ] }, { "type": "gemm", "args": { - "m": 8192, - "n": 8192, - "k": 8192, - "dtype_a": "fp4", - "dtype_b": "fp4", - "dtype_out": "bf16" - } + "m": 1, + "n": 512, + "k": 2880, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/gpt-oss-120b-w-mxfp4-a-fp8/k_proj", + "amd/gpt-oss-120b-w-mxfp4-a-fp8/q_proj", + "amd/gpt-oss-120b-w-mxfp4-a-fp8/v_proj", + "openai/gpt-oss-120b/k_proj", + "openai/gpt-oss-120b/q_proj", + "openai/gpt-oss-120b/v_proj" + ] }, { "type": "gemm", "args": { - "m": 8192, - "n": 8192, - "k": 8192, - "dtype_a": "mxfp4", - "dtype_b": "mxfp4", - "dtype_out": "bf16" - } + "m": 1, + "n": 576, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/Kimi-K2.5-MXFP4/kv_a_proj_with_mqa", + "moonshotai/Kimi-K2.5/kv_a_proj_with_mqa", + "nvidia/DeepSeek-R1-0528-FP4-V2/kv_a_proj_with_mqa", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/kv_a_proj_with_mqa", + "nvidia/Kimi-K2.5-NVFP4/kv_a_proj_with_mqa", + "nvidia/Kimi-K2.6-NVFP4/kv_a_proj_with_mqa" + ] }, { "type": "gemm", "args": { - "m": 8192, - "n": 8192, - "k": 8192, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 1, + "n": 576, + "k": 7168, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 32 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 1, + 32 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "amd/DeepSeek-R1-0528-MXFP4-Preview/kv_a_proj_with_mqa", + "amd/DeepSeek-R1-0528-MXFP4-v2/kv_a_proj_with_mqa", + "amd/DeepSeek-R1-0528-MXFP4/kv_a_proj_with_mqa" + ] }, { "type": "gemm", "args": { - "m": 310784, - "n": 1536, + "m": 1, + "n": 576, "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/kv_a_proj_with_mqa" + ] }, { "type": "gemm", "args": { - "m": 310784, - "n": 24576, - "k": 1536, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 1, + "n": 1024, + "k": 2880, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/gpt-oss-120b-w-mxfp4-a-fp8/q_proj", + "openai/gpt-oss-120b/q_proj" + ] }, { "type": "gemm", "args": { - "m": 310784, - "n": 576, + "m": 1, + "n": 1088, "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] }, { "type": "gemm", "args": { - "m": 310784, - "n": 32768, - "k": 512, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 1, + "n": 1152, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/gate_up_proj" + ] }, { "type": "gemm", "args": { - "m": 310784, - "n": 7168, - "k": 16384, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 1, + "n": 1536, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/Kimi-K2.5-MXFP4/q_a_proj", + "moonshotai/Kimi-K2.5/q_a_proj", + "nvidia/DeepSeek-R1-0528-FP4-V2/q_a_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/q_a_proj", + "nvidia/Kimi-K2.5-NVFP4/q_a_proj", + "nvidia/Kimi-K2.6-NVFP4/q_a_proj" + ] }, { "type": "gemm", "args": { - "m": 310784, - "n": 4608, + "m": 1, + "n": 1536, "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 32 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 1, + 32 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "amd/DeepSeek-R1-0528-MXFP4-Preview/q_a_proj", + "amd/DeepSeek-R1-0528-MXFP4-v2/q_a_proj", + "amd/DeepSeek-R1-0528-MXFP4/q_a_proj" + ] }, { "type": "gemm", "args": { - "m": 310784, - "n": 7168, - "k": 4608, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 1, + "n": 1536, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/gate_up_proj", + "deepseek-ai/DeepSeek-R1-0528/q_a_proj", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w3" + ] }, { "type": "gemm", "args": { - "m": 607, + "m": 1, "n": 1536, "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w3" + ] }, { "type": "gemm", "args": { - "m": 607, - "n": 24576, - "k": 1536, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 1, + "n": 2304, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/gate_up_proj" + ] }, { "type": "gemm", "args": { - "m": 607, - "n": 576, - "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 1, + "n": 2880, + "k": 512, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/gpt-oss-120b-w-mxfp4-a-fp8/o_proj", + "openai/gpt-oss-120b/o_proj" + ] }, { "type": "gemm", "args": { - "m": 607, - "n": 32768, - "k": 512, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 1, + "n": 2880, + "k": 1024, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/gpt-oss-120b-w-mxfp4-a-fp8/o_proj", + "openai/gpt-oss-120b/o_proj" + ] }, { "type": "gemm", "args": { - "m": 607, - "n": 7168, - "k": 16384, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 1, + "n": 2880, + "k": 4096, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/gpt-oss-120b-w-mxfp4-a-fp8/o_proj", + "openai/gpt-oss-120b/o_proj" + ] }, { "type": "gemm", "args": { - "m": 607, - "n": 4608, - "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 1, + "n": 3072, + "k": 1536, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/Kimi-K2.5-MXFP4/q_b_proj", + "amd/MiniMax-M2.5-MXFP4/o_proj", + "moonshotai/Kimi-K2.5/q_b_proj", + "nvidia/DeepSeek-R1-0528-FP4-V2/q_b_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/q_b_proj", + "nvidia/Kimi-K2.5-NVFP4/q_b_proj", + "nvidia/Kimi-K2.6-NVFP4/q_b_proj", + "nvidia/MiniMax-M2.5-NVFP4/o_proj" + ] }, { "type": "gemm", "args": { - "m": 607, - "n": 7168, - "k": 4608, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 1, + "n": 3072, + "k": 1536, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 32 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 1, + 32 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "amd/DeepSeek-R1-0528-MXFP4-Preview/q_b_proj", + "amd/DeepSeek-R1-0528-MXFP4-v2/q_b_proj", + "amd/DeepSeek-R1-0528-MXFP4/q_b_proj", + "amd/MiniMax-M2.5-MXFP4/experts.down_proj" + ] }, { "type": "gemm", "args": { - "m": 155392, - "n": 1536, - "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 1, + "n": 3072, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "MiniMaxAI/MiniMax-M2.5/experts.down_proj", + "MiniMaxAI/MiniMax-M2.5/o_proj", + "deepseek-ai/DeepSeek-R1-0528/q_b_proj" + ] }, { "type": "gemm", "args": { - "m": 155392, - "n": 24576, - "k": 1536, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 1, + "n": 4096, + "k": 512, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "Qwen/Qwen3.5-397B-A17B/o_proj", + "Qwen/Qwen3.5-397B-A17B/shared_expert.down_proj", + "amd/Kimi-K2.5-MXFP4/kv_b_proj", + "amd/Qwen3.5-397B-A17B-MXFP4/o_proj", + "amd/Qwen3.5-397B-A17B-MXFP4/shared_expert.down_proj", + "moonshotai/Kimi-K2.5/kv_b_proj", + "nvidia/DeepSeek-R1-0528-FP4-V2/kv_b_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/kv_b_proj", + "nvidia/Kimi-K2.5-NVFP4/kv_b_proj", + "nvidia/Kimi-K2.6-NVFP4/kv_b_proj", + "nvidia/Qwen3.5-397B-A17B-NVFP4/o_proj", + "nvidia/Qwen3.5-397B-A17B-NVFP4/shared_expert.down_proj" + ] }, { "type": "gemm", "args": { - "m": 155392, - "n": 576, - "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 1, + "n": 4096, + "k": 512, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 32 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 1, + 32 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "amd/DeepSeek-R1-0528-MXFP4-Preview/kv_b_proj", + "amd/DeepSeek-R1-0528-MXFP4-v2/kv_b_proj", + "amd/DeepSeek-R1-0528-MXFP4/kv_b_proj", + "amd/Qwen3.5-397B-A17B-MXFP4-AttnFP8-V2/shared_expert.down_proj" + ] }, { "type": "gemm", "args": { - "m": 155392, - "n": 32768, + "m": 1, + "n": 4096, "k": 512, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "Qwen/Qwen3.5-397B-A17B-FP8/o_proj", + "Qwen/Qwen3.5-397B-A17B-FP8/shared_expert.down_proj", + "deepseek-ai/DeepSeek-R1-0528/kv_b_proj" + ] }, { "type": "gemm", "args": { - "m": 155392, - "n": 7168, - "k": 16384, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 1, + "n": 4096, + "k": 2880, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/gpt-oss-120b-w-mxfp4-a-fp8/q_proj", + "openai/gpt-oss-120b/q_proj" + ] }, { "type": "gemm", "args": { - "m": 155392, - "n": 2304, + "m": 1, + "n": 4608, "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/gate_up_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/gate_up_proj", + "nvidia/Kimi-K2.5-NVFP4/gate_up_proj" + ] }, { "type": "gemm", "args": { - "m": 155392, - "n": 7168, - "k": 2304, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 1, + "n": 4608, + "k": 7168, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 32 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 1, + 32 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "amd/DeepSeek-R1-0528-MXFP4-Preview/gate_up_proj", + "amd/DeepSeek-R1-0528-MXFP4-v2/gate_up_proj", + "amd/DeepSeek-R1-0528-MXFP4/gate_up_proj", + "amd/Kimi-K2.5-MXFP4/gate_up_proj" + ] }, { "type": "gemm", "args": { - "m": 303, - "n": 1536, + "m": 1, + "n": 4608, "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/gate_up_proj" + ] }, { "type": "gemm", "args": { - "m": 303, - "n": 24576, + "m": 1, + "n": 6144, "k": 1536, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/GLM-5.1-MXFP4/down_proj", + "amd/GLM-5.2-MXFP4/down_proj", + "amd/Kimi-K2.5-MXFP4/q_b_proj", + "amd/MiniMax-M3-MXFP4/down_proj", + "moonshotai/Kimi-K2.5/q_b_proj", + "nvidia/DeepSeek-R1-0528-FP4-V2/q_b_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/q_b_proj", + "nvidia/GLM-5.1-NVFP4/down_proj", + "nvidia/GLM-5.2-NVFP4/down_proj", + "nvidia/Kimi-K2.5-NVFP4/q_b_proj", + "nvidia/Kimi-K2.6-NVFP4/q_b_proj" + ] }, { "type": "gemm", "args": { - "m": 303, - "n": 576, - "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 1, + "n": 7168, + "k": 1152, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/down_proj" + ] }, { "type": "gemm", "args": { - "m": 303, - "n": 32768, - "k": 512, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 1, + "n": 7168, + "k": 2048, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/Kimi-K2.5-MXFP4/o_proj", + "moonshotai/Kimi-K2.5/o_proj", + "nvidia/Kimi-K2.5-NVFP4/o_proj", + "nvidia/Kimi-K2.6-NVFP4/o_proj" + ] }, { "type": "gemm", "args": { - "m": 303, + "m": 1, "n": 7168, - "k": 16384, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "k": 2048, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/o_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/o_proj" + ] }, { "type": "gemm", "args": { - "m": 303, - "n": 2304, - "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 1, + "n": 7168, + "k": 2048, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 32 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 1, + 32 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "amd/DeepSeek-R1-0528-MXFP4-Preview/o_proj", + "amd/DeepSeek-R1-0528-MXFP4-v2/o_proj", + "amd/DeepSeek-R1-0528-MXFP4/o_proj" + ] }, { "type": "gemm", "args": { - "m": 303, + "m": 1, "n": 7168, - "k": 2304, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "k": 2048, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/o_proj", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] }, { "type": "gemm", "args": { - "m": 224000, - "n": 1536, - "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 1, + "n": 7168, + "k": 2304, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/down_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/down_proj", + "nvidia/Kimi-K2.5-NVFP4/down_proj" + ] }, { "type": "gemm", "args": { - "m": 224000, - "n": 24576, - "k": 1536, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 1, + "n": 7168, + "k": 2304, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 32 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 1, + 32 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "amd/DeepSeek-R1-0528-MXFP4-Preview/down_proj", + "amd/DeepSeek-R1-0528-MXFP4-v2/down_proj", + "amd/DeepSeek-R1-0528-MXFP4/down_proj", + "amd/Kimi-K2.5-MXFP4/down_proj" + ] }, { "type": "gemm", "args": { - "m": 224000, - "n": 576, - "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 1, + "n": 7168, + "k": 2304, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/down_proj" + ] }, { "type": "gemm", "args": { - "m": 224000, - "n": 32768, - "k": 512, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 1, + "n": 7168, + "k": 4096, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/o_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/o_proj" + ] }, { "type": "gemm", "args": { - "m": 224000, + "m": 1, "n": 7168, - "k": 16384, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "k": 4608, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/down_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/down_proj", + "nvidia/Kimi-K2.5-NVFP4/down_proj" + ] }, { "type": "gemm", "args": { - "m": 224000, - "n": 4608, - "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 1, + "n": 7168, + "k": 8192, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] }, { "type": "gemm", "args": { - "m": 224000, + "m": 1, "n": 7168, - "k": 4608, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "k": 16384, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/o_proj", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] }, { "type": "gemm", "args": { - "m": 437, - "n": 1536, - "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 1, + "n": 7168, + "k": 16384, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] }, { "type": "gemm", "args": { - "m": 437, - "n": 24576, - "k": 1536, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 1, + "n": 7168, + "k": 65536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] }, { "type": "gemm", "args": { - "m": 437, - "n": 576, - "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 1, + "n": 8192, + "k": 512, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/Kimi-K2.5-MXFP4/kv_b_proj", + "moonshotai/Kimi-K2.5/kv_b_proj", + "nvidia/DeepSeek-R1-0528-FP4-V2/kv_b_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/kv_b_proj", + "nvidia/Kimi-K2.5-NVFP4/kv_b_proj", + "nvidia/Kimi-K2.6-NVFP4/kv_b_proj" + ] }, { "type": "gemm", "args": { - "m": 437, - "n": 32768, - "k": 512, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 1, + "n": 8192, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_b" + ] }, { "type": "gemm", "args": { - "m": 437, - "n": 7168, - "k": 16384, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 1, + "n": 12288, + "k": 1536, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/Kimi-K2.5-MXFP4/q_b_proj", + "moonshotai/Kimi-K2.5/q_b_proj", + "nvidia/DeepSeek-R1-0528-FP4-V2/q_b_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/q_b_proj", + "nvidia/Kimi-K2.5-NVFP4/q_b_proj", + "nvidia/Kimi-K2.6-NVFP4/q_b_proj" + ] }, { "type": "gemm", "args": { - "m": 437, - "n": 4608, - "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 1, + "n": 15360, + "k": 1024, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] }, { "type": "gemm", "args": { - "m": 437, - "n": 7168, - "k": 4608, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 1, + "n": 16384, + "k": 512, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/Kimi-K2.5-MXFP4/kv_b_proj", + "moonshotai/Kimi-K2.5/kv_b_proj", + "nvidia/DeepSeek-R1-0528-FP4-V2/kv_b_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/kv_b_proj", + "nvidia/Kimi-K2.5-NVFP4/kv_b_proj", + "nvidia/Kimi-K2.6-NVFP4/kv_b_proj" + ] }, { "type": "gemm", "args": { - "m": 112000, - "n": 1536, - "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 1, + "n": 16384, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_b" + ] }, { "type": "gemm", "args": { - "m": 112000, + "m": 1, "n": 24576, "k": 1536, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/q_b_proj" + ] }, { "type": "gemm", "args": { - "m": 112000, - "n": 576, - "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 1, + "n": 30720, + "k": 1024, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] }, { "type": "gemm", "args": { - "m": 112000, + "m": 1, "n": 32768, "k": 512, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/kv_b_proj" + ] }, { "type": "gemm", "args": { - "m": 112000, - "n": 7168, - "k": 16384, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 1, + "n": 65536, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_b" + ] }, { "type": "gemm", "args": { - "m": 112000, - "n": 2304, - "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 1, + "n": 122880, + "k": 1024, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] }, { "type": "gemm", "args": { - "m": 112000, - "n": 7168, - "k": 2304, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 2, + "n": 64, + "k": 2880, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/gpt-oss-120b-w-mxfp4-a-fp8/k_proj", + "amd/gpt-oss-120b-w-mxfp4-a-fp8/v_proj", + "openai/gpt-oss-120b/k_proj", + "openai/gpt-oss-120b/v_proj" + ] }, { "type": "gemm", "args": { - "m": 218, - "n": 1536, - "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 2, + "n": 128, + "k": 2880, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/gpt-oss-120b-w-mxfp4-a-fp8/k_proj", + "amd/gpt-oss-120b-w-mxfp4-a-fp8/v_proj", + "openai/gpt-oss-120b/k_proj", + "openai/gpt-oss-120b/v_proj" + ] }, { "type": "gemm", "args": { - "m": 218, - "n": 24576, - "k": 1536, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 2, + "n": 128, + "k": 3072, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "MiniMaxAI/MiniMax-M2.5/k_proj", + "MiniMaxAI/MiniMax-M2.5/v_proj" + ] }, { "type": "gemm", "args": { - "m": 218, - "n": 576, - "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 2, + "n": 192, + "k": 3072, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "MiniMaxAI/MiniMax-M2.5/experts.gate_up_proj" + ] }, { "type": "gemm", "args": { - "m": 218, - "n": 32768, - "k": 512, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 2, + "n": 512, + "k": 2880, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/gpt-oss-120b-w-mxfp4-a-fp8/k_proj", + "amd/gpt-oss-120b-w-mxfp4-a-fp8/q_proj", + "amd/gpt-oss-120b-w-mxfp4-a-fp8/v_proj", + "openai/gpt-oss-120b/k_proj", + "openai/gpt-oss-120b/q_proj", + "openai/gpt-oss-120b/v_proj" + ] }, { "type": "gemm", "args": { - "m": 218, - "n": 7168, - "k": 16384, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 2, + "n": 512, + "k": 3072, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "MiniMaxAI/MiniMax-M2.5/k_proj", + "MiniMaxAI/MiniMax-M2.5/v_proj" + ] }, { "type": "gemm", "args": { - "m": 218, - "n": 2304, + "m": 2, + "n": 576, "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 218, - "n": 7168, - "k": 2304, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/Kimi-K2.5-MXFP4/kv_a_proj_with_mqa", + "moonshotai/Kimi-K2.5/kv_a_proj_with_mqa", + "nvidia/DeepSeek-R1-0528-FP4-V2/kv_a_proj_with_mqa", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/kv_a_proj_with_mqa", + "nvidia/Kimi-K2.5-NVFP4/kv_a_proj_with_mqa", + "nvidia/Kimi-K2.6-NVFP4/kv_a_proj_with_mqa" + ] }, { "type": "gemm", "args": { - "m": 1536, - "n": 1536, + "m": 2, + "n": 576, "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 1536, - "n": 24576, - "k": 1536, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 32 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 1, + 32 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "amd/DeepSeek-R1-0528-MXFP4-Preview/kv_a_proj_with_mqa", + "amd/DeepSeek-R1-0528-MXFP4-v2/kv_a_proj_with_mqa", + "amd/DeepSeek-R1-0528-MXFP4/kv_a_proj_with_mqa" + ] }, { "type": "gemm", "args": { - "m": 1536, + "m": 2, "n": 576, "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/kv_a_proj_with_mqa" + ] }, { "type": "gemm", "args": { - "m": 1536, - "n": 32768, - "k": 512, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 2, + "n": 768, + "k": 3072, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "MiniMaxAI/MiniMax-M2.5/experts.gate_up_proj", + "MiniMaxAI/MiniMax-M2.5/q_proj" + ] }, { "type": "gemm", "args": { - "m": 1536, - "n": 7168, - "k": 16384, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 2, + "n": 1024, + "k": 2880, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/gpt-oss-120b-w-mxfp4-a-fp8/q_proj", + "openai/gpt-oss-120b/q_proj" + ] }, { "type": "gemm", "args": { - "m": 1536, - "n": 4608, + "m": 2, + "n": 1088, "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 1536, - "n": 7168, - "k": 4608, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] }, { "type": "gemm", "args": { - "m": 3, - "n": 1536, + "m": 2, + "n": 1152, "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/gate_up_proj" + ] }, { "type": "gemm", "args": { - "m": 3, - "n": 24576, + "m": 2, + "n": 1536, "k": 1536, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/q_b_proj" + ] }, { "type": "gemm", "args": { - "m": 3, - "n": 576, + "m": 2, + "n": 1536, "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/Kimi-K2.5-MXFP4/q_a_proj", + "moonshotai/Kimi-K2.5/q_a_proj", + "nvidia/DeepSeek-R1-0528-FP4-V2/q_a_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/q_a_proj", + "nvidia/Kimi-K2.5-NVFP4/q_a_proj", + "nvidia/Kimi-K2.6-NVFP4/q_a_proj" + ] }, { "type": "gemm", "args": { - "m": 3, - "n": 32768, - "k": 512, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 2, + "n": 1536, + "k": 7168, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 32 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 1, + 32 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "amd/DeepSeek-R1-0528-MXFP4-Preview/q_a_proj", + "amd/DeepSeek-R1-0528-MXFP4-v2/q_a_proj", + "amd/DeepSeek-R1-0528-MXFP4/q_a_proj" + ] }, { "type": "gemm", "args": { - "m": 3, - "n": 7168, - "k": 16384, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 2, + "n": 1536, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/gate_up_proj", + "deepseek-ai/DeepSeek-R1-0528/q_a_proj", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w3" + ] }, { "type": "gemm", "args": { - "m": 3, - "n": 4608, + "m": 2, + "n": 1536, "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w3" + ] }, { "type": "gemm", "args": { - "m": 3, - "n": 7168, - "k": 4608, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 2, + "n": 2048, + "k": 512, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/kv_b_proj" + ] }, { "type": "gemm", "args": { - "m": 6144, - "n": 1536, + "m": 2, + "n": 2304, "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/gate_up_proj" + ] }, { "type": "gemm", "args": { - "m": 6144, - "n": 3072, - "k": 1536, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 2, + "n": 2880, + "k": 512, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/gpt-oss-120b-w-mxfp4-a-fp8/o_proj", + "openai/gpt-oss-120b/o_proj" + ] }, { "type": "gemm", "args": { - "m": 6144, - "n": 576, - "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 2, + "n": 2880, + "k": 1024, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/gpt-oss-120b-w-mxfp4-a-fp8/o_proj", + "openai/gpt-oss-120b/o_proj" + ] }, { "type": "gemm", "args": { - "m": 6144, - "n": 4096, - "k": 512, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 2, + "n": 2880, + "k": 4096, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/gpt-oss-120b-w-mxfp4-a-fp8/o_proj", + "openai/gpt-oss-120b/o_proj" + ] }, { "type": "gemm", "args": { - "m": 6144, - "n": 7168, - "k": 2048, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 2, + "n": 3072, + "k": 192, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "MiniMaxAI/MiniMax-M2.5/experts.down_proj" + ] }, { "type": "gemm", "args": { - "m": 6144, - "n": 2304, - "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 2, + "n": 3072, + "k": 768, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "MiniMaxAI/MiniMax-M2.5/experts.down_proj", + "MiniMaxAI/MiniMax-M2.5/o_proj" + ] }, { "type": "gemm", "args": { - "m": 6144, - "n": 7168, - "k": 2304, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 2, + "n": 3072, + "k": 1536, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/Kimi-K2.5-MXFP4/q_b_proj", + "amd/MiniMax-M2.5-MXFP4/o_proj", + "moonshotai/Kimi-K2.5/q_b_proj", + "nvidia/DeepSeek-R1-0528-FP4-V2/q_b_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/q_b_proj", + "nvidia/Kimi-K2.5-NVFP4/q_b_proj", + "nvidia/Kimi-K2.6-NVFP4/q_b_proj", + "nvidia/MiniMax-M2.5-NVFP4/o_proj" + ] }, { "type": "gemm", "args": { - "m": 12, - "n": 1536, - "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 2, + "n": 3072, + "k": 1536, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 32 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 1, + 32 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "amd/DeepSeek-R1-0528-MXFP4-Preview/q_b_proj", + "amd/DeepSeek-R1-0528-MXFP4-v2/q_b_proj", + "amd/DeepSeek-R1-0528-MXFP4/q_b_proj", + "amd/MiniMax-M2.5-MXFP4/experts.down_proj" + ] }, { "type": "gemm", "args": { - "m": 12, + "m": 2, "n": 3072, "k": 1536, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "MiniMaxAI/MiniMax-M2.5/experts.down_proj", + "MiniMaxAI/MiniMax-M2.5/o_proj", + "deepseek-ai/DeepSeek-R1-0528/q_b_proj" + ] }, { "type": "gemm", "args": { - "m": 12, - "n": 576, - "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 2, + "n": 3072, + "k": 3072, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "MiniMaxAI/MiniMax-M2.5/experts.gate_up_proj", + "MiniMaxAI/MiniMax-M2.5/o_proj", + "MiniMaxAI/MiniMax-M2.5/q_proj" + ] }, { "type": "gemm", "args": { - "m": 12, + "m": 2, "n": 4096, "k": 512, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "Qwen/Qwen3.5-397B-A17B/o_proj", + "Qwen/Qwen3.5-397B-A17B/shared_expert.down_proj", + "amd/Kimi-K2.5-MXFP4/kv_b_proj", + "amd/Qwen3.5-397B-A17B-MXFP4/o_proj", + "amd/Qwen3.5-397B-A17B-MXFP4/shared_expert.down_proj", + "moonshotai/Kimi-K2.5/kv_b_proj", + "nvidia/DeepSeek-R1-0528-FP4-V2/kv_b_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/kv_b_proj", + "nvidia/Kimi-K2.5-NVFP4/kv_b_proj", + "nvidia/Kimi-K2.6-NVFP4/kv_b_proj", + "nvidia/Qwen3.5-397B-A17B-NVFP4/o_proj", + "nvidia/Qwen3.5-397B-A17B-NVFP4/shared_expert.down_proj" + ] }, { "type": "gemm", "args": { - "m": 12, - "n": 7168, - "k": 2048, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 2, + "n": 4096, + "k": 512, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 32 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 1, + 32 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "amd/DeepSeek-R1-0528-MXFP4-Preview/kv_b_proj", + "amd/DeepSeek-R1-0528-MXFP4-v2/kv_b_proj", + "amd/DeepSeek-R1-0528-MXFP4/kv_b_proj", + "amd/Qwen3.5-397B-A17B-MXFP4-AttnFP8-V2/shared_expert.down_proj" + ] }, { "type": "gemm", "args": { - "m": 12, - "n": 2304, - "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 2, + "n": 4096, + "k": 512, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "Qwen/Qwen3.5-397B-A17B-FP8/o_proj", + "Qwen/Qwen3.5-397B-A17B-FP8/shared_expert.down_proj", + "deepseek-ai/DeepSeek-R1-0528/kv_b_proj" + ] }, { "type": "gemm", "args": { - "m": 12, - "n": 7168, - "k": 2304, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 2, + "n": 4096, + "k": 2880, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/gpt-oss-120b-w-mxfp4-a-fp8/q_proj", + "openai/gpt-oss-120b/q_proj" + ] }, { "type": "gemm", "args": { - "m": 2560, - "n": 1536, + "m": 2, + "n": 4608, "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/gate_up_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/gate_up_proj", + "nvidia/Kimi-K2.5-NVFP4/gate_up_proj" + ] }, { "type": "gemm", "args": { - "m": 2560, - "n": 24576, - "k": 1536, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 2, + "n": 4608, + "k": 7168, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 32 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 1, + 32 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "amd/DeepSeek-R1-0528-MXFP4-Preview/gate_up_proj", + "amd/DeepSeek-R1-0528-MXFP4-v2/gate_up_proj", + "amd/DeepSeek-R1-0528-MXFP4/gate_up_proj", + "amd/Kimi-K2.5-MXFP4/gate_up_proj" + ] }, { "type": "gemm", "args": { - "m": 2560, - "n": 576, + "m": 2, + "n": 4608, "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/gate_up_proj" + ] }, { "type": "gemm", "args": { - "m": 2560, - "n": 32768, - "k": 512, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 2, + "n": 6144, + "k": 1536, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/GLM-5.1-MXFP4/down_proj", + "amd/GLM-5.2-MXFP4/down_proj", + "amd/Kimi-K2.5-MXFP4/q_b_proj", + "amd/MiniMax-M3-MXFP4/down_proj", + "moonshotai/Kimi-K2.5/q_b_proj", + "nvidia/DeepSeek-R1-0528-FP4-V2/q_b_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/q_b_proj", + "nvidia/GLM-5.1-NVFP4/down_proj", + "nvidia/GLM-5.2-NVFP4/down_proj", + "nvidia/Kimi-K2.5-NVFP4/q_b_proj", + "nvidia/Kimi-K2.6-NVFP4/q_b_proj" + ] }, { "type": "gemm", "args": { - "m": 2560, + "m": 2, "n": 7168, - "k": 16384, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "k": 1024, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/o_proj", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] }, { "type": "gemm", "args": { - "m": 2560, - "n": 4608, - "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 2, + "n": 7168, + "k": 1152, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/down_proj" + ] }, { "type": "gemm", "args": { - "m": 2560, + "m": 2, "n": 7168, - "k": 4608, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "k": 2048, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/Kimi-K2.5-MXFP4/o_proj", + "moonshotai/Kimi-K2.5/o_proj", + "nvidia/Kimi-K2.5-NVFP4/o_proj", + "nvidia/Kimi-K2.6-NVFP4/o_proj" + ] }, { "type": "gemm", "args": { - "m": 5, - "n": 1536, - "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 2, + "n": 7168, + "k": 2048, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/o_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/o_proj" + ] }, { "type": "gemm", "args": { - "m": 5, - "n": 24576, - "k": 1536, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 2, + "n": 7168, + "k": 2048, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 32 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 1, + 32 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "amd/DeepSeek-R1-0528-MXFP4-Preview/o_proj", + "amd/DeepSeek-R1-0528-MXFP4-v2/o_proj", + "amd/DeepSeek-R1-0528-MXFP4/o_proj" + ] }, { "type": "gemm", "args": { - "m": 5, - "n": 576, - "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 2, + "n": 7168, + "k": 2048, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/o_proj", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] }, { "type": "gemm", "args": { - "m": 5, - "n": 32768, - "k": 512, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 2, + "n": 7168, + "k": 2304, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/down_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/down_proj", + "nvidia/Kimi-K2.5-NVFP4/down_proj" + ] }, { "type": "gemm", "args": { - "m": 5, + "m": 2, "n": 7168, - "k": 16384, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 5, - "n": 4608, - "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "k": 2304, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 32 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 1, + 32 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "amd/DeepSeek-R1-0528-MXFP4-Preview/down_proj", + "amd/DeepSeek-R1-0528-MXFP4-v2/down_proj", + "amd/DeepSeek-R1-0528-MXFP4/down_proj", + "amd/Kimi-K2.5-MXFP4/down_proj" + ] }, { "type": "gemm", "args": { - "m": 5, + "m": 2, "n": 7168, - "k": 4608, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "k": 2304, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/down_proj" + ] }, { "type": "gemm", "args": { - "m": 3840, - "n": 1536, - "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 2, + "n": 7168, + "k": 4096, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/o_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/o_proj" + ] }, { "type": "gemm", "args": { - "m": 3840, - "n": 24576, - "k": 1536, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 2, + "n": 7168, + "k": 4608, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/down_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/down_proj", + "nvidia/Kimi-K2.5-NVFP4/down_proj" + ] }, { "type": "gemm", "args": { - "m": 3840, - "n": 576, - "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 2, + "n": 7168, + "k": 8192, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] }, { "type": "gemm", "args": { - "m": 3840, - "n": 32768, - "k": 512, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 2, + "n": 7168, + "k": 9216, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/down_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/down_proj", + "nvidia/Kimi-K2.5-NVFP4/down_proj" + ] }, { "type": "gemm", "args": { - "m": 3840, + "m": 2, "n": 7168, "k": 16384, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/o_proj", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] }, { "type": "gemm", "args": { - "m": 3840, - "n": 4608, - "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 2, + "n": 7168, + "k": 16384, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] }, { "type": "gemm", "args": { - "m": 3840, + "m": 2, "n": 7168, - "k": 4608, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "k": 65536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] }, { "type": "gemm", "args": { - "m": 7, - "n": 1536, - "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 2, + "n": 8192, + "k": 512, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/Kimi-K2.5-MXFP4/kv_b_proj", + "moonshotai/Kimi-K2.5/kv_b_proj", + "nvidia/DeepSeek-R1-0528-FP4-V2/kv_b_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/kv_b_proj", + "nvidia/Kimi-K2.5-NVFP4/kv_b_proj", + "nvidia/Kimi-K2.6-NVFP4/kv_b_proj" + ] }, { "type": "gemm", "args": { - "m": 7, - "n": 24576, + "m": 2, + "n": 8192, "k": 1536, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_b" + ] }, { "type": "gemm", "args": { - "m": 7, - "n": 576, + "m": 2, + "n": 9216, "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 7, - "n": 32768, - "k": 512, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/gate_up_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/gate_up_proj", + "nvidia/Kimi-K2.5-NVFP4/gate_up_proj" + ] }, { "type": "gemm", "args": { - "m": 7, - "n": 7168, - "k": 16384, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 2, + "n": 12288, + "k": 1536, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/Kimi-K2.5-MXFP4/q_b_proj", + "moonshotai/Kimi-K2.5/q_b_proj", + "nvidia/DeepSeek-R1-0528-FP4-V2/q_b_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/q_b_proj", + "nvidia/Kimi-K2.5-NVFP4/q_b_proj", + "nvidia/Kimi-K2.6-NVFP4/q_b_proj" + ] }, { "type": "gemm", "args": { - "m": 7, - "n": 4608, - "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 2, + "n": 15360, + "k": 1024, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] }, { "type": "gemm", "args": { - "m": 7, - "n": 7168, - "k": 4608, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 2, + "n": 16384, + "k": 512, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/Kimi-K2.5-MXFP4/kv_b_proj", + "moonshotai/Kimi-K2.5/kv_b_proj", + "nvidia/DeepSeek-R1-0528-FP4-V2/kv_b_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/kv_b_proj", + "nvidia/Kimi-K2.5-NVFP4/kv_b_proj", + "nvidia/Kimi-K2.6-NVFP4/kv_b_proj" + ] }, { "type": "gemm", "args": { - "m": 6400, - "n": 1536, - "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 2, + "n": 16384, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_b" + ] }, { "type": "gemm", "args": { - "m": 6400, + "m": 2, "n": 24576, "k": 1536, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/q_b_proj" + ] }, { "type": "gemm", "args": { - "m": 6400, - "n": 576, - "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 2, + "n": 30720, + "k": 1024, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] }, { "type": "gemm", "args": { - "m": 6400, + "m": 2, "n": 32768, "k": 512, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 6400, - "n": 7168, - "k": 16384, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/kv_b_proj" + ] }, { "type": "gemm", "args": { - "m": 6400, - "n": 4608, - "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 2, + "n": 65536, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_b" + ] }, { "type": "gemm", "args": { - "m": 6400, - "n": 7168, - "k": 4608, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 2, + "n": 122880, + "k": 1024, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] }, { "type": "gemm", "args": { - "m": 12, - "n": 24576, - "k": 1536, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 3, + "n": 576, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/Kimi-K2.5-MXFP4/kv_a_proj_with_mqa", + "moonshotai/Kimi-K2.5/kv_a_proj_with_mqa", + "nvidia/DeepSeek-R1-0528-FP4-V2/kv_a_proj_with_mqa", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/kv_a_proj_with_mqa", + "nvidia/Kimi-K2.5-NVFP4/kv_a_proj_with_mqa", + "nvidia/Kimi-K2.6-NVFP4/kv_a_proj_with_mqa" + ] }, { "type": "gemm", "args": { - "m": 12, - "n": 32768, - "k": 512, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 3, + "n": 576, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/kv_a_proj_with_mqa" + ] }, { "type": "gemm", "args": { - "m": 12, - "n": 7168, - "k": 16384, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 3, + "n": 1152, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/gate_up_proj" + ] }, { "type": "gemm", "args": { - "m": 12, - "n": 4608, + "m": 3, + "n": 1536, "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/Kimi-K2.5-MXFP4/q_a_proj", + "moonshotai/Kimi-K2.5/q_a_proj", + "nvidia/DeepSeek-R1-0528-FP4-V2/q_a_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/q_a_proj", + "nvidia/Kimi-K2.5-NVFP4/q_a_proj", + "nvidia/Kimi-K2.6-NVFP4/q_a_proj" + ] }, { "type": "gemm", "args": { - "m": 12, - "n": 7168, - "k": 4608, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 3, + "n": 1536, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/gate_up_proj", + "deepseek-ai/DeepSeek-R1-0528/q_a_proj", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w3" + ] }, { "type": "gemm", "args": { - "m": 11520, - "n": 1536, + "m": 3, + "n": 2304, "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/gate_up_proj" + ] }, { "type": "gemm", "args": { - "m": 11520, - "n": 24576, + "m": 3, + "n": 3072, "k": 1536, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/Kimi-K2.5-MXFP4/q_b_proj", + "amd/MiniMax-M2.5-MXFP4/o_proj", + "moonshotai/Kimi-K2.5/q_b_proj", + "nvidia/DeepSeek-R1-0528-FP4-V2/q_b_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/q_b_proj", + "nvidia/Kimi-K2.5-NVFP4/q_b_proj", + "nvidia/Kimi-K2.6-NVFP4/q_b_proj", + "nvidia/MiniMax-M2.5-NVFP4/o_proj" + ] }, { "type": "gemm", "args": { - "m": 11520, - "n": 576, - "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 3, + "n": 3072, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "MiniMaxAI/MiniMax-M2.5/experts.down_proj", + "MiniMaxAI/MiniMax-M2.5/o_proj", + "deepseek-ai/DeepSeek-R1-0528/q_b_proj" + ] }, { "type": "gemm", "args": { - "m": 11520, - "n": 32768, + "m": 3, + "n": 4096, "k": 512, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "Qwen/Qwen3.5-397B-A17B/o_proj", + "Qwen/Qwen3.5-397B-A17B/shared_expert.down_proj", + "amd/Kimi-K2.5-MXFP4/kv_b_proj", + "amd/Qwen3.5-397B-A17B-MXFP4/o_proj", + "amd/Qwen3.5-397B-A17B-MXFP4/shared_expert.down_proj", + "moonshotai/Kimi-K2.5/kv_b_proj", + "nvidia/DeepSeek-R1-0528-FP4-V2/kv_b_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/kv_b_proj", + "nvidia/Kimi-K2.5-NVFP4/kv_b_proj", + "nvidia/Kimi-K2.6-NVFP4/kv_b_proj", + "nvidia/Qwen3.5-397B-A17B-NVFP4/o_proj", + "nvidia/Qwen3.5-397B-A17B-NVFP4/shared_expert.down_proj" + ] }, { "type": "gemm", "args": { - "m": 11520, - "n": 7168, - "k": 16384, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 3, + "n": 4096, + "k": 512, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "Qwen/Qwen3.5-397B-A17B-FP8/o_proj", + "Qwen/Qwen3.5-397B-A17B-FP8/shared_expert.down_proj", + "deepseek-ai/DeepSeek-R1-0528/kv_b_proj" + ] }, { "type": "gemm", "args": { - "m": 11520, + "m": 3, "n": 4608, "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/gate_up_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/gate_up_proj", + "nvidia/Kimi-K2.5-NVFP4/gate_up_proj" + ] }, { "type": "gemm", "args": { - "m": 11520, - "n": 7168, - "k": 4608, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 3, + "n": 4608, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/gate_up_proj" + ] }, { "type": "gemm", "args": { - "m": 22, - "n": 1536, - "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 3, + "n": 7168, + "k": 1152, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/down_proj" + ] }, { "type": "gemm", "args": { - "m": 22, - "n": 24576, - "k": 1536, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 3, + "n": 7168, + "k": 2048, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/Kimi-K2.5-MXFP4/o_proj", + "moonshotai/Kimi-K2.5/o_proj", + "nvidia/Kimi-K2.5-NVFP4/o_proj", + "nvidia/Kimi-K2.6-NVFP4/o_proj" + ] }, { "type": "gemm", "args": { - "m": 22, - "n": 576, - "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 3, + "n": 7168, + "k": 2048, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/o_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/o_proj" + ] }, { "type": "gemm", "args": { - "m": 22, - "n": 32768, - "k": 512, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 3, + "n": 7168, + "k": 2048, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/o_proj", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] }, { "type": "gemm", "args": { - "m": 22, + "m": 3, "n": 7168, - "k": 16384, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "k": 2304, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/down_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/down_proj", + "nvidia/Kimi-K2.5-NVFP4/down_proj" + ] }, { "type": "gemm", "args": { - "m": 22, - "n": 4608, - "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 3, + "n": 7168, + "k": 2304, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/down_proj" + ] }, { "type": "gemm", "args": { - "m": 22, + "m": 3, "n": 7168, "k": 4608, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/down_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/down_proj", + "nvidia/Kimi-K2.5-NVFP4/down_proj" + ] }, { "type": "gemm", "args": { - "m": 23040, - "n": 1536, - "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 3, + "n": 7168, + "k": 8192, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/Kimi-K2.5-MXFP4/o_proj", + "moonshotai/Kimi-K2.5/o_proj", + "nvidia/Kimi-K2.5-NVFP4/o_proj", + "nvidia/Kimi-K2.6-NVFP4/o_proj" + ] }, { "type": "gemm", "args": { - "m": 23040, - "n": 24576, - "k": 1536, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 3, + "n": 7168, + "k": 8192, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] }, { "type": "gemm", "args": { - "m": 23040, - "n": 576, - "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 3, + "n": 7168, + "k": 9216, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/down_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/down_proj", + "nvidia/Kimi-K2.5-NVFP4/down_proj" + ] }, { "type": "gemm", "args": { - "m": 23040, - "n": 32768, - "k": 512, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 3, + "n": 7168, + "k": 16384, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/o_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/o_proj" + ] }, { "type": "gemm", "args": { - "m": 23040, + "m": 3, "n": 7168, "k": 16384, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 32 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 1, + 32 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "amd/DeepSeek-R1-0528-MXFP4-Preview/o_proj", + "amd/DeepSeek-R1-0528-MXFP4-v2/o_proj", + "amd/DeepSeek-R1-0528-MXFP4/o_proj" + ] }, { "type": "gemm", "args": { - "m": 23040, - "n": 4608, - "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 3, + "n": 7168, + "k": 16384, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/o_proj", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] }, { "type": "gemm", "args": { - "m": 23040, + "m": 3, "n": 7168, - "k": 4608, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "k": 16384, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] }, { "type": "gemm", "args": { - "m": 45, - "n": 1536, + "m": 3, + "n": 9216, "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/gate_up_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/gate_up_proj", + "nvidia/Kimi-K2.5-NVFP4/gate_up_proj" + ] }, { "type": "gemm", "args": { - "m": 45, - "n": 24576, + "m": 3, + "n": 12288, "k": 1536, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 45, - "n": 576, - "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/Kimi-K2.5-MXFP4/q_b_proj", + "moonshotai/Kimi-K2.5/q_b_proj", + "nvidia/DeepSeek-R1-0528-FP4-V2/q_b_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/q_b_proj", + "nvidia/Kimi-K2.5-NVFP4/q_b_proj", + "nvidia/Kimi-K2.6-NVFP4/q_b_proj" + ] }, { "type": "gemm", "args": { - "m": 45, - "n": 32768, + "m": 3, + "n": 16384, "k": 512, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 45, - "n": 7168, - "k": 16384, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 45, - "n": 4608, - "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/Kimi-K2.5-MXFP4/kv_b_proj", + "moonshotai/Kimi-K2.5/kv_b_proj", + "nvidia/DeepSeek-R1-0528-FP4-V2/kv_b_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/kv_b_proj", + "nvidia/Kimi-K2.5-NVFP4/kv_b_proj", + "nvidia/Kimi-K2.6-NVFP4/kv_b_proj" + ] }, { "type": "gemm", "args": { - "m": 45, - "n": 7168, - "k": 4608, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 3, + "n": 24576, + "k": 1536, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/q_b_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/q_b_proj" + ] }, { "type": "gemm", "args": { - "m": 46080, - "n": 1536, - "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 3, + "n": 24576, + "k": 1536, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 32 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 1, + 32 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "amd/DeepSeek-R1-0528-MXFP4-Preview/q_b_proj", + "amd/DeepSeek-R1-0528-MXFP4-v2/q_b_proj", + "amd/DeepSeek-R1-0528-MXFP4/q_b_proj" + ] }, { "type": "gemm", "args": { - "m": 46080, + "m": 3, "n": 24576, "k": 1536, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/q_b_proj" + ] }, { "type": "gemm", "args": { - "m": 46080, - "n": 576, - "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 3, + "n": 32768, + "k": 512, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/kv_b_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/kv_b_proj" + ] }, { "type": "gemm", "args": { - "m": 46080, + "m": 3, "n": 32768, "k": 512, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 32 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 1, + 32 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "amd/DeepSeek-R1-0528-MXFP4-Preview/kv_b_proj", + "amd/DeepSeek-R1-0528-MXFP4-v2/kv_b_proj", + "amd/DeepSeek-R1-0528-MXFP4/kv_b_proj" + ] }, { "type": "gemm", "args": { - "m": 46080, - "n": 7168, - "k": 16384, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 3, + "n": 32768, + "k": 512, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/kv_b_proj" + ] }, { "type": "gemm", "args": { - "m": 46080, - "n": 4608, - "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 4, + "n": 64, + "k": 2880, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/gpt-oss-120b-w-mxfp4-a-fp8/k_proj", + "amd/gpt-oss-120b-w-mxfp4-a-fp8/v_proj", + "openai/gpt-oss-120b/k_proj", + "openai/gpt-oss-120b/v_proj" + ] }, { "type": "gemm", "args": { - "m": 46080, - "n": 7168, - "k": 4608, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 4, + "n": 128, + "k": 2880, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/gpt-oss-120b-w-mxfp4-a-fp8/k_proj", + "amd/gpt-oss-120b-w-mxfp4-a-fp8/v_proj", + "openai/gpt-oss-120b/k_proj", + "openai/gpt-oss-120b/v_proj" + ] }, { "type": "gemm", "args": { - "m": 90, - "n": 1536, - "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 4, + "n": 128, + "k": 3072, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/MiniMax-M2.5-MXFP4/k_proj", + "amd/MiniMax-M2.5-MXFP4/v_proj", + "nvidia/MiniMax-M2.5-NVFP4/k_proj", + "nvidia/MiniMax-M2.5-NVFP4/v_proj" + ] }, { "type": "gemm", "args": { - "m": 90, - "n": 24576, - "k": 1536, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 4, + "n": 128, + "k": 3072, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "MiniMaxAI/MiniMax-M2.5/k_proj", + "MiniMaxAI/MiniMax-M2.5/v_proj" + ] }, { "type": "gemm", "args": { - "m": 90, - "n": 576, - "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 4, + "n": 128, + "k": 4096, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "Qwen/Qwen3.5-397B-A17B/shared_expert.gate_up_proj", + "amd/Qwen3.5-397B-A17B-MXFP4/shared_expert.gate_up_proj", + "nvidia/Qwen3.5-397B-A17B-NVFP4/shared_expert.gate_up_proj" + ] }, { "type": "gemm", "args": { - "m": 90, - "n": 32768, - "k": 512, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 4, + "n": 128, + "k": 4096, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "Qwen/Qwen3.5-397B-A17B-FP8/shared_expert.gate_up_proj" + ] }, { "type": "gemm", "args": { - "m": 90, - "n": 7168, - "k": 16384, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 4, + "n": 192, + "k": 3072, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/MiniMax-M2.5-NVFP4/experts.gate_up_proj" + ] }, { "type": "gemm", "args": { - "m": 90, - "n": 4608, - "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 4, + "n": 192, + "k": 3072, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 32 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 1, + 32 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "amd/MiniMax-M2.5-MXFP4/experts.gate_up_proj" + ] }, { "type": "gemm", "args": { - "m": 90, - "n": 7168, - "k": 4608, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 4, + "n": 192, + "k": 3072, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "MiniMaxAI/MiniMax-M2.5/experts.gate_up_proj" + ] }, { "type": "gemm", "args": { - "m": 10240, - "n": 1536, - "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 4, + "n": 256, + "k": 2880, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/gpt-oss-120b-w-mxfp4-a-fp8/k_proj", + "amd/gpt-oss-120b-w-mxfp4-a-fp8/q_proj", + "amd/gpt-oss-120b-w-mxfp4-a-fp8/v_proj", + "openai/gpt-oss-120b/k_proj", + "openai/gpt-oss-120b/q_proj", + "openai/gpt-oss-120b/v_proj" + ] }, { "type": "gemm", "args": { - "m": 10240, - "n": 3072, - "k": 1536, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 4, + "n": 256, + "k": 3072, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/MiniMax-M2.5-MXFP4/k_proj", + "amd/MiniMax-M2.5-MXFP4/v_proj", + "nvidia/MiniMax-M2.5-NVFP4/k_proj", + "nvidia/MiniMax-M2.5-NVFP4/v_proj" + ] }, { "type": "gemm", "args": { - "m": 10240, - "n": 576, - "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 4, + "n": 256, + "k": 3072, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "MiniMaxAI/MiniMax-M2.5/k_proj", + "MiniMaxAI/MiniMax-M2.5/v_proj" + ] }, { "type": "gemm", "args": { - "m": 10240, - "n": 4096, - "k": 512, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 4, + "n": 256, + "k": 4096, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "Qwen/Qwen3.5-397B-A17B/k_proj", + "Qwen/Qwen3.5-397B-A17B/shared_expert.gate_up_proj", + "Qwen/Qwen3.5-397B-A17B/v_proj", + "amd/Qwen3.5-397B-A17B-MXFP4/k_proj", + "amd/Qwen3.5-397B-A17B-MXFP4/shared_expert.gate_up_proj", + "amd/Qwen3.5-397B-A17B-MXFP4/v_proj", + "nvidia/Qwen3.5-397B-A17B-NVFP4/k_proj", + "nvidia/Qwen3.5-397B-A17B-NVFP4/shared_expert.gate_up_proj", + "nvidia/Qwen3.5-397B-A17B-NVFP4/v_proj" + ] }, { "type": "gemm", "args": { - "m": 10240, - "n": 7168, - "k": 2048, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 4, + "n": 256, + "k": 4096, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "Qwen/Qwen3.5-397B-A17B-FP8/k_proj", + "Qwen/Qwen3.5-397B-A17B-FP8/shared_expert.gate_up_proj", + "Qwen/Qwen3.5-397B-A17B-FP8/v_proj" + ] }, { "type": "gemm", "args": { - "m": 10240, - "n": 2304, - "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 4, + "n": 384, + "k": 3072, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/MiniMax-M2.5-NVFP4/experts.gate_up_proj" + ] }, { "type": "gemm", "args": { - "m": 10240, - "n": 7168, - "k": 2304, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 4, + "n": 384, + "k": 3072, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 32 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 1, + 32 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "amd/MiniMax-M2.5-MXFP4/experts.gate_up_proj" + ] }, { "type": "gemm", "args": { - "m": 20, - "n": 1536, - "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 4, + "n": 384, + "k": 3072, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "MiniMaxAI/MiniMax-M2.5/experts.gate_up_proj", + "MiniMaxAI/MiniMax-M2.5/q_proj" + ] }, { "type": "gemm", "args": { - "m": 20, - "n": 3072, - "k": 1536, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 4, + "n": 512, + "k": 2880, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/gpt-oss-120b-w-mxfp4-a-fp8/k_proj", + "amd/gpt-oss-120b-w-mxfp4-a-fp8/q_proj", + "amd/gpt-oss-120b-w-mxfp4-a-fp8/v_proj", + "openai/gpt-oss-120b/k_proj", + "openai/gpt-oss-120b/q_proj", + "openai/gpt-oss-120b/v_proj" + ] }, { "type": "gemm", "args": { - "m": 20, - "n": 576, - "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 4, + "n": 512, + "k": 3072, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/MiniMax-M2.5-MXFP4/k_proj", + "amd/MiniMax-M2.5-MXFP4/v_proj", + "nvidia/MiniMax-M2.5-NVFP4/k_proj", + "nvidia/MiniMax-M2.5-NVFP4/v_proj" + ] }, { "type": "gemm", "args": { - "m": 20, - "n": 4096, - "k": 512, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 4, + "n": 512, + "k": 3072, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "MiniMaxAI/MiniMax-M2.5/k_proj", + "MiniMaxAI/MiniMax-M2.5/v_proj" + ] }, { "type": "gemm", "args": { - "m": 20, - "n": 7168, - "k": 2048, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 4, + "n": 512, + "k": 4096, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "Qwen/Qwen3.5-397B-A17B/k_proj", + "Qwen/Qwen3.5-397B-A17B/q_proj", + "Qwen/Qwen3.5-397B-A17B/shared_expert.gate_up_proj", + "Qwen/Qwen3.5-397B-A17B/v_proj", + "amd/Qwen3.5-397B-A17B-MXFP4/k_proj", + "amd/Qwen3.5-397B-A17B-MXFP4/q_proj", + "amd/Qwen3.5-397B-A17B-MXFP4/shared_expert.gate_up_proj", + "amd/Qwen3.5-397B-A17B-MXFP4/v_proj", + "nvidia/Qwen3.5-397B-A17B-NVFP4/k_proj", + "nvidia/Qwen3.5-397B-A17B-NVFP4/q_proj", + "nvidia/Qwen3.5-397B-A17B-NVFP4/shared_expert.gate_up_proj", + "nvidia/Qwen3.5-397B-A17B-NVFP4/v_proj" + ] }, { "type": "gemm", "args": { - "m": 20, - "n": 2304, - "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 4, + "n": 512, + "k": 4096, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "Qwen/Qwen3.5-397B-A17B-FP8/k_proj", + "Qwen/Qwen3.5-397B-A17B-FP8/q_proj", + "Qwen/Qwen3.5-397B-A17B-FP8/shared_expert.gate_up_proj", + "Qwen/Qwen3.5-397B-A17B-FP8/v_proj", + "deepseek-ai/DeepSeek-V4.1-Flash/attn.wo_a" + ] }, { "type": "gemm", "args": { - "m": 20, - "n": 7168, - "k": 2304, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 4, + "n": 576, + "k": 6144, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/GLM-5.1-MXFP4/kv_a_proj_with_mqa", + "amd/GLM-5.2-MXFP4/kv_a_proj_with_mqa", + "nvidia/GLM-5-NVFP4/kv_a_proj_with_mqa", + "nvidia/GLM-5.1-NVFP4/kv_a_proj_with_mqa", + "nvidia/GLM-5.2-NVFP4/kv_a_proj_with_mqa" + ] }, { "type": "gemm", "args": { - "m": 15360, - "n": 1536, - "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 4, + "n": 576, + "k": 6144, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "zai-org/GLM-5-FP8/kv_a_proj_with_mqa", + "zai-org/GLM-5.1-FP8/kv_a_proj_with_mqa", + "zai-org/GLM-5.2-FP8/kv_a_proj_with_mqa" + ] }, { "type": "gemm", "args": { - "m": 15360, - "n": 3072, - "k": 1536, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 4, + "n": 576, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/Kimi-K2.5-MXFP4/kv_a_proj_with_mqa", + "moonshotai/Kimi-K2.5/kv_a_proj_with_mqa", + "nvidia/DeepSeek-R1-0528-FP4-V2/kv_a_proj_with_mqa", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/kv_a_proj_with_mqa", + "nvidia/Kimi-K2.5-NVFP4/kv_a_proj_with_mqa", + "nvidia/Kimi-K2.6-NVFP4/kv_a_proj_with_mqa" + ] }, { "type": "gemm", "args": { - "m": 15360, + "m": 4, "n": 576, "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 32 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 1, + 32 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "amd/DeepSeek-R1-0528-MXFP4-Preview/kv_a_proj_with_mqa", + "amd/DeepSeek-R1-0528-MXFP4-v2/kv_a_proj_with_mqa", + "amd/DeepSeek-R1-0528-MXFP4/kv_a_proj_with_mqa" + ] }, { "type": "gemm", "args": { - "m": 15360, - "n": 4096, - "k": 512, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 4, + "n": 576, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/kv_a_proj_with_mqa" + ] }, { "type": "gemm", "args": { - "m": 15360, - "n": 7168, - "k": 2048, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 4, + "n": 768, + "k": 3072, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/MiniMax-M2.5-NVFP4/experts.gate_up_proj" + ] }, { "type": "gemm", "args": { - "m": 15360, - "n": 2304, - "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 4, + "n": 768, + "k": 3072, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 32 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 1, + 32 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "amd/MiniMax-M2.5-MXFP4/experts.gate_up_proj" + ] }, { "type": "gemm", "args": { - "m": 15360, - "n": 7168, - "k": 2304, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 4, + "n": 768, + "k": 3072, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "MiniMaxAI/MiniMax-M2.5/experts.gate_up_proj", + "MiniMaxAI/MiniMax-M2.5/q_proj" + ] }, { "type": "gemm", "args": { - "m": 30, - "n": 1536, - "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 4, + "n": 1024, + "k": 2880, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/gpt-oss-120b-w-mxfp4-a-fp8/q_proj", + "openai/gpt-oss-120b/q_proj" + ] }, { "type": "gemm", "args": { - "m": 30, - "n": 3072, - "k": 1536, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 4, + "n": 1024, + "k": 3072, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/MiniMax-M2.5-MXFP4/k_proj", + "amd/MiniMax-M2.5-MXFP4/qkv_proj", + "amd/MiniMax-M2.5-MXFP4/v_proj", + "nvidia/MiniMax-M2.5-NVFP4/k_proj", + "nvidia/MiniMax-M2.5-NVFP4/qkv_proj", + "nvidia/MiniMax-M2.5-NVFP4/v_proj" + ] }, { "type": "gemm", "args": { - "m": 30, - "n": 576, - "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 4, + "n": 1024, + "k": 3072, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "MiniMaxAI/MiniMax-M2.5/k_proj", + "MiniMaxAI/MiniMax-M2.5/qkv_proj", + "MiniMaxAI/MiniMax-M2.5/v_proj" + ] }, { "type": "gemm", "args": { - "m": 30, - "n": 4096, - "k": 512, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 4, + "n": 1024, + "k": 4096, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "Qwen/Qwen3.5-397B-A17B/q_proj", + "Qwen/Qwen3.5-397B-A17B/shared_expert.gate_up_proj", + "amd/Qwen3.5-397B-A17B-MXFP4/q_proj", + "amd/Qwen3.5-397B-A17B-MXFP4/shared_expert.gate_up_proj", + "nvidia/Qwen3.5-397B-A17B-NVFP4/q_proj", + "nvidia/Qwen3.5-397B-A17B-NVFP4/shared_expert.gate_up_proj" + ] }, { "type": "gemm", "args": { - "m": 30, - "n": 7168, - "k": 2048, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 4, + "n": 1024, + "k": 4096, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "Qwen/Qwen3.5-397B-A17B-FP8/q_proj", + "Qwen/Qwen3.5-397B-A17B-FP8/shared_expert.gate_up_proj", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_a", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_a", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_a" + ] }, { "type": "gemm", "args": { - "m": 30, - "n": 2304, + "m": 4, + "n": 1088, "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] }, { "type": "gemm", "args": { - "m": 30, - "n": 7168, - "k": 2304, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 4, + "n": 1152, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/gate_up_proj" + ] }, { "type": "gemm", "args": { - "m": 25600, + "m": 4, "n": 1536, - "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "k": 1536, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/Kimi-K2.5-MXFP4/q_b_proj", + "moonshotai/Kimi-K2.5/q_b_proj", + "nvidia/DeepSeek-R1-0528-FP4-V2/q_b_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/q_b_proj", + "nvidia/Kimi-K2.5-NVFP4/q_b_proj", + "nvidia/Kimi-K2.6-NVFP4/q_b_proj" + ] }, { "type": "gemm", "args": { - "m": 25600, - "n": 3072, + "m": 4, + "n": 1536, "k": 1536, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/q_b_proj" + ] }, { "type": "gemm", "args": { - "m": 25600, - "n": 576, - "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 4, + "n": 1536, + "k": 3072, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 32 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 1, + 32 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "amd/MiniMax-M2.5-MXFP4/experts.gate_up_proj" + ] }, { "type": "gemm", "args": { - "m": 25600, - "n": 4096, - "k": 512, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 4, + "n": 1536, + "k": 3072, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "MiniMaxAI/MiniMax-M2.5/experts.gate_up_proj", + "MiniMaxAI/MiniMax-M2.5/q_proj" + ] }, { "type": "gemm", "args": { - "m": 25600, - "n": 7168, - "k": 2048, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 4, + "n": 1536, + "k": 6144, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/GLM-5-NVFP4/gate_up_proj" + ] }, { "type": "gemm", "args": { - "m": 25600, - "n": 2304, - "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 4, + "n": 1536, + "k": 6144, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "zai-org/GLM-5-FP8/gate_up_proj", + "zai-org/GLM-5.1-FP8/gate_up_proj", + "zai-org/GLM-5.2-FP8/gate_up_proj" + ] }, { "type": "gemm", "args": { - "m": 25600, - "n": 7168, - "k": 2304, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 4, + "n": 1536, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/Kimi-K2.5-MXFP4/q_a_proj", + "moonshotai/Kimi-K2.5/q_a_proj", + "nvidia/DeepSeek-R1-0528-FP4-V2/q_a_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/q_a_proj", + "nvidia/Kimi-K2.5-NVFP4/q_a_proj", + "nvidia/Kimi-K2.6-NVFP4/q_a_proj" + ] }, { "type": "gemm", "args": { - "m": 50, + "m": 4, "n": 1536, "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 32 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 1, + 32 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "amd/DeepSeek-R1-0528-MXFP4-Preview/q_a_proj", + "amd/DeepSeek-R1-0528-MXFP4-v2/q_a_proj", + "amd/DeepSeek-R1-0528-MXFP4/q_a_proj" + ] }, { "type": "gemm", "args": { - "m": 50, - "n": 3072, - "k": 1536, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 4, + "n": 1536, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/gate_up_proj", + "deepseek-ai/DeepSeek-R1-0528/q_a_proj", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w3" + ] }, { "type": "gemm", "args": { - "m": 50, - "n": 576, + "m": 4, + "n": 1536, "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w3" + ] }, { "type": "gemm", "args": { - "m": 50, - "n": 4096, + "m": 4, + "n": 2048, "k": 512, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/Kimi-K2.5-MXFP4/kv_b_proj", + "moonshotai/Kimi-K2.5/kv_b_proj", + "nvidia/DeepSeek-R1-0528-FP4-V2/kv_b_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/kv_b_proj", + "nvidia/Kimi-K2.5-NVFP4/kv_b_proj", + "nvidia/Kimi-K2.6-NVFP4/kv_b_proj" + ] }, { "type": "gemm", "args": { - "m": 50, - "n": 7168, + "m": 4, + "n": 2048, + "k": 512, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/kv_b_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 4, + "n": 2048, "k": 2048, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/GLM-5.1-MXFP4/q_b_proj", + "amd/GLM-5.2-MXFP4/q_b_proj", + "nvidia/GLM-5-NVFP4/q_b_proj", + "nvidia/GLM-5.1-NVFP4/q_b_proj", + "nvidia/GLM-5.2-NVFP4/q_b_proj" + ] }, { "type": "gemm", "args": { - "m": 50, - "n": 2304, - "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 4, + "n": 2048, + "k": 2048, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "zai-org/GLM-5-FP8/q_b_proj", + "zai-org/GLM-5.1-FP8/q_b_proj", + "zai-org/GLM-5.2-FP8/q_b_proj" + ] }, { "type": "gemm", "args": { - "m": 50, - "n": 7168, - "k": 2304, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 4, + "n": 2048, + "k": 2880, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/gpt-oss-120b-w-mxfp4-a-fp8/q_proj", + "openai/gpt-oss-120b/q_proj" + ] }, { "type": "gemm", "args": { - "m": 46080, - "n": 3072, - "k": 1536, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 4, + "n": 2048, + "k": 4096, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "Qwen/Qwen3.5-397B-A17B/q_proj", + "Qwen/Qwen3.5-397B-A17B/shared_expert.gate_up_proj", + "amd/Qwen3.5-397B-A17B-MXFP4/q_proj", + "amd/Qwen3.5-397B-A17B-MXFP4/shared_expert.gate_up_proj", + "nvidia/Qwen3.5-397B-A17B-NVFP4/q_proj", + "nvidia/Qwen3.5-397B-A17B-NVFP4/shared_expert.gate_up_proj" + ] }, { "type": "gemm", "args": { - "m": 46080, - "n": 4096, - "k": 512, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 4, + "n": 2048, + "k": 4096, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "Qwen/Qwen3.5-397B-A17B-FP8/q_proj", + "Qwen/Qwen3.5-397B-A17B-FP8/shared_expert.gate_up_proj" + ] }, { "type": "gemm", "args": { - "m": 46080, - "n": 7168, - "k": 2048, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 4, + "n": 2048, + "k": 6144, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/GLM-5.1-MXFP4/q_a_proj", + "amd/GLM-5.2-MXFP4/q_a_proj", + "amd/MiniMax-M3-MXFP4/q_proj", + "nvidia/GLM-5-NVFP4/q_a_proj", + "nvidia/GLM-5.1-NVFP4/q_a_proj", + "nvidia/GLM-5.2-NVFP4/q_a_proj" + ] }, { "type": "gemm", "args": { - "m": 46080, + "m": 4, + "n": 2048, + "k": 6144, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "zai-org/GLM-5-FP8/q_a_proj", + "zai-org/GLM-5.1-FP8/q_a_proj", + "zai-org/GLM-5.2-FP8/q_a_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 4, "n": 2304, "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/gate_up_proj" + ] }, { "type": "gemm", "args": { - "m": 46080, - "n": 7168, - "k": 2304, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 4, + "n": 2880, + "k": 512, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/gpt-oss-120b-w-mxfp4-a-fp8/o_proj", + "openai/gpt-oss-120b/o_proj" + ] }, { "type": "gemm", "args": { - "m": 90, - "n": 3072, - "k": 1536, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 4, + "n": 2880, + "k": 1024, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/gpt-oss-120b-w-mxfp4-a-fp8/o_proj", + "openai/gpt-oss-120b/o_proj" + ] }, { "type": "gemm", "args": { - "m": 90, - "n": 4096, - "k": 512, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 4, + "n": 2880, + "k": 2048, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/gpt-oss-120b-w-mxfp4-a-fp8/o_proj", + "openai/gpt-oss-120b/o_proj" + ] }, { "type": "gemm", "args": { - "m": 90, - "n": 7168, - "k": 2048, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 4, + "n": 2880, + "k": 4096, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/gpt-oss-120b-w-mxfp4-a-fp8/o_proj", + "openai/gpt-oss-120b/o_proj" + ] }, { "type": "gemm", "args": { - "m": 90, - "n": 2304, - "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 4, + "n": 3072, + "k": 192, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/MiniMax-M2.5-NVFP4/experts.down_proj" + ] }, { "type": "gemm", "args": { - "m": 90, - "n": 7168, - "k": 2304, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 4, + "n": 3072, + "k": 192, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 32 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 1, + 32 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "amd/MiniMax-M2.5-MXFP4/experts.down_proj" + ] }, { "type": "gemm", "args": { - "m": 92160, - "n": 1536, - "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 4, + "n": 3072, + "k": 192, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "MiniMaxAI/MiniMax-M2.5/experts.down_proj" + ] }, { "type": "gemm", "args": { - "m": 92160, + "m": 4, "n": 3072, - "k": 1536, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "k": 384, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/MiniMax-M2.5-NVFP4/experts.down_proj" + ] }, { "type": "gemm", "args": { - "m": 92160, - "n": 576, - "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 4, + "n": 3072, + "k": 384, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 32 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 1, + 32 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "amd/MiniMax-M2.5-MXFP4/experts.down_proj" + ] }, { "type": "gemm", "args": { - "m": 92160, - "n": 4096, - "k": 512, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 4, + "n": 3072, + "k": 384, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "MiniMaxAI/MiniMax-M2.5/experts.down_proj", + "MiniMaxAI/MiniMax-M2.5/o_proj" + ] }, { "type": "gemm", "args": { - "m": 92160, - "n": 7168, - "k": 2048, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 4, + "n": 3072, + "k": 768, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/MiniMax-M2.5-MXFP4/o_proj", + "nvidia/MiniMax-M2.5-NVFP4/o_proj" + ] }, { "type": "gemm", "args": { - "m": 92160, - "n": 2304, - "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 4, + "n": 3072, + "k": 768, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 32 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 1, + 32 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "amd/MiniMax-M2.5-MXFP4/experts.down_proj" + ] }, { "type": "gemm", "args": { - "m": 92160, - "n": 7168, - "k": 2304, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 4, + "n": 3072, + "k": 768, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "MiniMaxAI/MiniMax-M2.5/experts.down_proj", + "MiniMaxAI/MiniMax-M2.5/o_proj" + ] }, { "type": "gemm", "args": { - "m": 180, - "n": 1536, - "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 4, + "n": 3072, + "k": 1536, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/Kimi-K2.5-MXFP4/q_b_proj", + "amd/MiniMax-M2.5-MXFP4/o_proj", + "moonshotai/Kimi-K2.5/q_b_proj", + "nvidia/DeepSeek-R1-0528-FP4-V2/q_b_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/q_b_proj", + "nvidia/Kimi-K2.5-NVFP4/q_b_proj", + "nvidia/Kimi-K2.6-NVFP4/q_b_proj", + "nvidia/MiniMax-M2.5-NVFP4/o_proj" + ] }, { "type": "gemm", "args": { - "m": 180, + "m": 4, "n": 3072, "k": 1536, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/MiniMax-M2.5-NVFP4/experts.down_proj" + ] }, { "type": "gemm", "args": { - "m": 180, - "n": 576, - "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 4, + "n": 3072, + "k": 1536, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 32 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 1, + 32 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "amd/DeepSeek-R1-0528-MXFP4-Preview/q_b_proj", + "amd/DeepSeek-R1-0528-MXFP4-v2/q_b_proj", + "amd/DeepSeek-R1-0528-MXFP4/q_b_proj", + "amd/MiniMax-M2.5-MXFP4/experts.down_proj" + ] }, { "type": "gemm", "args": { - "m": 180, - "n": 4096, - "k": 512, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 4, + "n": 3072, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "MiniMaxAI/MiniMax-M2.5/experts.down_proj", + "MiniMaxAI/MiniMax-M2.5/o_proj", + "deepseek-ai/DeepSeek-R1-0528/q_b_proj" + ] }, { "type": "gemm", "args": { - "m": 180, - "n": 7168, - "k": 2048, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 4, + "n": 3072, + "k": 3072, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/MiniMax-M2.5-MXFP4/o_proj", + "amd/MiniMax-M2.5-MXFP4/q_proj", + "nvidia/MiniMax-M2.5-NVFP4/o_proj", + "nvidia/MiniMax-M2.5-NVFP4/q_proj" + ] }, { "type": "gemm", "args": { - "m": 180, - "n": 2304, - "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 4, + "n": 3072, + "k": 3072, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/MiniMax-M2.5-NVFP4/experts.gate_up_proj" + ] }, { "type": "gemm", "args": { - "m": 180, - "n": 7168, - "k": 2304, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 4, + "n": 3072, + "k": 3072, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 32 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 1, + 32 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "amd/MiniMax-M2.5-MXFP4/experts.gate_up_proj" + ] }, { "type": "gemm", "args": { - "m": 184320, - "n": 1536, - "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 4, + "n": 3072, + "k": 3072, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "MiniMaxAI/MiniMax-M2.5/experts.gate_up_proj", + "MiniMaxAI/MiniMax-M2.5/o_proj", + "MiniMaxAI/MiniMax-M2.5/q_proj" + ] }, { "type": "gemm", "args": { - "m": 184320, + "m": 4, "n": 3072, - "k": 1536, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "k": 6144, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/GLM-5.1-MXFP4/gate_up_proj", + "amd/GLM-5.2-MXFP4/gate_up_proj", + "amd/MiniMax-M2.5-MXFP4/o_proj", + "amd/MiniMax-M3-MXFP4/gate_up_proj", + "nvidia/GLM-5.1-NVFP4/gate_up_proj", + "nvidia/GLM-5.2-NVFP4/gate_up_proj", + "nvidia/MiniMax-M2.5-NVFP4/o_proj" + ] }, { "type": "gemm", "args": { - "m": 184320, - "n": 576, - "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 4, + "n": 3072, + "k": 6144, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/GLM-5-NVFP4/gate_up_proj" + ] }, { "type": "gemm", "args": { - "m": 184320, - "n": 4096, + "m": 4, + "n": 3072, + "k": 6144, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "MiniMaxAI/MiniMax-M2.5/o_proj", + "zai-org/GLM-5-FP8/gate_up_proj", + "zai-org/GLM-5.1-FP8/gate_up_proj", + "zai-org/GLM-5.2-FP8/gate_up_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 4, + "n": 3584, "k": 512, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/GLM-5.1-MXFP4/kv_b_proj", + "amd/GLM-5.2-MXFP4/kv_b_proj", + "nvidia/GLM-5-NVFP4/kv_b_proj", + "nvidia/GLM-5.1-NVFP4/kv_b_proj", + "nvidia/GLM-5.2-NVFP4/kv_b_proj" + ] }, { "type": "gemm", "args": { - "m": 184320, - "n": 7168, - "k": 2048, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 4, + "n": 3584, + "k": 512, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "zai-org/GLM-5-FP8/kv_b_proj", + "zai-org/GLM-5.1-FP8/kv_b_proj", + "zai-org/GLM-5.2-FP8/kv_b_proj" + ] }, { "type": "gemm", "args": { - "m": 184320, - "n": 2304, - "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 4, + "n": 4096, + "k": 128, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "Qwen/Qwen3.5-397B-A17B/shared_expert.down_proj", + "amd/Qwen3.5-397B-A17B-MXFP4/shared_expert.down_proj", + "nvidia/Qwen3.5-397B-A17B-NVFP4/shared_expert.down_proj" + ] }, { "type": "gemm", "args": { - "m": 184320, - "n": 7168, - "k": 2304, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 4, + "n": 4096, + "k": 128, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "Qwen/Qwen3.5-397B-A17B-FP8/shared_expert.down_proj" + ] }, { "type": "gemm", "args": { - "m": 360, - "n": 1536, - "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 4, + "n": 4096, + "k": 256, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "Qwen/Qwen3.5-397B-A17B/shared_expert.down_proj", + "amd/Qwen3.5-397B-A17B-MXFP4/shared_expert.down_proj", + "nvidia/Qwen3.5-397B-A17B-NVFP4/shared_expert.down_proj" + ] }, { "type": "gemm", "args": { - "m": 360, - "n": 3072, - "k": 1536, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 4, + "n": 4096, + "k": 256, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "Qwen/Qwen3.5-397B-A17B-FP8/shared_expert.down_proj" + ] }, { "type": "gemm", "args": { - "m": 360, - "n": 576, - "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 4, + "n": 4096, + "k": 512, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "Qwen/Qwen3.5-397B-A17B/o_proj", + "Qwen/Qwen3.5-397B-A17B/shared_expert.down_proj", + "amd/Kimi-K2.5-MXFP4/kv_b_proj", + "amd/Qwen3.5-397B-A17B-MXFP4/o_proj", + "amd/Qwen3.5-397B-A17B-MXFP4/shared_expert.down_proj", + "moonshotai/Kimi-K2.5/kv_b_proj", + "nvidia/DeepSeek-R1-0528-FP4-V2/kv_b_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/kv_b_proj", + "nvidia/Kimi-K2.5-NVFP4/kv_b_proj", + "nvidia/Kimi-K2.6-NVFP4/kv_b_proj", + "nvidia/Qwen3.5-397B-A17B-NVFP4/o_proj", + "nvidia/Qwen3.5-397B-A17B-NVFP4/shared_expert.down_proj" + ] }, { "type": "gemm", "args": { - "m": 360, + "m": 4, "n": 4096, "k": 512, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 32 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 1, + 32 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "amd/DeepSeek-R1-0528-MXFP4-Preview/kv_b_proj", + "amd/DeepSeek-R1-0528-MXFP4-v2/kv_b_proj", + "amd/DeepSeek-R1-0528-MXFP4/kv_b_proj", + "amd/Qwen3.5-397B-A17B-MXFP4-AttnFP8-V2/shared_expert.down_proj" + ] }, { "type": "gemm", "args": { - "m": 360, - "n": 7168, - "k": 2048, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 4, + "n": 4096, + "k": 512, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "Qwen/Qwen3.5-397B-A17B-FP8/o_proj", + "Qwen/Qwen3.5-397B-A17B-FP8/shared_expert.down_proj", + "deepseek-ai/DeepSeek-R1-0528/kv_b_proj" + ] }, { "type": "gemm", "args": { - "m": 360, - "n": 2304, - "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 4, + "n": 4096, + "k": 1024, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "Qwen/Qwen3.5-397B-A17B/o_proj", + "Qwen/Qwen3.5-397B-A17B/shared_expert.down_proj", + "amd/Qwen3.5-397B-A17B-MXFP4/o_proj", + "amd/Qwen3.5-397B-A17B-MXFP4/shared_expert.down_proj", + "nvidia/Qwen3.5-397B-A17B-NVFP4/o_proj", + "nvidia/Qwen3.5-397B-A17B-NVFP4/shared_expert.down_proj" + ] }, { "type": "gemm", "args": { - "m": 360, - "n": 7168, - "k": 2304, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 4, + "n": 4096, + "k": 1024, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "Qwen/Qwen3.5-397B-A17B-FP8/o_proj", + "Qwen/Qwen3.5-397B-A17B-FP8/shared_expert.down_proj" + ] }, { "type": "gemm", "args": { - "m": 1271808, - "n": 1536, - "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 4, + "n": 4096, + "k": 2048, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "Qwen/Qwen3.5-397B-A17B/o_proj", + "amd/GLM-5.1-MXFP4/q_b_proj", + "amd/GLM-5.2-MXFP4/q_b_proj", + "amd/Qwen3.5-397B-A17B-MXFP4/o_proj", + "nvidia/GLM-5-NVFP4/q_b_proj", + "nvidia/GLM-5.1-NVFP4/q_b_proj", + "nvidia/GLM-5.2-NVFP4/q_b_proj", + "nvidia/Qwen3.5-397B-A17B-NVFP4/o_proj" + ] }, { "type": "gemm", "args": { - "m": 1271808, - "n": 24576, - "k": 1536, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 4, + "n": 4096, + "k": 2048, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "Qwen/Qwen3.5-397B-A17B-FP8/o_proj", + "zai-org/GLM-5-FP8/q_b_proj", + "zai-org/GLM-5.1-FP8/q_b_proj", + "zai-org/GLM-5.2-FP8/q_b_proj" + ] }, { "type": "gemm", "args": { - "m": 1271808, - "n": 576, - "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 4, + "n": 4096, + "k": 2880, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/gpt-oss-120b-w-mxfp4-a-fp8/q_proj", + "openai/gpt-oss-120b/q_proj" + ] }, { "type": "gemm", "args": { - "m": 1271808, - "n": 32768, - "k": 512, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 4, + "n": 4096, + "k": 4096, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "Qwen/Qwen3.5-397B-A17B/o_proj", + "Qwen/Qwen3.5-397B-A17B/q_proj", + "amd/Qwen3.5-397B-A17B-MXFP4/o_proj", + "amd/Qwen3.5-397B-A17B-MXFP4/q_proj", + "nvidia/Qwen3.5-397B-A17B-NVFP4/o_proj", + "nvidia/Qwen3.5-397B-A17B-NVFP4/q_proj" + ] }, { "type": "gemm", "args": { - "m": 1271808, - "n": 7168, - "k": 16384, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 4, + "n": 4096, + "k": 4096, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "Qwen/Qwen3.5-397B-A17B-FP8/o_proj", + "Qwen/Qwen3.5-397B-A17B-FP8/q_proj" + ] }, { "type": "gemm", "args": { - "m": 1271808, + "m": 4, "n": 4608, "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "moonshotai/Kimi-K2.5/gate_up_proj", + "nvidia/Kimi-K2.6-NVFP4/gate_up_proj" + ] }, { "type": "gemm", "args": { - "m": 1271808, - "n": 7168, - "k": 4608, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 4, + "n": 4608, + "k": 7168, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/gate_up_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/gate_up_proj", + "nvidia/Kimi-K2.5-NVFP4/gate_up_proj" + ] }, { "type": "gemm", "args": { - "m": 2484, - "n": 1536, + "m": 4, + "n": 4608, "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 32 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 1, + 32 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "amd/DeepSeek-R1-0528-MXFP4-Preview/gate_up_proj", + "amd/DeepSeek-R1-0528-MXFP4-v2/gate_up_proj", + "amd/DeepSeek-R1-0528-MXFP4/gate_up_proj", + "amd/Kimi-K2.5-MXFP4/gate_up_proj" + ] }, { "type": "gemm", "args": { - "m": 2484, - "n": 24576, - "k": 1536, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 4, + "n": 4608, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/gate_up_proj" + ] }, { "type": "gemm", "args": { - "m": 2484, - "n": 576, - "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 4, + "n": 6144, + "k": 1536, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/GLM-5.1-MXFP4/down_proj", + "amd/GLM-5.2-MXFP4/down_proj", + "amd/Kimi-K2.5-MXFP4/q_b_proj", + "amd/MiniMax-M3-MXFP4/down_proj", + "moonshotai/Kimi-K2.5/q_b_proj", + "nvidia/DeepSeek-R1-0528-FP4-V2/q_b_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/q_b_proj", + "nvidia/GLM-5.1-NVFP4/down_proj", + "nvidia/GLM-5.2-NVFP4/down_proj", + "nvidia/Kimi-K2.5-NVFP4/q_b_proj", + "nvidia/Kimi-K2.6-NVFP4/q_b_proj" + ] }, { "type": "gemm", "args": { - "m": 2484, - "n": 32768, - "k": 512, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 4, + "n": 6144, + "k": 1536, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/GLM-5-NVFP4/down_proj" + ] }, { "type": "gemm", "args": { - "m": 2484, - "n": 7168, - "k": 16384, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 4, + "n": 6144, + "k": 1536, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 32 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 1, + 32 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "amd/DeepSeek-R1-0528-MXFP4-Preview/q_b_proj", + "amd/DeepSeek-R1-0528-MXFP4-v2/q_b_proj", + "amd/DeepSeek-R1-0528-MXFP4/q_b_proj" + ] }, { "type": "gemm", "args": { - "m": 2484, - "n": 4608, - "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 4, + "n": 6144, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/q_b_proj", + "zai-org/GLM-5-FP8/down_proj", + "zai-org/GLM-5.1-FP8/down_proj", + "zai-org/GLM-5.2-FP8/down_proj" + ] }, { "type": "gemm", "args": { - "m": 2484, - "n": 7168, - "k": 4608, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 4, + "n": 6144, + "k": 2048, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/GLM-5.1-MXFP4/o_proj", + "amd/GLM-5.2-MXFP4/o_proj", + "amd/MiniMax-M3-MXFP4/o_proj", + "nvidia/GLM-5-NVFP4/o_proj", + "nvidia/GLM-5.1-NVFP4/o_proj", + "nvidia/GLM-5.2-NVFP4/o_proj" + ] }, { "type": "gemm", "args": { - "m": 635904, - "n": 1536, - "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 4, + "n": 6144, + "k": 2048, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "zai-org/GLM-5-FP8/o_proj", + "zai-org/GLM-5.1-FP8/o_proj", + "zai-org/GLM-5.2-FP8/o_proj" + ] }, { "type": "gemm", "args": { - "m": 635904, - "n": 24576, - "k": 1536, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 4, + "n": 6144, + "k": 3072, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/GLM-5.1-MXFP4/down_proj", + "amd/GLM-5.2-MXFP4/down_proj", + "amd/MiniMax-M2.5-MXFP4/q_proj", + "amd/MiniMax-M3-MXFP4/down_proj", + "nvidia/GLM-5.1-NVFP4/down_proj", + "nvidia/GLM-5.2-NVFP4/down_proj", + "nvidia/MiniMax-M2.5-NVFP4/q_proj" + ] }, { "type": "gemm", "args": { - "m": 635904, - "n": 576, - "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 4, + "n": 6144, + "k": 3072, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/GLM-5-NVFP4/down_proj" + ] }, { "type": "gemm", "args": { - "m": 635904, - "n": 32768, - "k": 512, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 4, + "n": 6144, + "k": 3072, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "MiniMaxAI/MiniMax-M2.5/q_proj", + "zai-org/GLM-5-FP8/down_proj", + "zai-org/GLM-5.1-FP8/down_proj", + "zai-org/GLM-5.2-FP8/down_proj" + ] }, { "type": "gemm", "args": { - "m": 635904, - "n": 7168, - "k": 16384, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 4, + "n": 6144, + "k": 4096, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/GLM-5.1-MXFP4/o_proj", + "amd/GLM-5.2-MXFP4/o_proj", + "amd/MiniMax-M3-MXFP4/o_proj", + "nvidia/GLM-5-NVFP4/o_proj", + "nvidia/GLM-5.1-NVFP4/o_proj", + "nvidia/GLM-5.2-NVFP4/o_proj" + ] }, { "type": "gemm", "args": { - "m": 635904, - "n": 2304, - "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 4, + "n": 6144, + "k": 4096, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "zai-org/GLM-5-FP8/o_proj", + "zai-org/GLM-5.1-FP8/o_proj", + "zai-org/GLM-5.2-FP8/o_proj" + ] }, { "type": "gemm", "args": { - "m": 635904, - "n": 7168, - "k": 2304, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 4, + "n": 6144, + "k": 6144, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/GLM-5.1-MXFP4/down_proj", + "amd/GLM-5.1-MXFP4/gate_up_proj", + "amd/GLM-5.2-MXFP4/down_proj", + "amd/GLM-5.2-MXFP4/gate_up_proj", + "amd/MiniMax-M3-MXFP4/gate_up_proj", + "nvidia/GLM-5.1-NVFP4/down_proj", + "nvidia/GLM-5.1-NVFP4/gate_up_proj", + "nvidia/GLM-5.2-NVFP4/down_proj", + "nvidia/GLM-5.2-NVFP4/gate_up_proj" + ] }, { "type": "gemm", "args": { - "m": 1242, - "n": 1536, - "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 4, + "n": 6144, + "k": 6144, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/GLM-5-NVFP4/down_proj", + "nvidia/GLM-5-NVFP4/gate_up_proj" + ] }, { "type": "gemm", "args": { - "m": 1242, - "n": 24576, - "k": 1536, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 4, + "n": 6144, + "k": 6144, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "zai-org/GLM-5-FP8/down_proj", + "zai-org/GLM-5-FP8/gate_up_proj", + "zai-org/GLM-5.1-FP8/down_proj", + "zai-org/GLM-5.1-FP8/gate_up_proj", + "zai-org/GLM-5.2-FP8/down_proj", + "zai-org/GLM-5.2-FP8/gate_up_proj" + ] }, { "type": "gemm", "args": { - "m": 1242, - "n": 576, - "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 4, + "n": 6144, + "k": 8192, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/GLM-5.1-MXFP4/o_proj", + "amd/GLM-5.2-MXFP4/o_proj", + "amd/MiniMax-M3-MXFP4/o_proj", + "nvidia/GLM-5-NVFP4/o_proj", + "nvidia/GLM-5.1-NVFP4/o_proj", + "nvidia/GLM-5.2-NVFP4/o_proj" + ] }, { "type": "gemm", "args": { - "m": 1242, - "n": 32768, + "m": 4, + "n": 7168, "k": 512, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/GLM-5.1-MXFP4/kv_b_proj", + "amd/GLM-5.2-MXFP4/kv_b_proj", + "amd/Kimi-K2.5-MXFP4/o_proj", + "moonshotai/Kimi-K2.5/o_proj", + "nvidia/GLM-5-NVFP4/kv_b_proj", + "nvidia/GLM-5.1-NVFP4/kv_b_proj", + "nvidia/GLM-5.2-NVFP4/kv_b_proj", + "nvidia/Kimi-K2.5-NVFP4/o_proj", + "nvidia/Kimi-K2.6-NVFP4/o_proj" + ] }, { "type": "gemm", "args": { - "m": 1242, + "m": 4, "n": 7168, - "k": 16384, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "k": 512, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "zai-org/GLM-5-FP8/kv_b_proj", + "zai-org/GLM-5.1-FP8/kv_b_proj", + "zai-org/GLM-5.2-FP8/kv_b_proj" + ] }, { "type": "gemm", "args": { - "m": 1242, - "n": 2304, - "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 4, + "n": 7168, + "k": 576, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/down_proj" + ] }, { "type": "gemm", "args": { - "m": 1242, + "m": 4, "n": 7168, - "k": 2304, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "k": 1024, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/Kimi-K2.5-MXFP4/o_proj", + "moonshotai/Kimi-K2.5/o_proj", + "nvidia/Kimi-K2.5-NVFP4/o_proj", + "nvidia/Kimi-K2.6-NVFP4/o_proj" + ] }, { "type": "gemm", "args": { - "m": 2780160, - "n": 1536, - "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 4, + "n": 7168, + "k": 1024, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/o_proj", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] }, { "type": "gemm", "args": { - "m": 2780160, - "n": 24576, - "k": 1536, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 4, + "n": 7168, + "k": 1152, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/down_proj" + ] }, { "type": "gemm", "args": { - "m": 2780160, - "n": 576, - "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 4, + "n": 7168, + "k": 2048, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/Kimi-K2.5-MXFP4/o_proj", + "moonshotai/Kimi-K2.5/o_proj", + "nvidia/Kimi-K2.5-NVFP4/o_proj", + "nvidia/Kimi-K2.6-NVFP4/o_proj" + ] }, { "type": "gemm", "args": { - "m": 2780160, - "n": 32768, - "k": 512, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 4, + "n": 7168, + "k": 2048, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/o_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/o_proj" + ] }, { "type": "gemm", "args": { - "m": 2780160, + "m": 4, "n": 7168, - "k": 16384, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "k": 2048, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 32 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 1, + 32 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "amd/DeepSeek-R1-0528-MXFP4-Preview/o_proj", + "amd/DeepSeek-R1-0528-MXFP4-v2/o_proj", + "amd/DeepSeek-R1-0528-MXFP4/o_proj" + ] }, { "type": "gemm", "args": { - "m": 2780160, - "n": 4608, - "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 4, + "n": 7168, + "k": 2048, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/o_proj", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] }, { "type": "gemm", "args": { - "m": 2780160, + "m": 4, "n": 7168, - "k": 4608, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "k": 2304, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "moonshotai/Kimi-K2.5/down_proj", + "nvidia/Kimi-K2.6-NVFP4/down_proj" + ] }, { "type": "gemm", "args": { - "m": 5430, - "n": 1536, - "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 4, + "n": 7168, + "k": 2304, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/down_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/down_proj", + "nvidia/Kimi-K2.5-NVFP4/down_proj" + ] }, { "type": "gemm", "args": { - "m": 5430, - "n": 24576, - "k": 1536, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 4, + "n": 7168, + "k": 2304, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 32 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 1, + 32 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "amd/DeepSeek-R1-0528-MXFP4-Preview/down_proj", + "amd/DeepSeek-R1-0528-MXFP4-v2/down_proj", + "amd/DeepSeek-R1-0528-MXFP4/down_proj", + "amd/Kimi-K2.5-MXFP4/down_proj" + ] }, { "type": "gemm", "args": { - "m": 5430, - "n": 576, - "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 4, + "n": 7168, + "k": 2304, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/down_proj" + ] }, { "type": "gemm", "args": { - "m": 5430, - "n": 32768, - "k": 512, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 4, + "n": 7168, + "k": 4096, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/o_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/o_proj" + ] }, { "type": "gemm", "args": { - "m": 5430, + "m": 4, "n": 7168, - "k": 16384, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "k": 4096, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 32 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 1, + 32 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "amd/DeepSeek-R1-0528-MXFP4-Preview/o_proj", + "amd/DeepSeek-R1-0528-MXFP4-v2/o_proj", + "amd/DeepSeek-R1-0528-MXFP4/o_proj" + ] }, { "type": "gemm", "args": { - "m": 5430, - "n": 4608, - "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 4, + "n": 7168, + "k": 4096, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/o_proj", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] }, { "type": "gemm", "args": { - "m": 5430, + "m": 4, "n": 7168, "k": 4608, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "moonshotai/Kimi-K2.5/down_proj", + "nvidia/Kimi-K2.6-NVFP4/down_proj" + ] }, { "type": "gemm", "args": { - "m": 1048576, - "n": 1536, - "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 4, + "n": 7168, + "k": 4608, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/down_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/down_proj", + "nvidia/Kimi-K2.5-NVFP4/down_proj" + ] }, { "type": "gemm", "args": { - "m": 1048576, - "n": 24576, - "k": 1536, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 4, + "n": 7168, + "k": 4608, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 32 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 1, + 32 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "amd/DeepSeek-R1-0528-MXFP4-Preview/down_proj", + "amd/DeepSeek-R1-0528-MXFP4-v2/down_proj", + "amd/DeepSeek-R1-0528-MXFP4/down_proj", + "amd/Kimi-K2.5-MXFP4/down_proj" + ] }, { "type": "gemm", "args": { - "m": 1048576, - "n": 576, - "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 4, + "n": 7168, + "k": 4608, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/down_proj" + ] }, { "type": "gemm", "args": { - "m": 1048576, - "n": 32768, - "k": 512, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 4, + "n": 7168, + "k": 8192, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] }, { "type": "gemm", "args": { - "m": 1048576, + "m": 4, "n": 7168, "k": 16384, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 1048576, - "n": 4608, - "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/o_proj", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] }, { "type": "gemm", "args": { - "m": 1048576, + "m": 4, "n": 7168, - "k": 4608, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "k": 16384, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] }, { "type": "gemm", "args": { - "m": 1024, - "n": 1536, - "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 4, + "n": 7168, + "k": 65536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] }, { "type": "gemm", "args": { - "m": 1024, - "n": 24576, - "k": 1536, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 4, + "n": 8192, + "k": 512, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/Kimi-K2.5-MXFP4/kv_b_proj", + "moonshotai/Kimi-K2.5/kv_b_proj", + "nvidia/DeepSeek-R1-0528-FP4-V2/kv_b_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/kv_b_proj", + "nvidia/Kimi-K2.5-NVFP4/kv_b_proj", + "nvidia/Kimi-K2.6-NVFP4/kv_b_proj" + ] }, { "type": "gemm", "args": { - "m": 1024, - "n": 576, - "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 4, + "n": 8192, + "k": 512, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 32 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 1, + 32 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "amd/DeepSeek-R1-0528-MXFP4-Preview/kv_b_proj", + "amd/DeepSeek-R1-0528-MXFP4-v2/kv_b_proj", + "amd/DeepSeek-R1-0528-MXFP4/kv_b_proj" + ] }, { "type": "gemm", "args": { - "m": 1024, - "n": 32768, + "m": 4, + "n": 8192, "k": 512, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/kv_b_proj" + ] }, { "type": "gemm", "args": { - "m": 1024, - "n": 7168, - "k": 16384, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 4, + "n": 8192, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_b" + ] }, { "type": "gemm", "args": { - "m": 1024, - "n": 4608, - "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 4, + "n": 8192, + "k": 2048, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/GLM-5.1-MXFP4/q_b_proj", + "amd/GLM-5.2-MXFP4/q_b_proj", + "nvidia/GLM-5-NVFP4/q_b_proj", + "nvidia/GLM-5.1-NVFP4/q_b_proj", + "nvidia/GLM-5.2-NVFP4/q_b_proj" + ] }, { "type": "gemm", "args": { - "m": 1024, - "n": 7168, - "k": 4608, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 4, + "n": 12288, + "k": 1536, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/Kimi-K2.5-MXFP4/q_b_proj", + "moonshotai/Kimi-K2.5/q_b_proj", + "nvidia/DeepSeek-R1-0528-FP4-V2/q_b_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/q_b_proj", + "nvidia/Kimi-K2.5-NVFP4/q_b_proj", + "nvidia/Kimi-K2.6-NVFP4/q_b_proj" + ] }, { "type": "gemm", "args": { - "m": 524288, - "n": 1536, - "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 4, + "n": 14336, + "k": 512, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/GLM-5.1-MXFP4/kv_b_proj", + "amd/GLM-5.2-MXFP4/kv_b_proj", + "nvidia/GLM-5-NVFP4/kv_b_proj", + "nvidia/GLM-5.1-NVFP4/kv_b_proj", + "nvidia/GLM-5.2-NVFP4/kv_b_proj" + ] }, { "type": "gemm", "args": { - "m": 524288, - "n": 24576, - "k": 1536, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 4, + "n": 15360, + "k": 1024, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] }, { "type": "gemm", "args": { - "m": 524288, - "n": 576, - "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 4, + "n": 16384, + "k": 512, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/Kimi-K2.5-MXFP4/kv_b_proj", + "moonshotai/Kimi-K2.5/kv_b_proj", + "nvidia/DeepSeek-R1-0528-FP4-V2/kv_b_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/kv_b_proj", + "nvidia/Kimi-K2.5-NVFP4/kv_b_proj", + "nvidia/Kimi-K2.6-NVFP4/kv_b_proj" + ] }, { "type": "gemm", "args": { - "m": 524288, - "n": 32768, - "k": 512, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 4, + "n": 16384, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_b" + ] }, { "type": "gemm", "args": { - "m": 524288, - "n": 7168, - "k": 16384, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 4, + "n": 24576, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/q_b_proj" + ] }, { "type": "gemm", "args": { - "m": 524288, - "n": 2304, - "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 4, + "n": 30720, + "k": 1024, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] }, { "type": "gemm", "args": { - "m": 524288, - "n": 7168, - "k": 2304, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 4, + "n": 32768, + "k": 512, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/kv_b_proj" + ] }, { "type": "gemm", "args": { - "m": 512, - "n": 1536, - "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 4, + "n": 65536, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_b" + ] }, { "type": "gemm", "args": { - "m": 512, - "n": 24576, - "k": 1536, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 4, + "n": 122880, + "k": 1024, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] }, { "type": "gemm", "args": { - "m": 512, + "m": 5, "n": 576, "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/Kimi-K2.5-MXFP4/kv_a_proj_with_mqa", + "moonshotai/Kimi-K2.5/kv_a_proj_with_mqa", + "nvidia/DeepSeek-R1-0528-FP4-V2/kv_a_proj_with_mqa", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/kv_a_proj_with_mqa", + "nvidia/Kimi-K2.5-NVFP4/kv_a_proj_with_mqa", + "nvidia/Kimi-K2.6-NVFP4/kv_a_proj_with_mqa" + ] }, { "type": "gemm", "args": { - "m": 512, - "n": 32768, - "k": 512, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 5, + "n": 576, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/kv_a_proj_with_mqa" + ] }, { "type": "gemm", "args": { - "m": 512, - "n": 7168, - "k": 16384, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 5, + "n": 1536, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/Kimi-K2.5-MXFP4/q_a_proj", + "moonshotai/Kimi-K2.5/q_a_proj", + "nvidia/DeepSeek-R1-0528-FP4-V2/q_a_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/q_a_proj", + "nvidia/Kimi-K2.5-NVFP4/q_a_proj", + "nvidia/Kimi-K2.6-NVFP4/q_a_proj" + ] }, { "type": "gemm", "args": { - "m": 512, - "n": 2304, + "m": 5, + "n": 1536, "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/gate_up_proj", + "deepseek-ai/DeepSeek-R1-0528/q_a_proj", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w3" + ] }, { "type": "gemm", "args": { - "m": 512, - "n": 7168, - "k": 2304, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 5, + "n": 2304, + "k": 7168, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/gate_up_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/gate_up_proj", + "nvidia/Kimi-K2.5-NVFP4/gate_up_proj" + ] }, { "type": "gemm", "args": { - "m": 561152, - "n": 1536, - "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 5, + "n": 3072, + "k": 1536, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/Kimi-K2.5-MXFP4/q_b_proj", + "amd/MiniMax-M2.5-MXFP4/o_proj", + "moonshotai/Kimi-K2.5/q_b_proj", + "nvidia/DeepSeek-R1-0528-FP4-V2/q_b_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/q_b_proj", + "nvidia/Kimi-K2.5-NVFP4/q_b_proj", + "nvidia/Kimi-K2.6-NVFP4/q_b_proj", + "nvidia/MiniMax-M2.5-NVFP4/o_proj" + ] }, { "type": "gemm", "args": { - "m": 561152, - "n": 24576, + "m": 5, + "n": 3072, "k": 1536, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "MiniMaxAI/MiniMax-M2.5/experts.down_proj", + "MiniMaxAI/MiniMax-M2.5/o_proj", + "deepseek-ai/DeepSeek-R1-0528/q_b_proj" + ] }, { "type": "gemm", "args": { - "m": 561152, - "n": 576, - "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 5, + "n": 4096, + "k": 512, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "Qwen/Qwen3.5-397B-A17B/o_proj", + "Qwen/Qwen3.5-397B-A17B/shared_expert.down_proj", + "amd/Kimi-K2.5-MXFP4/kv_b_proj", + "amd/Qwen3.5-397B-A17B-MXFP4/o_proj", + "amd/Qwen3.5-397B-A17B-MXFP4/shared_expert.down_proj", + "moonshotai/Kimi-K2.5/kv_b_proj", + "nvidia/DeepSeek-R1-0528-FP4-V2/kv_b_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/kv_b_proj", + "nvidia/Kimi-K2.5-NVFP4/kv_b_proj", + "nvidia/Kimi-K2.6-NVFP4/kv_b_proj", + "nvidia/Qwen3.5-397B-A17B-NVFP4/o_proj", + "nvidia/Qwen3.5-397B-A17B-NVFP4/shared_expert.down_proj" + ] }, { "type": "gemm", "args": { - "m": 561152, - "n": 32768, + "m": 5, + "n": 4096, "k": 512, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "Qwen/Qwen3.5-397B-A17B-FP8/o_proj", + "Qwen/Qwen3.5-397B-A17B-FP8/shared_expert.down_proj", + "deepseek-ai/DeepSeek-R1-0528/kv_b_proj" + ] }, { "type": "gemm", "args": { - "m": 561152, - "n": 7168, - "k": 16384, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 5, + "n": 4608, + "k": 7168, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/gate_up_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/gate_up_proj", + "nvidia/Kimi-K2.5-NVFP4/gate_up_proj" + ] }, { "type": "gemm", "args": { - "m": 561152, + "m": 5, "n": 4608, "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/gate_up_proj" + ] }, { "type": "gemm", "args": { - "m": 561152, - "n": 7168, - "k": 4608, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 5, + "n": 6144, + "k": 1536, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/GLM-5.1-MXFP4/down_proj", + "amd/GLM-5.2-MXFP4/down_proj", + "amd/Kimi-K2.5-MXFP4/q_b_proj", + "amd/MiniMax-M3-MXFP4/down_proj", + "moonshotai/Kimi-K2.5/q_b_proj", + "nvidia/DeepSeek-R1-0528-FP4-V2/q_b_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/q_b_proj", + "nvidia/GLM-5.1-NVFP4/down_proj", + "nvidia/GLM-5.2-NVFP4/down_proj", + "nvidia/Kimi-K2.5-NVFP4/q_b_proj", + "nvidia/Kimi-K2.6-NVFP4/q_b_proj" + ] }, { "type": "gemm", "args": { - "m": 548, - "n": 1536, - "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 5, + "n": 7168, + "k": 576, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/down_proj" + ] }, { "type": "gemm", "args": { - "m": 548, - "n": 24576, - "k": 1536, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 5, + "n": 7168, + "k": 2048, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/o_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/o_proj" + ] }, { "type": "gemm", "args": { - "m": 548, - "n": 576, - "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 5, + "n": 7168, + "k": 2048, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/o_proj", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] }, { "type": "gemm", "args": { - "m": 548, - "n": 32768, - "k": 512, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 5, + "n": 7168, + "k": 2304, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/down_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/down_proj", + "nvidia/Kimi-K2.5-NVFP4/down_proj" + ] }, { "type": "gemm", "args": { - "m": 548, + "m": 5, "n": 7168, - "k": 16384, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "k": 2304, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/down_proj" + ] }, { "type": "gemm", "args": { - "m": 548, - "n": 4608, - "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 5, + "n": 7168, + "k": 4096, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/o_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/o_proj" + ] }, { "type": "gemm", "args": { - "m": 548, + "m": 5, "n": 7168, "k": 4608, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/down_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/down_proj", + "nvidia/Kimi-K2.5-NVFP4/down_proj" + ] }, { "type": "gemm", "args": { - "m": 280576, - "n": 1536, - "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 5, + "n": 7168, + "k": 9216, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/down_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/down_proj", + "nvidia/Kimi-K2.5-NVFP4/down_proj" + ] }, { "type": "gemm", "args": { - "m": 280576, - "n": 24576, - "k": 1536, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 5, + "n": 7168, + "k": 16384, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/o_proj", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] }, { "type": "gemm", "args": { - "m": 280576, - "n": 576, + "m": 5, + "n": 8192, + "k": 512, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/Kimi-K2.5-MXFP4/kv_b_proj", + "moonshotai/Kimi-K2.5/kv_b_proj", + "nvidia/DeepSeek-R1-0528-FP4-V2/kv_b_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/kv_b_proj", + "nvidia/Kimi-K2.5-NVFP4/kv_b_proj", + "nvidia/Kimi-K2.6-NVFP4/kv_b_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 5, + "n": 9216, "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/gate_up_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/gate_up_proj", + "nvidia/Kimi-K2.5-NVFP4/gate_up_proj" + ] }, { "type": "gemm", "args": { - "m": 280576, - "n": 32768, - "k": 512, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 5, + "n": 24576, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/q_b_proj" + ] }, { "type": "gemm", "args": { - "m": 280576, - "n": 7168, - "k": 16384, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 5, + "n": 32768, + "k": 512, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/kv_b_proj" + ] }, { "type": "gemm", "args": { - "m": 280576, - "n": 2304, + "m": 6, + "n": 576, "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/Kimi-K2.5-MXFP4/kv_a_proj_with_mqa", + "moonshotai/Kimi-K2.5/kv_a_proj_with_mqa", + "nvidia/DeepSeek-R1-0528-FP4-V2/kv_a_proj_with_mqa", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/kv_a_proj_with_mqa", + "nvidia/Kimi-K2.5-NVFP4/kv_a_proj_with_mqa", + "nvidia/Kimi-K2.6-NVFP4/kv_a_proj_with_mqa" + ] }, { "type": "gemm", "args": { - "m": 280576, - "n": 7168, - "k": 2304, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 6, + "n": 576, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/kv_a_proj_with_mqa" + ] }, { "type": "gemm", "args": { - "m": 274, - "n": 1536, + "m": 6, + "n": 1152, "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/gate_up_proj" + ] }, { "type": "gemm", "args": { - "m": 274, - "n": 24576, + "m": 6, + "n": 1536, "k": 1536, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/q_b_proj" + ] }, { "type": "gemm", "args": { - "m": 274, - "n": 576, + "m": 6, + "n": 1536, "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/Kimi-K2.5-MXFP4/q_a_proj", + "moonshotai/Kimi-K2.5/q_a_proj", + "nvidia/DeepSeek-R1-0528-FP4-V2/q_a_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/q_a_proj", + "nvidia/Kimi-K2.5-NVFP4/q_a_proj", + "nvidia/Kimi-K2.6-NVFP4/q_a_proj" + ] }, { "type": "gemm", "args": { - "m": 274, - "n": 32768, - "k": 512, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 6, + "n": 1536, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/gate_up_proj", + "deepseek-ai/DeepSeek-R1-0528/q_a_proj", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w3" + ] }, { "type": "gemm", "args": { - "m": 274, - "n": 7168, - "k": 16384, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 6, + "n": 2048, + "k": 512, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/kv_b_proj" + ] }, { "type": "gemm", "args": { - "m": 274, + "m": 6, "n": 2304, "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 274, - "n": 7168, - "k": 2304, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/gate_up_proj" + ] }, { "type": "gemm", "args": { - "m": 349440, - "n": 1536, - "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 6, + "n": 3072, + "k": 1536, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/Kimi-K2.5-MXFP4/q_b_proj", + "amd/MiniMax-M2.5-MXFP4/o_proj", + "moonshotai/Kimi-K2.5/q_b_proj", + "nvidia/DeepSeek-R1-0528-FP4-V2/q_b_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/q_b_proj", + "nvidia/Kimi-K2.5-NVFP4/q_b_proj", + "nvidia/Kimi-K2.6-NVFP4/q_b_proj", + "nvidia/MiniMax-M2.5-NVFP4/o_proj" + ] }, { "type": "gemm", "args": { - "m": 349440, - "n": 24576, + "m": 6, + "n": 3072, "k": 1536, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "MiniMaxAI/MiniMax-M2.5/experts.down_proj", + "MiniMaxAI/MiniMax-M2.5/o_proj", + "deepseek-ai/DeepSeek-R1-0528/q_b_proj" + ] }, { "type": "gemm", "args": { - "m": 349440, - "n": 576, - "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 6, + "n": 4096, + "k": 512, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "Qwen/Qwen3.5-397B-A17B/o_proj", + "Qwen/Qwen3.5-397B-A17B/shared_expert.down_proj", + "amd/Kimi-K2.5-MXFP4/kv_b_proj", + "amd/Qwen3.5-397B-A17B-MXFP4/o_proj", + "amd/Qwen3.5-397B-A17B-MXFP4/shared_expert.down_proj", + "moonshotai/Kimi-K2.5/kv_b_proj", + "nvidia/DeepSeek-R1-0528-FP4-V2/kv_b_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/kv_b_proj", + "nvidia/Kimi-K2.5-NVFP4/kv_b_proj", + "nvidia/Kimi-K2.6-NVFP4/kv_b_proj", + "nvidia/Qwen3.5-397B-A17B-NVFP4/o_proj", + "nvidia/Qwen3.5-397B-A17B-NVFP4/shared_expert.down_proj" + ] }, { "type": "gemm", "args": { - "m": 349440, - "n": 32768, + "m": 6, + "n": 4096, "k": 512, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "Qwen/Qwen3.5-397B-A17B-FP8/o_proj", + "Qwen/Qwen3.5-397B-A17B-FP8/shared_expert.down_proj", + "deepseek-ai/DeepSeek-R1-0528/kv_b_proj" + ] }, { "type": "gemm", "args": { - "m": 349440, - "n": 7168, - "k": 16384, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 6, + "n": 4608, + "k": 7168, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/gate_up_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/gate_up_proj", + "nvidia/Kimi-K2.5-NVFP4/gate_up_proj" + ] }, { "type": "gemm", "args": { - "m": 349440, + "m": 6, "n": 4608, "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/gate_up_proj" + ] }, { "type": "gemm", "args": { - "m": 349440, + "m": 6, "n": 7168, - "k": 4608, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "k": 1024, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/o_proj", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] }, { "type": "gemm", "args": { - "m": 341, - "n": 1536, - "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 6, + "n": 7168, + "k": 1152, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/down_proj" + ] }, { "type": "gemm", "args": { - "m": 341, - "n": 24576, - "k": 1536, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 6, + "n": 7168, + "k": 2048, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/Kimi-K2.5-MXFP4/o_proj", + "moonshotai/Kimi-K2.5/o_proj", + "nvidia/Kimi-K2.5-NVFP4/o_proj", + "nvidia/Kimi-K2.6-NVFP4/o_proj" + ] }, { "type": "gemm", "args": { - "m": 341, - "n": 576, - "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 6, + "n": 7168, + "k": 2048, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/o_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/o_proj" + ] }, { "type": "gemm", "args": { - "m": 341, - "n": 32768, - "k": 512, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 6, + "n": 7168, + "k": 2048, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/o_proj", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] }, { "type": "gemm", "args": { - "m": 341, + "m": 6, "n": 7168, - "k": 16384, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "k": 2304, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/down_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/down_proj", + "nvidia/Kimi-K2.5-NVFP4/down_proj" + ] }, { "type": "gemm", "args": { - "m": 341, - "n": 4608, - "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 6, + "n": 7168, + "k": 2304, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/down_proj" + ] }, { "type": "gemm", "args": { - "m": 341, + "m": 6, "n": 7168, "k": 4608, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/down_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/down_proj", + "nvidia/Kimi-K2.5-NVFP4/down_proj" + ] }, { "type": "gemm", "args": { - "m": 174720, - "n": 1536, - "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 6, + "n": 7168, + "k": 9216, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/down_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/down_proj", + "nvidia/Kimi-K2.5-NVFP4/down_proj" + ] }, { "type": "gemm", "args": { - "m": 174720, - "n": 24576, - "k": 1536, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 6, + "n": 7168, + "k": 16384, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/o_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/o_proj" + ] }, { "type": "gemm", "args": { - "m": 174720, - "n": 576, - "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 6, + "n": 7168, + "k": 16384, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 32 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 1, + 32 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "amd/DeepSeek-R1-0528-MXFP4-Preview/o_proj", + "amd/DeepSeek-R1-0528-MXFP4-v2/o_proj", + "amd/DeepSeek-R1-0528-MXFP4/o_proj" + ] }, { "type": "gemm", "args": { - "m": 174720, - "n": 32768, - "k": 512, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 6, + "n": 7168, + "k": 16384, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/o_proj", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] }, { "type": "gemm", "args": { - "m": 174720, + "m": 6, "n": 7168, "k": 16384, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] }, { "type": "gemm", "args": { - "m": 174720, - "n": 2304, + "m": 6, + "n": 9216, "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/gate_up_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/gate_up_proj", + "nvidia/Kimi-K2.5-NVFP4/gate_up_proj" + ] }, { "type": "gemm", "args": { - "m": 174720, - "n": 7168, - "k": 2304, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 6, + "n": 24576, + "k": 1536, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/q_b_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/q_b_proj" + ] }, { "type": "gemm", "args": { - "m": 170, - "n": 1536, - "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 6, + "n": 24576, + "k": 1536, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 32 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 1, + 32 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "amd/DeepSeek-R1-0528-MXFP4-Preview/q_b_proj", + "amd/DeepSeek-R1-0528-MXFP4-v2/q_b_proj", + "amd/DeepSeek-R1-0528-MXFP4/q_b_proj" + ] }, { "type": "gemm", "args": { - "m": 170, + "m": 6, "n": 24576, "k": 1536, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/q_b_proj" + ] }, { "type": "gemm", "args": { - "m": 170, - "n": 576, - "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 6, + "n": 32768, + "k": 512, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/kv_b_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/kv_b_proj" + ] }, { "type": "gemm", "args": { - "m": 170, + "m": 6, "n": 32768, "k": 512, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 32 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 1, + 32 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "amd/DeepSeek-R1-0528-MXFP4-Preview/kv_b_proj", + "amd/DeepSeek-R1-0528-MXFP4-v2/kv_b_proj", + "amd/DeepSeek-R1-0528-MXFP4/kv_b_proj" + ] }, { "type": "gemm", "args": { - "m": 170, - "n": 7168, - "k": 16384, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 6, + "n": 32768, + "k": 512, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/kv_b_proj" + ] }, { "type": "gemm", "args": { - "m": 170, - "n": 2304, + "m": 7, + "n": 576, "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/kv_a_proj_with_mqa" + ] }, { "type": "gemm", "args": { - "m": 170, - "n": 7168, - "k": 2304, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 7, + "n": 1152, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/gate_up_proj" + ] }, { "type": "gemm", "args": { - "m": 1, + "m": 7, "n": 1536, "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/gate_up_proj", + "deepseek-ai/DeepSeek-R1-0528/q_a_proj", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w3" + ] }, { "type": "gemm", "args": { - "m": 1, - "n": 24576, - "k": 1536, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 7, + "n": 2304, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/gate_up_proj" + ] }, { "type": "gemm", "args": { - "m": 1, - "n": 576, + "m": 7, + "n": 4608, "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/gate_up_proj" + ] }, { "type": "gemm", "args": { - "m": 1, - "n": 32768, - "k": 512, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 7, + "n": 7168, + "k": 1152, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/down_proj" + ] }, { "type": "gemm", "args": { - "m": 1, + "m": 7, "n": 7168, - "k": 16384, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "k": 2304, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/down_proj" + ] }, { "type": "gemm", "args": { - "m": 1, - "n": 4608, - "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 7, + "n": 7168, + "k": 4608, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/down_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/down_proj", + "nvidia/Kimi-K2.5-NVFP4/down_proj" + ] }, { "type": "gemm", "args": { - "m": 1, + "m": 7, "n": 7168, "k": 4608, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 32 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 1, + 32 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "amd/DeepSeek-R1-0528-MXFP4-Preview/down_proj", + "amd/DeepSeek-R1-0528-MXFP4-v2/down_proj", + "amd/DeepSeek-R1-0528-MXFP4/down_proj", + "amd/Kimi-K2.5-MXFP4/down_proj" + ] }, { "type": "gemm", "args": { - "m": 6, - "n": 1536, - "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 7, + "n": 7168, + "k": 8192, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/Kimi-K2.5-MXFP4/o_proj", + "moonshotai/Kimi-K2.5/o_proj", + "nvidia/Kimi-K2.5-NVFP4/o_proj", + "nvidia/Kimi-K2.6-NVFP4/o_proj" + ] }, { "type": "gemm", "args": { - "m": 6, - "n": 3072, - "k": 1536, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 7, + "n": 7168, + "k": 8192, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] }, { "type": "gemm", "args": { - "m": 6, - "n": 576, - "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 7, + "n": 7168, + "k": 9216, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/down_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/down_proj", + "nvidia/Kimi-K2.5-NVFP4/down_proj" + ] }, { "type": "gemm", "args": { - "m": 6, - "n": 4096, - "k": 512, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 7, + "n": 7168, + "k": 16384, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/o_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/o_proj" + ] }, { "type": "gemm", "args": { - "m": 6, + "m": 7, "n": 7168, - "k": 2048, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "k": 16384, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 32 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 1, + 32 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "amd/DeepSeek-R1-0528-MXFP4-Preview/o_proj", + "amd/DeepSeek-R1-0528-MXFP4-v2/o_proj", + "amd/DeepSeek-R1-0528-MXFP4/o_proj" + ] }, { "type": "gemm", "args": { - "m": 6, - "n": 2304, - "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 7, + "n": 7168, + "k": 16384, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/o_proj", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] }, { "type": "gemm", "args": { - "m": 6, + "m": 7, "n": 7168, - "k": 2304, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "k": 16384, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] }, { "type": "gemm", "args": { - "m": 2, - "n": 1536, + "m": 7, + "n": 9216, "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 2, - "n": 24576, - "k": 1536, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/gate_up_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/gate_up_proj", + "nvidia/Kimi-K2.5-NVFP4/gate_up_proj" + ] }, { "type": "gemm", "args": { - "m": 2, - "n": 576, + "m": 7, + "n": 9216, "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 32 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 1, + 32 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "amd/DeepSeek-R1-0528-MXFP4-Preview/gate_up_proj", + "amd/DeepSeek-R1-0528-MXFP4-v2/gate_up_proj", + "amd/DeepSeek-R1-0528-MXFP4/gate_up_proj", + "amd/Kimi-K2.5-MXFP4/gate_up_proj" + ] }, { "type": "gemm", "args": { - "m": 2, - "n": 32768, - "k": 512, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 7, + "n": 12288, + "k": 1536, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/Kimi-K2.5-MXFP4/q_b_proj", + "moonshotai/Kimi-K2.5/q_b_proj", + "nvidia/DeepSeek-R1-0528-FP4-V2/q_b_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/q_b_proj", + "nvidia/Kimi-K2.5-NVFP4/q_b_proj", + "nvidia/Kimi-K2.6-NVFP4/q_b_proj" + ] }, { "type": "gemm", "args": { - "m": 2, - "n": 7168, - "k": 16384, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 7, + "n": 16384, + "k": 512, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/Kimi-K2.5-MXFP4/kv_b_proj", + "moonshotai/Kimi-K2.5/kv_b_proj", + "nvidia/DeepSeek-R1-0528-FP4-V2/kv_b_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/kv_b_proj", + "nvidia/Kimi-K2.5-NVFP4/kv_b_proj", + "nvidia/Kimi-K2.6-NVFP4/kv_b_proj" + ] }, { "type": "gemm", "args": { - "m": 2, - "n": 4608, - "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 7, + "n": 24576, + "k": 1536, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/q_b_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/q_b_proj" + ] }, { "type": "gemm", "args": { - "m": 2, - "n": 7168, - "k": 4608, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 7, + "n": 24576, + "k": 1536, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 32 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 1, + 32 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "amd/DeepSeek-R1-0528-MXFP4-Preview/q_b_proj", + "amd/DeepSeek-R1-0528-MXFP4-v2/q_b_proj", + "amd/DeepSeek-R1-0528-MXFP4/q_b_proj" + ] }, { "type": "gemm", "args": { - "m": 6, + "m": 7, "n": 24576, "k": 1536, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/q_b_proj" + ] }, { "type": "gemm", "args": { - "m": 6, + "m": 7, "n": 32768, "k": 512, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/kv_b_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/kv_b_proj" + ] }, { "type": "gemm", "args": { - "m": 6, - "n": 7168, - "k": 16384, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 7, + "n": 32768, + "k": 512, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 32 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 1, + 32 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "amd/DeepSeek-R1-0528-MXFP4-Preview/kv_b_proj", + "amd/DeepSeek-R1-0528-MXFP4-v2/kv_b_proj", + "amd/DeepSeek-R1-0528-MXFP4/kv_b_proj" + ] }, { "type": "gemm", "args": { - "m": 6, - "n": 4608, - "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 7, + "n": 32768, + "k": 512, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/kv_b_proj" + ] }, { "type": "gemm", "args": { - "m": 6, - "n": 7168, - "k": 4608, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 8, + "n": 64, + "k": 2880, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/gpt-oss-120b-w-mxfp4-a-fp8/k_proj", + "amd/gpt-oss-120b-w-mxfp4-a-fp8/v_proj", + "openai/gpt-oss-120b/k_proj", + "openai/gpt-oss-120b/v_proj" + ] }, { "type": "gemm", "args": { - "m": 11, - "n": 1536, - "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 8, + "n": 128, + "k": 2880, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/gpt-oss-120b-w-mxfp4-a-fp8/k_proj", + "amd/gpt-oss-120b-w-mxfp4-a-fp8/v_proj", + "openai/gpt-oss-120b/k_proj", + "openai/gpt-oss-120b/v_proj" + ] }, { "type": "gemm", "args": { - "m": 11, - "n": 24576, - "k": 1536, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 8, + "n": 128, + "k": 3072, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/MiniMax-M2.5-MXFP4/k_proj", + "amd/MiniMax-M2.5-MXFP4/v_proj", + "nvidia/MiniMax-M2.5-NVFP4/k_proj", + "nvidia/MiniMax-M2.5-NVFP4/v_proj" + ] }, { "type": "gemm", "args": { - "m": 11, - "n": 576, - "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 8, + "n": 128, + "k": 3072, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "MiniMaxAI/MiniMax-M2.5/k_proj", + "MiniMaxAI/MiniMax-M2.5/v_proj" + ] }, { "type": "gemm", "args": { - "m": 11, - "n": 32768, - "k": 512, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 8, + "n": 128, + "k": 4096, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "Qwen/Qwen3.5-397B-A17B/shared_expert.gate_up_proj", + "amd/Qwen3.5-397B-A17B-MXFP4/shared_expert.gate_up_proj", + "nvidia/Qwen3.5-397B-A17B-NVFP4/shared_expert.gate_up_proj" + ] }, { "type": "gemm", "args": { - "m": 11, - "n": 7168, - "k": 16384, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 8, + "n": 128, + "k": 4096, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "Qwen/Qwen3.5-397B-A17B-FP8/shared_expert.gate_up_proj" + ] }, { "type": "gemm", "args": { - "m": 11, - "n": 4608, - "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 8, + "n": 192, + "k": 3072, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/MiniMax-M2.5-NVFP4/experts.gate_up_proj" + ] }, { "type": "gemm", "args": { - "m": 11, - "n": 7168, - "k": 4608, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 8, + "n": 192, + "k": 3072, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 32 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 1, + 32 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "amd/MiniMax-M2.5-MXFP4/experts.gate_up_proj" + ] }, { "type": "gemm", "args": { - "m": 10, - "n": 1536, - "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 8, + "n": 192, + "k": 3072, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "MiniMaxAI/MiniMax-M2.5/experts.gate_up_proj" + ] }, { "type": "gemm", "args": { - "m": 10, - "n": 3072, - "k": 1536, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 8, + "n": 256, + "k": 2880, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/gpt-oss-120b-w-mxfp4-a-fp8/k_proj", + "amd/gpt-oss-120b-w-mxfp4-a-fp8/q_proj", + "amd/gpt-oss-120b-w-mxfp4-a-fp8/v_proj", + "openai/gpt-oss-120b/k_proj", + "openai/gpt-oss-120b/q_proj", + "openai/gpt-oss-120b/v_proj" + ] }, { "type": "gemm", "args": { - "m": 10, - "n": 576, - "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 8, + "n": 256, + "k": 3072, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/MiniMax-M2.5-MXFP4/k_proj", + "amd/MiniMax-M2.5-MXFP4/v_proj", + "nvidia/MiniMax-M2.5-NVFP4/k_proj", + "nvidia/MiniMax-M2.5-NVFP4/v_proj" + ] }, { "type": "gemm", "args": { - "m": 10, - "n": 4096, - "k": 512, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 8, + "n": 256, + "k": 3072, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "MiniMaxAI/MiniMax-M2.5/k_proj", + "MiniMaxAI/MiniMax-M2.5/v_proj" + ] }, { "type": "gemm", "args": { - "m": 10, - "n": 7168, - "k": 2048, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 8, + "n": 256, + "k": 4096, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "Qwen/Qwen3.5-397B-A17B/k_proj", + "Qwen/Qwen3.5-397B-A17B/shared_expert.gate_up_proj", + "Qwen/Qwen3.5-397B-A17B/v_proj", + "amd/Qwen3.5-397B-A17B-MXFP4/k_proj", + "amd/Qwen3.5-397B-A17B-MXFP4/shared_expert.gate_up_proj", + "amd/Qwen3.5-397B-A17B-MXFP4/v_proj", + "nvidia/Qwen3.5-397B-A17B-NVFP4/k_proj", + "nvidia/Qwen3.5-397B-A17B-NVFP4/shared_expert.gate_up_proj", + "nvidia/Qwen3.5-397B-A17B-NVFP4/v_proj" + ] }, { "type": "gemm", "args": { - "m": 10, - "n": 2304, - "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 8, + "n": 256, + "k": 4096, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "Qwen/Qwen3.5-397B-A17B-FP8/k_proj", + "Qwen/Qwen3.5-397B-A17B-FP8/shared_expert.gate_up_proj", + "Qwen/Qwen3.5-397B-A17B-FP8/v_proj" + ] }, { "type": "gemm", "args": { - "m": 10, - "n": 7168, - "k": 2304, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 8, + "n": 384, + "k": 3072, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/MiniMax-M2.5-NVFP4/experts.gate_up_proj" + ] }, { "type": "gemm", "args": { - "m": 15, - "n": 1536, - "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 8, + "n": 384, + "k": 3072, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 32 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 1, + 32 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "amd/MiniMax-M2.5-MXFP4/experts.gate_up_proj" + ] }, { "type": "gemm", "args": { - "m": 15, - "n": 3072, - "k": 1536, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 8, + "n": 384, + "k": 3072, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "MiniMaxAI/MiniMax-M2.5/experts.gate_up_proj", + "MiniMaxAI/MiniMax-M2.5/q_proj" + ] }, { "type": "gemm", "args": { - "m": 15, - "n": 576, - "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 8, + "n": 512, + "k": 2880, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/gpt-oss-120b-w-mxfp4-a-fp8/k_proj", + "amd/gpt-oss-120b-w-mxfp4-a-fp8/q_proj", + "amd/gpt-oss-120b-w-mxfp4-a-fp8/v_proj", + "openai/gpt-oss-120b/k_proj", + "openai/gpt-oss-120b/q_proj", + "openai/gpt-oss-120b/v_proj" + ] }, { "type": "gemm", "args": { - "m": 15, - "n": 4096, - "k": 512, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 8, + "n": 512, + "k": 3072, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/MiniMax-M2.5-MXFP4/k_proj", + "amd/MiniMax-M2.5-MXFP4/v_proj", + "nvidia/MiniMax-M2.5-NVFP4/k_proj", + "nvidia/MiniMax-M2.5-NVFP4/v_proj" + ] }, { "type": "gemm", "args": { - "m": 15, - "n": 7168, - "k": 2048, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 8, + "n": 512, + "k": 3072, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "MiniMaxAI/MiniMax-M2.5/k_proj", + "MiniMaxAI/MiniMax-M2.5/v_proj" + ] }, { "type": "gemm", "args": { - "m": 15, - "n": 2304, - "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 8, + "n": 512, + "k": 4096, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "Qwen/Qwen3.5-397B-A17B/k_proj", + "Qwen/Qwen3.5-397B-A17B/q_proj", + "Qwen/Qwen3.5-397B-A17B/shared_expert.gate_up_proj", + "Qwen/Qwen3.5-397B-A17B/v_proj", + "amd/Qwen3.5-397B-A17B-MXFP4/k_proj", + "amd/Qwen3.5-397B-A17B-MXFP4/q_proj", + "amd/Qwen3.5-397B-A17B-MXFP4/shared_expert.gate_up_proj", + "amd/Qwen3.5-397B-A17B-MXFP4/v_proj", + "nvidia/Qwen3.5-397B-A17B-NVFP4/k_proj", + "nvidia/Qwen3.5-397B-A17B-NVFP4/q_proj", + "nvidia/Qwen3.5-397B-A17B-NVFP4/shared_expert.gate_up_proj", + "nvidia/Qwen3.5-397B-A17B-NVFP4/v_proj" + ] }, { "type": "gemm", "args": { - "m": 15, - "n": 7168, - "k": 2304, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 8, + "n": 512, + "k": 4096, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "Qwen/Qwen3.5-397B-A17B-FP8/k_proj", + "Qwen/Qwen3.5-397B-A17B-FP8/q_proj", + "Qwen/Qwen3.5-397B-A17B-FP8/shared_expert.gate_up_proj", + "Qwen/Qwen3.5-397B-A17B-FP8/v_proj", + "deepseek-ai/DeepSeek-V4.1-Flash/attn.wo_a" + ] }, { "type": "gemm", "args": { - "m": 25, - "n": 1536, - "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 8, + "n": 576, + "k": 6144, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/GLM-5.1-MXFP4/kv_a_proj_with_mqa", + "amd/GLM-5.2-MXFP4/kv_a_proj_with_mqa", + "nvidia/GLM-5-NVFP4/kv_a_proj_with_mqa", + "nvidia/GLM-5.1-NVFP4/kv_a_proj_with_mqa", + "nvidia/GLM-5.2-NVFP4/kv_a_proj_with_mqa" + ] }, { "type": "gemm", "args": { - "m": 25, - "n": 3072, - "k": 1536, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 8, + "n": 576, + "k": 6144, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "zai-org/GLM-5-FP8/kv_a_proj_with_mqa", + "zai-org/GLM-5.1-FP8/kv_a_proj_with_mqa", + "zai-org/GLM-5.2-FP8/kv_a_proj_with_mqa" + ] }, { "type": "gemm", "args": { - "m": 25, + "m": 8, "n": 576, "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/Kimi-K2.5-MXFP4/kv_a_proj_with_mqa", + "moonshotai/Kimi-K2.5/kv_a_proj_with_mqa", + "nvidia/DeepSeek-R1-0528-FP4-V2/kv_a_proj_with_mqa", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/kv_a_proj_with_mqa", + "nvidia/Kimi-K2.5-NVFP4/kv_a_proj_with_mqa", + "nvidia/Kimi-K2.6-NVFP4/kv_a_proj_with_mqa" + ] }, { "type": "gemm", "args": { - "m": 25, - "n": 4096, - "k": 512, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 25, - "n": 7168, - "k": 2048, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 8, + "n": 576, + "k": 7168, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 32 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 1, + 32 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "amd/DeepSeek-R1-0528-MXFP4-Preview/kv_a_proj_with_mqa", + "amd/DeepSeek-R1-0528-MXFP4-v2/kv_a_proj_with_mqa", + "amd/DeepSeek-R1-0528-MXFP4/kv_a_proj_with_mqa" + ] }, { "type": "gemm", "args": { - "m": 25, - "n": 2304, + "m": 8, + "n": 576, "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/kv_a_proj_with_mqa" + ] }, { "type": "gemm", "args": { - "m": 25, - "n": 7168, - "k": 2304, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 8, + "n": 768, + "k": 3072, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/MiniMax-M2.5-NVFP4/experts.gate_up_proj" + ] }, { "type": "gemm", "args": { - "m": 45, - "n": 3072, - "k": 1536, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 8, + "n": 768, + "k": 3072, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 32 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 1, + 32 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "amd/MiniMax-M2.5-MXFP4/experts.gate_up_proj" + ] }, { "type": "gemm", "args": { - "m": 45, - "n": 4096, - "k": 512, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 8, + "n": 768, + "k": 3072, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "MiniMaxAI/MiniMax-M2.5/experts.gate_up_proj", + "MiniMaxAI/MiniMax-M2.5/q_proj" + ] }, { "type": "gemm", "args": { - "m": 45, - "n": 7168, - "k": 2048, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 8, + "n": 1024, + "k": 2880, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/gpt-oss-120b-w-mxfp4-a-fp8/q_proj", + "openai/gpt-oss-120b/q_proj" + ] }, { "type": "gemm", "args": { - "m": 45, - "n": 2304, - "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 8, + "n": 1024, + "k": 3072, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/MiniMax-M2.5-MXFP4/k_proj", + "amd/MiniMax-M2.5-MXFP4/qkv_proj", + "amd/MiniMax-M2.5-MXFP4/v_proj", + "nvidia/MiniMax-M2.5-NVFP4/k_proj", + "nvidia/MiniMax-M2.5-NVFP4/qkv_proj", + "nvidia/MiniMax-M2.5-NVFP4/v_proj" + ] }, { "type": "gemm", "args": { - "m": 45, - "n": 7168, - "k": 2304, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 8, + "n": 1024, + "k": 3072, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "MiniMaxAI/MiniMax-M2.5/k_proj", + "MiniMaxAI/MiniMax-M2.5/qkv_proj", + "MiniMaxAI/MiniMax-M2.5/v_proj" + ] }, { "type": "gemm", "args": { - "m": 115200, - "n": 1536, - "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 8, + "n": 1024, + "k": 4096, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "Qwen/Qwen3.5-397B-A17B/q_proj", + "Qwen/Qwen3.5-397B-A17B/shared_expert.gate_up_proj", + "amd/Qwen3.5-397B-A17B-MXFP4/q_proj", + "amd/Qwen3.5-397B-A17B-MXFP4/shared_expert.gate_up_proj", + "nvidia/Qwen3.5-397B-A17B-NVFP4/q_proj", + "nvidia/Qwen3.5-397B-A17B-NVFP4/shared_expert.gate_up_proj" + ] }, { "type": "gemm", "args": { - "m": 115200, - "n": 24576, - "k": 1536, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 8, + "n": 1024, + "k": 4096, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "Qwen/Qwen3.5-397B-A17B-FP8/q_proj", + "Qwen/Qwen3.5-397B-A17B-FP8/shared_expert.gate_up_proj", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_a", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_a", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_a" + ] }, { "type": "gemm", "args": { - "m": 115200, - "n": 576, + "m": 8, + "n": 1088, "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] }, { "type": "gemm", "args": { - "m": 115200, - "n": 32768, - "k": 512, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 8, + "n": 1152, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/gate_up_proj" + ] }, { "type": "gemm", "args": { - "m": 115200, - "n": 7168, - "k": 16384, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 8, + "n": 1536, + "k": 1536, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/Kimi-K2.5-MXFP4/q_b_proj", + "moonshotai/Kimi-K2.5/q_b_proj", + "nvidia/DeepSeek-R1-0528-FP4-V2/q_b_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/q_b_proj", + "nvidia/Kimi-K2.5-NVFP4/q_b_proj", + "nvidia/Kimi-K2.6-NVFP4/q_b_proj" + ] }, { "type": "gemm", "args": { - "m": 115200, - "n": 4608, - "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 8, + "n": 1536, + "k": 3072, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 32 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 1, + 32 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "amd/MiniMax-M2.5-MXFP4/experts.gate_up_proj" + ] }, { "type": "gemm", "args": { - "m": 115200, - "n": 7168, - "k": 4608, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 8, + "n": 1536, + "k": 3072, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "MiniMaxAI/MiniMax-M2.5/experts.gate_up_proj", + "MiniMaxAI/MiniMax-M2.5/q_proj" + ] }, { "type": "gemm", "args": { - "m": 112, + "m": 8, "n": 1536, - "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "k": 6144, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/GLM-5-NVFP4/gate_up_proj" + ] }, { "type": "gemm", "args": { - "m": 112, - "n": 24576, - "k": 1536, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 8, + "n": 1536, + "k": 6144, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "zai-org/GLM-5-FP8/gate_up_proj", + "zai-org/GLM-5.1-FP8/gate_up_proj", + "zai-org/GLM-5.2-FP8/gate_up_proj" + ] }, { "type": "gemm", "args": { - "m": 112, - "n": 576, + "m": 8, + "n": 1536, "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/Kimi-K2.5-MXFP4/q_a_proj", + "moonshotai/Kimi-K2.5/q_a_proj", + "nvidia/DeepSeek-R1-0528-FP4-V2/q_a_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/q_a_proj", + "nvidia/Kimi-K2.5-NVFP4/q_a_proj", + "nvidia/Kimi-K2.6-NVFP4/q_a_proj" + ] }, { "type": "gemm", "args": { - "m": 112, - "n": 32768, - "k": 512, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 8, + "n": 1536, + "k": 7168, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 32 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 1, + 32 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "amd/DeepSeek-R1-0528-MXFP4-Preview/q_a_proj", + "amd/DeepSeek-R1-0528-MXFP4-v2/q_a_proj", + "amd/DeepSeek-R1-0528-MXFP4/q_a_proj" + ] }, { "type": "gemm", "args": { - "m": 112, - "n": 7168, - "k": 16384, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 8, + "n": 1536, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/gate_up_proj", + "deepseek-ai/DeepSeek-R1-0528/q_a_proj", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w3" + ] }, { "type": "gemm", "args": { - "m": 112, - "n": 4608, + "m": 8, + "n": 1536, "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w3" + ] }, { "type": "gemm", "args": { - "m": 112, - "n": 7168, - "k": 4608, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 8, + "n": 2048, + "k": 512, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/Kimi-K2.5-MXFP4/kv_b_proj", + "moonshotai/Kimi-K2.5/kv_b_proj", + "nvidia/DeepSeek-R1-0528-FP4-V2/kv_b_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/kv_b_proj", + "nvidia/Kimi-K2.5-NVFP4/kv_b_proj", + "nvidia/Kimi-K2.6-NVFP4/kv_b_proj" + ] }, { "type": "gemm", "args": { - "m": 460800, - "n": 1536, - "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 8, + "n": 2048, + "k": 2048, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/GLM-5.1-MXFP4/q_b_proj", + "amd/GLM-5.2-MXFP4/q_b_proj", + "nvidia/GLM-5-NVFP4/q_b_proj", + "nvidia/GLM-5.1-NVFP4/q_b_proj", + "nvidia/GLM-5.2-NVFP4/q_b_proj" + ] }, { "type": "gemm", "args": { - "m": 460800, - "n": 3072, - "k": 1536, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 8, + "n": 2048, + "k": 2048, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "zai-org/GLM-5-FP8/q_b_proj", + "zai-org/GLM-5.1-FP8/q_b_proj", + "zai-org/GLM-5.2-FP8/q_b_proj" + ] }, { "type": "gemm", "args": { - "m": 460800, - "n": 576, - "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 8, + "n": 2048, + "k": 2880, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/gpt-oss-120b-w-mxfp4-a-fp8/q_proj", + "openai/gpt-oss-120b/q_proj" + ] }, { "type": "gemm", "args": { - "m": 460800, - "n": 4096, - "k": 512, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 8, + "n": 2048, + "k": 4096, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "Qwen/Qwen3.5-397B-A17B/q_proj", + "Qwen/Qwen3.5-397B-A17B/shared_expert.gate_up_proj", + "amd/Qwen3.5-397B-A17B-MXFP4/q_proj", + "amd/Qwen3.5-397B-A17B-MXFP4/shared_expert.gate_up_proj", + "nvidia/Qwen3.5-397B-A17B-NVFP4/q_proj", + "nvidia/Qwen3.5-397B-A17B-NVFP4/shared_expert.gate_up_proj" + ] }, { "type": "gemm", "args": { - "m": 460800, - "n": 7168, - "k": 2048, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 8, + "n": 2048, + "k": 4096, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "Qwen/Qwen3.5-397B-A17B-FP8/q_proj", + "Qwen/Qwen3.5-397B-A17B-FP8/shared_expert.gate_up_proj" + ] }, { "type": "gemm", "args": { - "m": 460800, - "n": 2304, - "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 8, + "n": 2048, + "k": 6144, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/GLM-5.1-MXFP4/q_a_proj", + "amd/GLM-5.2-MXFP4/q_a_proj", + "amd/MiniMax-M3-MXFP4/q_proj", + "nvidia/GLM-5-NVFP4/q_a_proj", + "nvidia/GLM-5.1-NVFP4/q_a_proj", + "nvidia/GLM-5.2-NVFP4/q_a_proj" + ] }, { "type": "gemm", "args": { - "m": 460800, - "n": 7168, - "k": 2304, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 8, + "n": 2048, + "k": 6144, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "zai-org/GLM-5-FP8/q_a_proj", + "zai-org/GLM-5.1-FP8/q_a_proj", + "zai-org/GLM-5.2-FP8/q_a_proj" + ] }, { "type": "gemm", "args": { - "m": 450, - "n": 1536, + "m": 8, + "n": 2304, "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 450, - "n": 3072, - "k": 1536, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/gate_up_proj" + ] }, { "type": "gemm", "args": { - "m": 450, - "n": 576, - "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 8, + "n": 2880, + "k": 512, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/gpt-oss-120b-w-mxfp4-a-fp8/o_proj", + "openai/gpt-oss-120b/o_proj" + ] }, { "type": "gemm", "args": { - "m": 450, - "n": 4096, - "k": 512, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 8, + "n": 2880, + "k": 1024, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/gpt-oss-120b-w-mxfp4-a-fp8/o_proj", + "openai/gpt-oss-120b/o_proj" + ] }, { "type": "gemm", "args": { - "m": 450, - "n": 7168, + "m": 8, + "n": 2880, "k": 2048, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/gpt-oss-120b-w-mxfp4-a-fp8/o_proj", + "openai/gpt-oss-120b/o_proj" + ] }, { "type": "gemm", "args": { - "m": 450, - "n": 2304, - "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 8, + "n": 2880, + "k": 4096, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/gpt-oss-120b-w-mxfp4-a-fp8/o_proj", + "openai/gpt-oss-120b/o_proj" + ] }, { "type": "gemm", "args": { - "m": 450, - "n": 7168, - "k": 2304, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 8, + "n": 3072, + "k": 192, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/MiniMax-M2.5-NVFP4/experts.down_proj" + ] }, { "type": "gemm", "args": { - "m": 184320, - "n": 24576, - "k": 1536, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 8, + "n": 3072, + "k": 192, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 32 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 1, + 32 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "amd/MiniMax-M2.5-MXFP4/experts.down_proj" + ] }, { "type": "gemm", "args": { - "m": 184320, - "n": 32768, - "k": 512, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 8, + "n": 3072, + "k": 192, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "MiniMaxAI/MiniMax-M2.5/experts.down_proj" + ] }, { "type": "gemm", "args": { - "m": 184320, - "n": 7168, - "k": 16384, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 8, + "n": 3072, + "k": 384, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/MiniMax-M2.5-NVFP4/experts.down_proj" + ] }, { "type": "gemm", "args": { - "m": 184320, - "n": 4608, - "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 8, + "n": 3072, + "k": 384, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 32 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 1, + 32 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "amd/MiniMax-M2.5-MXFP4/experts.down_proj" + ] }, { "type": "gemm", "args": { - "m": 184320, - "n": 7168, - "k": 4608, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 8, + "n": 3072, + "k": 384, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "MiniMaxAI/MiniMax-M2.5/experts.down_proj", + "MiniMaxAI/MiniMax-M2.5/o_proj" + ] }, { "type": "gemm", "args": { - "m": 92160, - "n": 24576, - "k": 1536, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 8, + "n": 3072, + "k": 768, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/MiniMax-M2.5-MXFP4/o_proj", + "nvidia/MiniMax-M2.5-NVFP4/o_proj" + ] }, { "type": "gemm", "args": { - "m": 92160, - "n": 32768, - "k": 512, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 8, + "n": 3072, + "k": 768, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 32 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 1, + 32 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "amd/MiniMax-M2.5-MXFP4/experts.down_proj" + ] }, { "type": "gemm", "args": { - "m": 92160, - "n": 7168, - "k": 16384, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 8, + "n": 3072, + "k": 768, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "MiniMaxAI/MiniMax-M2.5/experts.down_proj", + "MiniMaxAI/MiniMax-M2.5/o_proj" + ] }, { "type": "gemm", "args": { - "m": 22, - "n": 2304, - "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 8, + "n": 3072, + "k": 1536, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/Kimi-K2.5-MXFP4/q_b_proj", + "amd/MiniMax-M2.5-MXFP4/o_proj", + "moonshotai/Kimi-K2.5/q_b_proj", + "nvidia/DeepSeek-R1-0528-FP4-V2/q_b_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/q_b_proj", + "nvidia/Kimi-K2.5-NVFP4/q_b_proj", + "nvidia/Kimi-K2.6-NVFP4/q_b_proj", + "nvidia/MiniMax-M2.5-NVFP4/o_proj" + ] }, { "type": "gemm", "args": { - "m": 22, - "n": 7168, - "k": 2304, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 8, + "n": 3072, + "k": 1536, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/MiniMax-M2.5-NVFP4/experts.down_proj" + ] }, { "type": "gemm", "args": { - "m": 135168, - "n": 1536, - "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 8, + "n": 3072, + "k": 1536, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 32 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 1, + 32 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "amd/DeepSeek-R1-0528-MXFP4-Preview/q_b_proj", + "amd/DeepSeek-R1-0528-MXFP4-v2/q_b_proj", + "amd/DeepSeek-R1-0528-MXFP4/q_b_proj", + "amd/MiniMax-M2.5-MXFP4/experts.down_proj" + ] }, { "type": "gemm", "args": { - "m": 135168, - "n": 24576, + "m": 8, + "n": 3072, "k": 1536, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "MiniMaxAI/MiniMax-M2.5/experts.down_proj", + "MiniMaxAI/MiniMax-M2.5/o_proj", + "deepseek-ai/DeepSeek-R1-0528/q_b_proj" + ] }, { "type": "gemm", "args": { - "m": 135168, - "n": 576, - "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 8, + "n": 3072, + "k": 3072, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/MiniMax-M2.5-MXFP4/o_proj", + "amd/MiniMax-M2.5-MXFP4/q_proj", + "nvidia/MiniMax-M2.5-NVFP4/o_proj", + "nvidia/MiniMax-M2.5-NVFP4/q_proj" + ] }, { "type": "gemm", "args": { - "m": 135168, - "n": 32768, - "k": 512, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 8, + "n": 3072, + "k": 3072, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/MiniMax-M2.5-NVFP4/experts.gate_up_proj" + ] }, { "type": "gemm", "args": { - "m": 135168, - "n": 7168, - "k": 16384, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 8, + "n": 3072, + "k": 3072, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 32 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 1, + 32 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "amd/MiniMax-M2.5-MXFP4/experts.gate_up_proj" + ] }, { "type": "gemm", "args": { - "m": 135168, - "n": 4608, - "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 8, + "n": 3072, + "k": 3072, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "MiniMaxAI/MiniMax-M2.5/experts.gate_up_proj", + "MiniMaxAI/MiniMax-M2.5/o_proj", + "MiniMaxAI/MiniMax-M2.5/q_proj" + ] }, { "type": "gemm", "args": { - "m": 135168, - "n": 7168, - "k": 4608, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 8, + "n": 3072, + "k": 6144, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/GLM-5.1-MXFP4/gate_up_proj", + "amd/GLM-5.2-MXFP4/gate_up_proj", + "amd/MiniMax-M2.5-MXFP4/o_proj", + "amd/MiniMax-M3-MXFP4/gate_up_proj", + "nvidia/GLM-5.1-NVFP4/gate_up_proj", + "nvidia/GLM-5.2-NVFP4/gate_up_proj", + "nvidia/MiniMax-M2.5-NVFP4/o_proj" + ] }, { "type": "gemm", "args": { - "m": 33, - "n": 1536, - "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 8, + "n": 3072, + "k": 6144, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/GLM-5-NVFP4/gate_up_proj" + ] }, { "type": "gemm", "args": { - "m": 33, - "n": 24576, - "k": 1536, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 8, + "n": 3072, + "k": 6144, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "MiniMaxAI/MiniMax-M2.5/o_proj", + "zai-org/GLM-5-FP8/gate_up_proj", + "zai-org/GLM-5.1-FP8/gate_up_proj", + "zai-org/GLM-5.2-FP8/gate_up_proj" + ] }, { "type": "gemm", "args": { - "m": 33, - "n": 576, - "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 8, + "n": 3584, + "k": 512, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/GLM-5.1-MXFP4/kv_b_proj", + "amd/GLM-5.2-MXFP4/kv_b_proj", + "nvidia/GLM-5-NVFP4/kv_b_proj", + "nvidia/GLM-5.1-NVFP4/kv_b_proj", + "nvidia/GLM-5.2-NVFP4/kv_b_proj" + ] }, { "type": "gemm", "args": { - "m": 33, - "n": 32768, + "m": 8, + "n": 3584, "k": 512, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "zai-org/GLM-5-FP8/kv_b_proj", + "zai-org/GLM-5.1-FP8/kv_b_proj", + "zai-org/GLM-5.2-FP8/kv_b_proj" + ] }, { "type": "gemm", "args": { - "m": 33, - "n": 7168, - "k": 16384, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 8, + "n": 4096, + "k": 128, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "Qwen/Qwen3.5-397B-A17B/shared_expert.down_proj", + "amd/Qwen3.5-397B-A17B-MXFP4/shared_expert.down_proj", + "nvidia/Qwen3.5-397B-A17B-NVFP4/shared_expert.down_proj" + ] }, { "type": "gemm", "args": { - "m": 33, - "n": 4608, - "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 8, + "n": 4096, + "k": 128, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "Qwen/Qwen3.5-397B-A17B-FP8/shared_expert.down_proj" + ] }, { "type": "gemm", "args": { - "m": 33, - "n": 7168, - "k": 4608, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 8, + "n": 4096, + "k": 256, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "Qwen/Qwen3.5-397B-A17B/shared_expert.down_proj", + "amd/Qwen3.5-397B-A17B-MXFP4/shared_expert.down_proj", + "nvidia/Qwen3.5-397B-A17B-NVFP4/shared_expert.down_proj" + ] }, { "type": "gemm", "args": { - "m": 540672, - "n": 1536, - "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 8, + "n": 4096, + "k": 256, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "Qwen/Qwen3.5-397B-A17B-FP8/shared_expert.down_proj" + ] }, { "type": "gemm", "args": { - "m": 540672, - "n": 3072, - "k": 1536, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 8, + "n": 4096, + "k": 512, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "Qwen/Qwen3.5-397B-A17B/o_proj", + "Qwen/Qwen3.5-397B-A17B/shared_expert.down_proj", + "amd/Kimi-K2.5-MXFP4/kv_b_proj", + "amd/Qwen3.5-397B-A17B-MXFP4/o_proj", + "amd/Qwen3.5-397B-A17B-MXFP4/shared_expert.down_proj", + "moonshotai/Kimi-K2.5/kv_b_proj", + "nvidia/DeepSeek-R1-0528-FP4-V2/kv_b_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/kv_b_proj", + "nvidia/Kimi-K2.5-NVFP4/kv_b_proj", + "nvidia/Kimi-K2.6-NVFP4/kv_b_proj", + "nvidia/Qwen3.5-397B-A17B-NVFP4/o_proj", + "nvidia/Qwen3.5-397B-A17B-NVFP4/shared_expert.down_proj" + ] }, { "type": "gemm", "args": { - "m": 540672, - "n": 576, - "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 8, + "n": 4096, + "k": 512, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 32 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 1, + 32 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "amd/DeepSeek-R1-0528-MXFP4-Preview/kv_b_proj", + "amd/DeepSeek-R1-0528-MXFP4-v2/kv_b_proj", + "amd/DeepSeek-R1-0528-MXFP4/kv_b_proj", + "amd/Qwen3.5-397B-A17B-MXFP4-AttnFP8-V2/shared_expert.down_proj" + ] }, { "type": "gemm", "args": { - "m": 540672, + "m": 8, "n": 4096, "k": 512, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "Qwen/Qwen3.5-397B-A17B-FP8/o_proj", + "Qwen/Qwen3.5-397B-A17B-FP8/shared_expert.down_proj", + "deepseek-ai/DeepSeek-R1-0528/kv_b_proj" + ] }, { "type": "gemm", "args": { - "m": 540672, - "n": 7168, - "k": 2048, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 8, + "n": 4096, + "k": 1024, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "Qwen/Qwen3.5-397B-A17B/o_proj", + "Qwen/Qwen3.5-397B-A17B/shared_expert.down_proj", + "amd/Qwen3.5-397B-A17B-MXFP4/o_proj", + "amd/Qwen3.5-397B-A17B-MXFP4/shared_expert.down_proj", + "nvidia/Qwen3.5-397B-A17B-NVFP4/o_proj", + "nvidia/Qwen3.5-397B-A17B-NVFP4/shared_expert.down_proj" + ] }, { "type": "gemm", "args": { - "m": 540672, - "n": 2304, - "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 8, + "n": 4096, + "k": 1024, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "Qwen/Qwen3.5-397B-A17B-FP8/o_proj", + "Qwen/Qwen3.5-397B-A17B-FP8/shared_expert.down_proj" + ] }, { "type": "gemm", "args": { - "m": 540672, - "n": 7168, - "k": 2304, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 8, + "n": 4096, + "k": 2048, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "Qwen/Qwen3.5-397B-A17B/o_proj", + "amd/GLM-5.1-MXFP4/q_b_proj", + "amd/GLM-5.2-MXFP4/q_b_proj", + "amd/Qwen3.5-397B-A17B-MXFP4/o_proj", + "nvidia/GLM-5-NVFP4/q_b_proj", + "nvidia/GLM-5.1-NVFP4/q_b_proj", + "nvidia/GLM-5.2-NVFP4/q_b_proj", + "nvidia/Qwen3.5-397B-A17B-NVFP4/o_proj" + ] }, { "type": "gemm", "args": { - "m": 132, - "n": 1536, - "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 8, + "n": 4096, + "k": 2048, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "Qwen/Qwen3.5-397B-A17B-FP8/o_proj", + "zai-org/GLM-5-FP8/q_b_proj", + "zai-org/GLM-5.1-FP8/q_b_proj", + "zai-org/GLM-5.2-FP8/q_b_proj" + ] }, { "type": "gemm", "args": { - "m": 132, - "n": 3072, - "k": 1536, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 8, + "n": 4096, + "k": 2880, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/gpt-oss-120b-w-mxfp4-a-fp8/q_proj", + "openai/gpt-oss-120b/q_proj" + ] }, { "type": "gemm", "args": { - "m": 132, - "n": 576, - "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 8, + "n": 4096, + "k": 4096, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "Qwen/Qwen3.5-397B-A17B/o_proj", + "Qwen/Qwen3.5-397B-A17B/q_proj", + "amd/Qwen3.5-397B-A17B-MXFP4/o_proj", + "amd/Qwen3.5-397B-A17B-MXFP4/q_proj", + "nvidia/Qwen3.5-397B-A17B-NVFP4/o_proj", + "nvidia/Qwen3.5-397B-A17B-NVFP4/q_proj" + ] }, { "type": "gemm", "args": { - "m": 132, + "m": 8, "n": 4096, - "k": 512, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "k": 4096, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "Qwen/Qwen3.5-397B-A17B-FP8/o_proj", + "Qwen/Qwen3.5-397B-A17B-FP8/q_proj" + ] }, { "type": "gemm", "args": { - "m": 132, - "n": 7168, - "k": 2048, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 8, + "n": 4608, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "moonshotai/Kimi-K2.5/gate_up_proj", + "nvidia/Kimi-K2.6-NVFP4/gate_up_proj" + ] }, { "type": "gemm", "args": { - "m": 132, - "n": 2304, + "m": 8, + "n": 4608, "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/gate_up_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/gate_up_proj", + "nvidia/Kimi-K2.5-NVFP4/gate_up_proj" + ] }, { "type": "gemm", "args": { - "m": 132, - "n": 7168, - "k": 2304, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 8, + "n": 4608, + "k": 7168, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 32 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 1, + 32 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "amd/DeepSeek-R1-0528-MXFP4-Preview/gate_up_proj", + "amd/DeepSeek-R1-0528-MXFP4-v2/gate_up_proj", + "amd/DeepSeek-R1-0528-MXFP4/gate_up_proj", + "amd/Kimi-K2.5-MXFP4/gate_up_proj" + ] }, { "type": "gemm", "args": { - "m": 12288, - "n": 1536, + "m": 8, + "n": 4608, "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/gate_up_proj" + ] }, { "type": "gemm", "args": { - "m": 12288, - "n": 24576, + "m": 8, + "n": 6144, "k": 1536, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/GLM-5.1-MXFP4/down_proj", + "amd/GLM-5.2-MXFP4/down_proj", + "amd/Kimi-K2.5-MXFP4/q_b_proj", + "amd/MiniMax-M3-MXFP4/down_proj", + "moonshotai/Kimi-K2.5/q_b_proj", + "nvidia/DeepSeek-R1-0528-FP4-V2/q_b_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/q_b_proj", + "nvidia/GLM-5.1-NVFP4/down_proj", + "nvidia/GLM-5.2-NVFP4/down_proj", + "nvidia/Kimi-K2.5-NVFP4/q_b_proj", + "nvidia/Kimi-K2.6-NVFP4/q_b_proj" + ] }, { "type": "gemm", "args": { - "m": 12288, - "n": 576, - "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 8, + "n": 6144, + "k": 1536, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/GLM-5-NVFP4/down_proj" + ] }, { "type": "gemm", "args": { - "m": 12288, - "n": 32768, - "k": 512, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 8, + "n": 6144, + "k": 1536, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 32 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 1, + 32 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "amd/DeepSeek-R1-0528-MXFP4-Preview/q_b_proj", + "amd/DeepSeek-R1-0528-MXFP4-v2/q_b_proj", + "amd/DeepSeek-R1-0528-MXFP4/q_b_proj" + ] }, { "type": "gemm", "args": { - "m": 12288, - "n": 7168, - "k": 16384, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 8, + "n": 6144, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/q_b_proj", + "zai-org/GLM-5-FP8/down_proj", + "zai-org/GLM-5.1-FP8/down_proj", + "zai-org/GLM-5.2-FP8/down_proj" + ] }, { "type": "gemm", "args": { - "m": 12288, - "n": 4608, - "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 8, + "n": 6144, + "k": 2048, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/GLM-5.1-MXFP4/o_proj", + "amd/GLM-5.2-MXFP4/o_proj", + "amd/MiniMax-M3-MXFP4/o_proj", + "nvidia/GLM-5-NVFP4/o_proj", + "nvidia/GLM-5.1-NVFP4/o_proj", + "nvidia/GLM-5.2-NVFP4/o_proj" + ] }, { "type": "gemm", "args": { - "m": 12288, - "n": 7168, - "k": 4608, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 8, + "n": 6144, + "k": 2048, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "zai-org/GLM-5-FP8/o_proj", + "zai-org/GLM-5.1-FP8/o_proj", + "zai-org/GLM-5.2-FP8/o_proj" + ] }, { "type": "gemm", "args": { - "m": 49152, - "n": 1536, - "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 8, + "n": 6144, + "k": 3072, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/GLM-5.1-MXFP4/down_proj", + "amd/GLM-5.2-MXFP4/down_proj", + "amd/MiniMax-M2.5-MXFP4/q_proj", + "amd/MiniMax-M3-MXFP4/down_proj", + "nvidia/GLM-5.1-NVFP4/down_proj", + "nvidia/GLM-5.2-NVFP4/down_proj", + "nvidia/MiniMax-M2.5-NVFP4/q_proj" + ] }, { "type": "gemm", "args": { - "m": 49152, - "n": 3072, - "k": 1536, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 8, + "n": 6144, + "k": 3072, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/GLM-5-NVFP4/down_proj" + ] }, { "type": "gemm", "args": { - "m": 49152, - "n": 576, - "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 8, + "n": 6144, + "k": 3072, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "MiniMaxAI/MiniMax-M2.5/q_proj", + "zai-org/GLM-5-FP8/down_proj", + "zai-org/GLM-5.1-FP8/down_proj", + "zai-org/GLM-5.2-FP8/down_proj" + ] }, { "type": "gemm", "args": { - "m": 49152, - "n": 4096, - "k": 512, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 8, + "n": 6144, + "k": 4096, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/GLM-5.1-MXFP4/o_proj", + "amd/GLM-5.2-MXFP4/o_proj", + "amd/MiniMax-M3-MXFP4/o_proj", + "nvidia/GLM-5-NVFP4/o_proj", + "nvidia/GLM-5.1-NVFP4/o_proj", + "nvidia/GLM-5.2-NVFP4/o_proj" + ] }, { "type": "gemm", "args": { - "m": 49152, - "n": 7168, - "k": 2048, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 8, + "n": 6144, + "k": 4096, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "zai-org/GLM-5-FP8/o_proj", + "zai-org/GLM-5.1-FP8/o_proj", + "zai-org/GLM-5.2-FP8/o_proj" + ] }, { "type": "gemm", "args": { - "m": 49152, - "n": 2304, - "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 8, + "n": 6144, + "k": 6144, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/GLM-5.1-MXFP4/down_proj", + "amd/GLM-5.1-MXFP4/gate_up_proj", + "amd/GLM-5.2-MXFP4/down_proj", + "amd/GLM-5.2-MXFP4/gate_up_proj", + "amd/MiniMax-M3-MXFP4/gate_up_proj", + "nvidia/GLM-5.1-NVFP4/down_proj", + "nvidia/GLM-5.1-NVFP4/gate_up_proj", + "nvidia/GLM-5.2-NVFP4/down_proj", + "nvidia/GLM-5.2-NVFP4/gate_up_proj" + ] }, { "type": "gemm", "args": { - "m": 49152, - "n": 7168, - "k": 2304, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 8, + "n": 6144, + "k": 6144, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/GLM-5-NVFP4/down_proj", + "nvidia/GLM-5-NVFP4/gate_up_proj" + ] }, { "type": "gemm", "args": { - "m": 20480, - "n": 1536, - "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 8, + "n": 6144, + "k": 6144, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "zai-org/GLM-5-FP8/down_proj", + "zai-org/GLM-5-FP8/gate_up_proj", + "zai-org/GLM-5.1-FP8/down_proj", + "zai-org/GLM-5.1-FP8/gate_up_proj", + "zai-org/GLM-5.2-FP8/down_proj", + "zai-org/GLM-5.2-FP8/gate_up_proj" + ] }, { "type": "gemm", "args": { - "m": 20480, - "n": 24576, - "k": 1536, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 8, + "n": 6144, + "k": 8192, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/GLM-5.1-MXFP4/o_proj", + "amd/GLM-5.2-MXFP4/o_proj", + "amd/MiniMax-M3-MXFP4/o_proj", + "nvidia/GLM-5-NVFP4/o_proj", + "nvidia/GLM-5.1-NVFP4/o_proj", + "nvidia/GLM-5.2-NVFP4/o_proj" + ] }, { "type": "gemm", "args": { - "m": 20480, - "n": 576, - "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 8, + "n": 7168, + "k": 512, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/GLM-5.1-MXFP4/kv_b_proj", + "amd/GLM-5.2-MXFP4/kv_b_proj", + "amd/Kimi-K2.5-MXFP4/o_proj", + "moonshotai/Kimi-K2.5/o_proj", + "nvidia/GLM-5-NVFP4/kv_b_proj", + "nvidia/GLM-5.1-NVFP4/kv_b_proj", + "nvidia/GLM-5.2-NVFP4/kv_b_proj", + "nvidia/Kimi-K2.5-NVFP4/o_proj", + "nvidia/Kimi-K2.6-NVFP4/o_proj" + ] }, { "type": "gemm", "args": { - "m": 20480, - "n": 32768, + "m": 8, + "n": 7168, "k": 512, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "zai-org/GLM-5-FP8/kv_b_proj", + "zai-org/GLM-5.1-FP8/kv_b_proj", + "zai-org/GLM-5.2-FP8/kv_b_proj" + ] }, { "type": "gemm", "args": { - "m": 20480, + "m": 8, "n": 7168, - "k": 16384, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "k": 576, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/down_proj" + ] }, { "type": "gemm", "args": { - "m": 20480, - "n": 4608, - "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 8, + "n": 7168, + "k": 1024, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/Kimi-K2.5-MXFP4/o_proj", + "moonshotai/Kimi-K2.5/o_proj", + "nvidia/Kimi-K2.5-NVFP4/o_proj", + "nvidia/Kimi-K2.6-NVFP4/o_proj" + ] }, { "type": "gemm", "args": { - "m": 20480, + "m": 8, "n": 7168, - "k": 4608, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "k": 1152, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/down_proj" + ] }, { "type": "gemm", "args": { - "m": 30720, - "n": 1536, - "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 8, + "n": 7168, + "k": 2048, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/Kimi-K2.5-MXFP4/o_proj", + "moonshotai/Kimi-K2.5/o_proj", + "nvidia/Kimi-K2.5-NVFP4/o_proj", + "nvidia/Kimi-K2.6-NVFP4/o_proj" + ] }, { "type": "gemm", "args": { - "m": 30720, - "n": 24576, - "k": 1536, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 8, + "n": 7168, + "k": 2048, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/o_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/o_proj" + ] }, { "type": "gemm", "args": { - "m": 30720, - "n": 576, - "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 8, + "n": 7168, + "k": 2048, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 32 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 1, + 32 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "amd/DeepSeek-R1-0528-MXFP4-Preview/o_proj", + "amd/DeepSeek-R1-0528-MXFP4-v2/o_proj", + "amd/DeepSeek-R1-0528-MXFP4/o_proj" + ] }, { "type": "gemm", "args": { - "m": 30720, - "n": 32768, - "k": 512, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 8, + "n": 7168, + "k": 2048, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/o_proj", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] }, { "type": "gemm", "args": { - "m": 30720, + "m": 8, "n": 7168, - "k": 16384, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "k": 2304, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "moonshotai/Kimi-K2.5/down_proj", + "nvidia/Kimi-K2.6-NVFP4/down_proj" + ] }, { "type": "gemm", "args": { - "m": 30720, - "n": 4608, - "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 8, + "n": 7168, + "k": 2304, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/down_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/down_proj", + "nvidia/Kimi-K2.5-NVFP4/down_proj" + ] }, { "type": "gemm", "args": { - "m": 30720, + "m": 8, "n": 7168, - "k": 4608, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "k": 2304, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 32 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 1, + 32 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "amd/DeepSeek-R1-0528-MXFP4-Preview/down_proj", + "amd/DeepSeek-R1-0528-MXFP4-v2/down_proj", + "amd/DeepSeek-R1-0528-MXFP4/down_proj", + "amd/Kimi-K2.5-MXFP4/down_proj" + ] }, { "type": "gemm", "args": { - "m": 61440, - "n": 1536, - "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 8, + "n": 7168, + "k": 2304, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/down_proj" + ] }, { "type": "gemm", "args": { - "m": 61440, - "n": 24576, - "k": 1536, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 8, + "n": 7168, + "k": 4096, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/o_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/o_proj" + ] }, { "type": "gemm", "args": { - "m": 61440, - "n": 576, - "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 8, + "n": 7168, + "k": 4096, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 32 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 1, + 32 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "amd/DeepSeek-R1-0528-MXFP4-Preview/o_proj", + "amd/DeepSeek-R1-0528-MXFP4-v2/o_proj", + "amd/DeepSeek-R1-0528-MXFP4/o_proj" + ] }, { "type": "gemm", "args": { - "m": 61440, - "n": 32768, - "k": 512, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 8, + "n": 7168, + "k": 4096, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/o_proj", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] }, { "type": "gemm", "args": { - "m": 61440, + "m": 8, "n": 7168, - "k": 16384, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "k": 4608, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "moonshotai/Kimi-K2.5/down_proj", + "nvidia/Kimi-K2.6-NVFP4/down_proj" + ] }, { "type": "gemm", "args": { - "m": 61440, - "n": 4608, - "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 8, + "n": 7168, + "k": 4608, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/down_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/down_proj", + "nvidia/Kimi-K2.5-NVFP4/down_proj" + ] }, { "type": "gemm", "args": { - "m": 61440, + "m": 8, "n": 7168, "k": 4608, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 32 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 1, + 32 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "amd/DeepSeek-R1-0528-MXFP4-Preview/down_proj", + "amd/DeepSeek-R1-0528-MXFP4-v2/down_proj", + "amd/DeepSeek-R1-0528-MXFP4/down_proj", + "amd/Kimi-K2.5-MXFP4/down_proj" + ] }, { "type": "gemm", "args": { - "m": 15, - "n": 24576, - "k": 1536, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 8, + "n": 7168, + "k": 4608, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/down_proj" + ] }, { "type": "gemm", "args": { - "m": 15, - "n": 32768, - "k": 512, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 8, + "n": 7168, + "k": 8192, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] }, { "type": "gemm", "args": { - "m": 15, + "m": 8, "n": 7168, - "k": 16384, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "k": 9216, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/down_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/down_proj", + "nvidia/Kimi-K2.5-NVFP4/down_proj" + ] }, { "type": "gemm", "args": { - "m": 15, - "n": 4608, - "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 8, + "n": 7168, + "k": 16384, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/o_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/o_proj" + ] }, { "type": "gemm", "args": { - "m": 15, + "m": 8, "n": 7168, - "k": 4608, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "k": 16384, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/o_proj", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] }, { "type": "gemm", "args": { - "m": 122880, - "n": 1536, - "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 8, + "n": 7168, + "k": 16384, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] }, { "type": "gemm", "args": { - "m": 122880, - "n": 24576, - "k": 1536, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 8, + "n": 7168, + "k": 65536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] }, { "type": "gemm", "args": { - "m": 122880, - "n": 576, - "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 8, + "n": 8192, + "k": 512, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/Kimi-K2.5-MXFP4/kv_b_proj", + "moonshotai/Kimi-K2.5/kv_b_proj", + "nvidia/DeepSeek-R1-0528-FP4-V2/kv_b_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/kv_b_proj", + "nvidia/Kimi-K2.5-NVFP4/kv_b_proj", + "nvidia/Kimi-K2.6-NVFP4/kv_b_proj" + ] }, { "type": "gemm", "args": { - "m": 122880, - "n": 32768, + "m": 8, + "n": 8192, "k": 512, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 32 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 1, + 32 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "amd/DeepSeek-R1-0528-MXFP4-Preview/kv_b_proj", + "amd/DeepSeek-R1-0528-MXFP4-v2/kv_b_proj", + "amd/DeepSeek-R1-0528-MXFP4/kv_b_proj" + ] }, { "type": "gemm", "args": { - "m": 122880, - "n": 7168, - "k": 16384, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 8, + "n": 8192, + "k": 512, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/kv_b_proj" + ] }, { "type": "gemm", "args": { - "m": 122880, - "n": 4608, - "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 8, + "n": 8192, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_b" + ] }, { "type": "gemm", "args": { - "m": 122880, - "n": 7168, - "k": 4608, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 8, + "n": 8192, + "k": 2048, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/GLM-5.1-MXFP4/q_b_proj", + "amd/GLM-5.2-MXFP4/q_b_proj", + "nvidia/GLM-5-NVFP4/q_b_proj", + "nvidia/GLM-5.1-NVFP4/q_b_proj", + "nvidia/GLM-5.2-NVFP4/q_b_proj" + ] }, { "type": "gemm", "args": { - "m": 30, - "n": 24576, - "k": 1536, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 8, + "n": 9216, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "moonshotai/Kimi-K2.5/gate_up_proj", + "nvidia/Kimi-K2.6-NVFP4/gate_up_proj" + ] }, { "type": "gemm", "args": { - "m": 30, - "n": 32768, - "k": 512, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 8, + "n": 9216, + "k": 7168, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/gate_up_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/gate_up_proj", + "nvidia/Kimi-K2.5-NVFP4/gate_up_proj" + ] }, { "type": "gemm", "args": { - "m": 30, - "n": 7168, - "k": 16384, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 8, + "n": 9216, + "k": 7168, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 32 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 1, + 32 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "amd/DeepSeek-R1-0528-MXFP4-Preview/gate_up_proj", + "amd/DeepSeek-R1-0528-MXFP4-v2/gate_up_proj", + "amd/DeepSeek-R1-0528-MXFP4/gate_up_proj", + "amd/Kimi-K2.5-MXFP4/gate_up_proj" + ] }, { "type": "gemm", "args": { - "m": 30, - "n": 4608, + "m": 8, + "n": 9216, "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/gate_up_proj" + ] }, { "type": "gemm", "args": { - "m": 30, - "n": 7168, - "k": 4608, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 8, + "n": 12288, + "k": 1536, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/Kimi-K2.5-MXFP4/q_b_proj", + "moonshotai/Kimi-K2.5/q_b_proj", + "nvidia/DeepSeek-R1-0528-FP4-V2/q_b_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/q_b_proj", + "nvidia/Kimi-K2.5-NVFP4/q_b_proj", + "nvidia/Kimi-K2.6-NVFP4/q_b_proj" + ] }, { "type": "gemm", "args": { - "m": 81920, - "n": 1536, - "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 8, + "n": 14336, + "k": 512, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/GLM-5.1-MXFP4/kv_b_proj", + "amd/GLM-5.2-MXFP4/kv_b_proj", + "nvidia/GLM-5-NVFP4/kv_b_proj", + "nvidia/GLM-5.1-NVFP4/kv_b_proj", + "nvidia/GLM-5.2-NVFP4/kv_b_proj" + ] }, { "type": "gemm", "args": { - "m": 81920, - "n": 3072, - "k": 1536, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 8, + "n": 15360, + "k": 1024, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] }, { "type": "gemm", "args": { - "m": 81920, - "n": 576, - "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 8, + "n": 16384, + "k": 512, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/Kimi-K2.5-MXFP4/kv_b_proj", + "moonshotai/Kimi-K2.5/kv_b_proj", + "nvidia/DeepSeek-R1-0528-FP4-V2/kv_b_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/kv_b_proj", + "nvidia/Kimi-K2.5-NVFP4/kv_b_proj", + "nvidia/Kimi-K2.6-NVFP4/kv_b_proj" + ] }, { "type": "gemm", "args": { - "m": 81920, - "n": 4096, - "k": 512, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 8, + "n": 16384, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_b" + ] }, { "type": "gemm", "args": { - "m": 81920, - "n": 7168, - "k": 2048, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 8, + "n": 24576, + "k": 1536, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/q_b_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/q_b_proj" + ] }, { "type": "gemm", "args": { - "m": 81920, - "n": 2304, - "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 8, + "n": 24576, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/q_b_proj" + ] }, { "type": "gemm", "args": { - "m": 81920, - "n": 7168, - "k": 2304, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 8, + "n": 30720, + "k": 1024, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] }, { "type": "gemm", "args": { - "m": 122880, - "n": 3072, - "k": 1536, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 8, + "n": 32768, + "k": 512, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/kv_b_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/kv_b_proj" + ] }, { "type": "gemm", "args": { - "m": 122880, - "n": 4096, + "m": 8, + "n": 32768, "k": 512, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/kv_b_proj" + ] }, { "type": "gemm", "args": { - "m": 122880, - "n": 7168, - "k": 2048, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 8, + "n": 65536, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_b" + ] }, { "type": "gemm", "args": { - "m": 122880, - "n": 2304, + "m": 8, + "n": 122880, + "k": 1024, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 9, + "n": 576, "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/Kimi-K2.5-MXFP4/kv_a_proj_with_mqa", + "moonshotai/Kimi-K2.5/kv_a_proj_with_mqa", + "nvidia/DeepSeek-R1-0528-FP4-V2/kv_a_proj_with_mqa", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/kv_a_proj_with_mqa", + "nvidia/Kimi-K2.5-NVFP4/kv_a_proj_with_mqa", + "nvidia/Kimi-K2.6-NVFP4/kv_a_proj_with_mqa" + ] }, { "type": "gemm", "args": { - "m": 122880, - "n": 7168, - "k": 2304, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 9, + "n": 576, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/kv_a_proj_with_mqa" + ] }, { "type": "gemm", "args": { - "m": 245760, - "n": 1536, + "m": 9, + "n": 1152, "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/gate_up_proj" + ] }, { "type": "gemm", "args": { - "m": 245760, - "n": 3072, + "m": 9, + "n": 1536, "k": 1536, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/q_b_proj" + ] }, { "type": "gemm", "args": { - "m": 245760, - "n": 576, + "m": 9, + "n": 1536, "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/Kimi-K2.5-MXFP4/q_a_proj", + "moonshotai/Kimi-K2.5/q_a_proj", + "nvidia/DeepSeek-R1-0528-FP4-V2/q_a_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/q_a_proj", + "nvidia/Kimi-K2.5-NVFP4/q_a_proj", + "nvidia/Kimi-K2.6-NVFP4/q_a_proj" + ] }, { "type": "gemm", "args": { - "m": 245760, - "n": 4096, - "k": 512, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 9, + "n": 1536, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/gate_up_proj", + "deepseek-ai/DeepSeek-R1-0528/q_a_proj", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w3" + ] }, { "type": "gemm", "args": { - "m": 245760, - "n": 7168, - "k": 2048, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 9, + "n": 2048, + "k": 512, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/kv_b_proj" + ] }, { "type": "gemm", "args": { - "m": 245760, + "m": 9, "n": 2304, "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/gate_up_proj" + ] }, { "type": "gemm", "args": { - "m": 245760, - "n": 7168, - "k": 2304, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 9, + "n": 4608, + "k": 7168, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/gate_up_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/gate_up_proj", + "nvidia/Kimi-K2.5-NVFP4/gate_up_proj" + ] }, { "type": "gemm", "args": { - "m": 60, - "n": 1536, + "m": 9, + "n": 4608, "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/gate_up_proj" + ] }, { "type": "gemm", "args": { - "m": 60, - "n": 3072, - "k": 1536, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 9, + "n": 7168, + "k": 1024, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/o_proj", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] }, { "type": "gemm", "args": { - "m": 60, - "n": 576, - "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 9, + "n": 7168, + "k": 1152, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/down_proj" + ] }, { "type": "gemm", "args": { - "m": 60, - "n": 4096, - "k": 512, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 9, + "n": 7168, + "k": 2304, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/down_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/down_proj", + "nvidia/Kimi-K2.5-NVFP4/down_proj" + ] }, { "type": "gemm", "args": { - "m": 60, + "m": 9, "n": 7168, - "k": 2048, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "k": 2304, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/down_proj" + ] }, { "type": "gemm", "args": { - "m": 60, - "n": 2304, - "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 9, + "n": 7168, + "k": 4608, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/down_proj" + ] }, { "type": "gemm", "args": { - "m": 60, + "m": 9, "n": 7168, - "k": 2304, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "k": 16384, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/o_proj", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] }, { "type": "gemm", "args": { - "m": 491520, - "n": 1536, - "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 9, + "n": 24576, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/q_b_proj" + ] }, { "type": "gemm", "args": { - "m": 491520, - "n": 3072, - "k": 1536, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 9, + "n": 32768, + "k": 512, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/kv_b_proj" + ] }, { "type": "gemm", "args": { - "m": 491520, - "n": 576, + "m": 10, + "n": 384, "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [] }, { "type": "gemm", "args": { - "m": 491520, - "n": 4096, - "k": 512, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 10, + "n": 576, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/Kimi-K2.5-MXFP4/kv_a_proj_with_mqa", + "moonshotai/Kimi-K2.5/kv_a_proj_with_mqa", + "nvidia/DeepSeek-R1-0528-FP4-V2/kv_a_proj_with_mqa", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/kv_a_proj_with_mqa", + "nvidia/Kimi-K2.5-NVFP4/kv_a_proj_with_mqa", + "nvidia/Kimi-K2.6-NVFP4/kv_a_proj_with_mqa" + ] }, { "type": "gemm", "args": { - "m": 491520, - "n": 7168, - "k": 2048, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 10, + "n": 576, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/kv_a_proj_with_mqa" + ] }, { "type": "gemm", "args": { - "m": 491520, - "n": 2304, + "m": 10, + "n": 1536, "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/Kimi-K2.5-MXFP4/q_a_proj", + "moonshotai/Kimi-K2.5/q_a_proj", + "nvidia/DeepSeek-R1-0528-FP4-V2/q_a_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/q_a_proj", + "nvidia/Kimi-K2.5-NVFP4/q_a_proj", + "nvidia/Kimi-K2.6-NVFP4/q_a_proj" + ] }, { "type": "gemm", "args": { - "m": 491520, - "n": 7168, - "k": 2304, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 10, + "n": 1536, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/gate_up_proj", + "deepseek-ai/DeepSeek-R1-0528/q_a_proj", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w3" + ] }, { "type": "gemm", "args": { - "m": 120, - "n": 1536, + "m": 10, + "n": 2304, "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/gate_up_proj" + ] }, { "type": "gemm", "args": { - "m": 120, + "m": 10, "n": 3072, "k": 1536, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/Kimi-K2.5-MXFP4/q_b_proj", + "amd/MiniMax-M2.5-MXFP4/o_proj", + "moonshotai/Kimi-K2.5/q_b_proj", + "nvidia/DeepSeek-R1-0528-FP4-V2/q_b_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/q_b_proj", + "nvidia/Kimi-K2.5-NVFP4/q_b_proj", + "nvidia/Kimi-K2.6-NVFP4/q_b_proj", + "nvidia/MiniMax-M2.5-NVFP4/o_proj" + ] }, { "type": "gemm", "args": { - "m": 120, - "n": 576, - "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 10, + "n": 3072, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "MiniMaxAI/MiniMax-M2.5/experts.down_proj", + "MiniMaxAI/MiniMax-M2.5/o_proj", + "deepseek-ai/DeepSeek-R1-0528/q_b_proj" + ] }, { "type": "gemm", "args": { - "m": 120, + "m": 10, "n": 4096, "k": 512, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "Qwen/Qwen3.5-397B-A17B/o_proj", + "Qwen/Qwen3.5-397B-A17B/shared_expert.down_proj", + "amd/Kimi-K2.5-MXFP4/kv_b_proj", + "amd/Qwen3.5-397B-A17B-MXFP4/o_proj", + "amd/Qwen3.5-397B-A17B-MXFP4/shared_expert.down_proj", + "moonshotai/Kimi-K2.5/kv_b_proj", + "nvidia/DeepSeek-R1-0528-FP4-V2/kv_b_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/kv_b_proj", + "nvidia/Kimi-K2.5-NVFP4/kv_b_proj", + "nvidia/Kimi-K2.6-NVFP4/kv_b_proj", + "nvidia/Qwen3.5-397B-A17B-NVFP4/o_proj", + "nvidia/Qwen3.5-397B-A17B-NVFP4/shared_expert.down_proj" + ] }, { "type": "gemm", "args": { - "m": 120, - "n": 7168, - "k": 2048, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 10, + "n": 4096, + "k": 512, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "Qwen/Qwen3.5-397B-A17B-FP8/o_proj", + "Qwen/Qwen3.5-397B-A17B-FP8/shared_expert.down_proj", + "deepseek-ai/DeepSeek-R1-0528/kv_b_proj" + ] }, { "type": "gemm", "args": { - "m": 120, - "n": 2304, + "m": 10, + "n": 4608, "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 120, - "n": 7168, - "k": 2304, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/gate_up_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/gate_up_proj", + "nvidia/Kimi-K2.5-NVFP4/gate_up_proj" + ] }, { "type": "gemm", "args": { - "m": 1122304, - "n": 1536, + "m": 10, + "n": 4608, "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/gate_up_proj" + ] }, { "type": "gemm", "args": { - "m": 1122304, - "n": 24576, + "m": 10, + "n": 6144, "k": 1536, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/GLM-5.1-MXFP4/down_proj", + "amd/GLM-5.2-MXFP4/down_proj", + "amd/Kimi-K2.5-MXFP4/q_b_proj", + "amd/MiniMax-M3-MXFP4/down_proj", + "moonshotai/Kimi-K2.5/q_b_proj", + "nvidia/DeepSeek-R1-0528-FP4-V2/q_b_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/q_b_proj", + "nvidia/GLM-5.1-NVFP4/down_proj", + "nvidia/GLM-5.2-NVFP4/down_proj", + "nvidia/Kimi-K2.5-NVFP4/q_b_proj", + "nvidia/Kimi-K2.6-NVFP4/q_b_proj" + ] }, { "type": "gemm", "args": { - "m": 1122304, - "n": 576, - "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 10, + "n": 7168, + "k": 384, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-NVFP4-v2/down_proj" + ] }, { "type": "gemm", "args": { - "m": 1122304, - "n": 32768, - "k": 512, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 1122304, + "m": 10, "n": 7168, - "k": 16384, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 1122304, - "n": 4608, - "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "k": 576, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-NVFP4-v2/down_proj" + ] }, { "type": "gemm", "args": { - "m": 1122304, + "m": 10, "n": 7168, - "k": 4608, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "k": 576, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/down_proj" + ] }, { "type": "gemm", "args": { - "m": 274, - "n": 4608, - "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 10, + "n": 7168, + "k": 2048, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/o_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/o_proj" + ] }, { "type": "gemm", "args": { - "m": 274, + "m": 10, "n": 7168, - "k": 4608, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "k": 2048, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/o_proj", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] }, { "type": "gemm", "args": { - "m": 561152, - "n": 2304, - "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 10, + "n": 7168, + "k": 2304, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/down_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/down_proj", + "nvidia/Kimi-K2.5-NVFP4/down_proj" + ] }, { "type": "gemm", "args": { - "m": 561152, + "m": 10, "n": 7168, "k": 2304, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/down_proj" + ] }, { "type": "gemm", "args": { - "m": 137, - "n": 1536, - "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 10, + "n": 7168, + "k": 4096, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/o_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/o_proj" + ] }, { "type": "gemm", "args": { - "m": 137, - "n": 24576, - "k": 1536, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 10, + "n": 7168, + "k": 4608, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/down_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/down_proj", + "nvidia/Kimi-K2.5-NVFP4/down_proj" + ] }, { "type": "gemm", "args": { - "m": 137, - "n": 576, - "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 10, + "n": 7168, + "k": 9216, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/down_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/down_proj", + "nvidia/Kimi-K2.5-NVFP4/down_proj" + ] }, { "type": "gemm", "args": { - "m": 137, - "n": 32768, - "k": 512, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 10, + "n": 7168, + "k": 16384, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/o_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/o_proj" + ] }, { "type": "gemm", "args": { - "m": 137, + "m": 10, "n": 7168, "k": 16384, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/o_proj", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] }, { "type": "gemm", "args": { - "m": 137, - "n": 2304, - "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 10, + "n": 8192, + "k": 512, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/Kimi-K2.5-MXFP4/kv_b_proj", + "moonshotai/Kimi-K2.5/kv_b_proj", + "nvidia/DeepSeek-R1-0528-FP4-V2/kv_b_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/kv_b_proj", + "nvidia/Kimi-K2.5-NVFP4/kv_b_proj", + "nvidia/Kimi-K2.6-NVFP4/kv_b_proj" + ] }, { "type": "gemm", "args": { - "m": 137, - "n": 7168, - "k": 2304, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 10, + "n": 9216, + "k": 7168, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/gate_up_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/gate_up_proj", + "nvidia/Kimi-K2.5-NVFP4/gate_up_proj" + ] }, { "type": "gemm", "args": { - "m": 2244608, - "n": 1536, - "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 10, + "n": 24576, + "k": 1536, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/q_b_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/q_b_proj" + ] }, { "type": "gemm", "args": { - "m": 2244608, + "m": 10, "n": 24576, "k": 1536, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/q_b_proj" + ] }, { "type": "gemm", "args": { - "m": 2244608, - "n": 576, - "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 10, + "n": 32768, + "k": 512, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/kv_b_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/kv_b_proj" + ] }, { "type": "gemm", "args": { - "m": 2244608, + "m": 10, "n": 32768, "k": 512, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/kv_b_proj" + ] }, { "type": "gemm", "args": { - "m": 2244608, - "n": 7168, - "k": 16384, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 11, + "n": 576, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/Kimi-K2.5-MXFP4/kv_a_proj_with_mqa", + "moonshotai/Kimi-K2.5/kv_a_proj_with_mqa", + "nvidia/DeepSeek-R1-0528-FP4-V2/kv_a_proj_with_mqa", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/kv_a_proj_with_mqa", + "nvidia/Kimi-K2.5-NVFP4/kv_a_proj_with_mqa", + "nvidia/Kimi-K2.6-NVFP4/kv_a_proj_with_mqa" + ] }, { "type": "gemm", "args": { - "m": 2244608, - "n": 4608, + "m": 11, + "n": 576, "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/kv_a_proj_with_mqa" + ] }, { "type": "gemm", "args": { - "m": 2244608, - "n": 7168, - "k": 4608, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 11, + "n": 1536, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/Kimi-K2.5-MXFP4/q_a_proj", + "moonshotai/Kimi-K2.5/q_a_proj", + "nvidia/DeepSeek-R1-0528-FP4-V2/q_a_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/q_a_proj", + "nvidia/Kimi-K2.5-NVFP4/q_a_proj", + "nvidia/Kimi-K2.6-NVFP4/q_a_proj" + ] }, { "type": "gemm", "args": { - "m": 3463168, + "m": 11, "n": 1536, "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/gate_up_proj", + "deepseek-ai/DeepSeek-R1-0528/q_a_proj", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w3" + ] }, { "type": "gemm", "args": { - "m": 3463168, - "n": 24576, - "k": 1536, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 11, + "n": 4608, + "k": 7168, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/gate_up_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/gate_up_proj", + "nvidia/Kimi-K2.5-NVFP4/gate_up_proj" + ] }, { "type": "gemm", "args": { - "m": 3463168, - "n": 576, - "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 11, + "n": 7168, + "k": 576, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-NVFP4-v2/down_proj" + ] }, { "type": "gemm", "args": { - "m": 3463168, - "n": 32768, - "k": 512, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 11, + "n": 7168, + "k": 576, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/down_proj" + ] }, { "type": "gemm", "args": { - "m": 3463168, + "m": 11, "n": 7168, - "k": 16384, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "k": 4608, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/down_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/down_proj", + "nvidia/Kimi-K2.5-NVFP4/down_proj" + ] }, { "type": "gemm", "args": { - "m": 3463168, - "n": 4608, - "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 11, + "n": 7168, + "k": 9216, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/down_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/down_proj", + "nvidia/Kimi-K2.5-NVFP4/down_proj" + ] }, { "type": "gemm", "args": { - "m": 3463168, + "m": 11, "n": 7168, - "k": 4608, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "k": 16384, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/o_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/o_proj" + ] }, { "type": "gemm", "args": { - "m": 845, - "n": 1536, + "m": 11, + "n": 7168, + "k": 16384, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/o_proj", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 11, + "n": 9216, "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/gate_up_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/gate_up_proj", + "nvidia/Kimi-K2.5-NVFP4/gate_up_proj" + ] }, { "type": "gemm", "args": { - "m": 845, + "m": 11, "n": 24576, "k": 1536, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/q_b_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/q_b_proj" + ] }, { "type": "gemm", "args": { - "m": 845, - "n": 576, - "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 11, + "n": 24576, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/q_b_proj" + ] }, { "type": "gemm", "args": { - "m": 845, + "m": 11, "n": 32768, "k": 512, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/kv_b_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/kv_b_proj" + ] }, { "type": "gemm", "args": { - "m": 845, - "n": 7168, - "k": 16384, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 11, + "n": 32768, + "k": 512, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/kv_b_proj" + ] }, { "type": "gemm", "args": { - "m": 845, - "n": 4608, + "m": 12, + "n": 576, "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/Kimi-K2.5-MXFP4/kv_a_proj_with_mqa", + "moonshotai/Kimi-K2.5/kv_a_proj_with_mqa", + "nvidia/DeepSeek-R1-0528-FP4-V2/kv_a_proj_with_mqa", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/kv_a_proj_with_mqa", + "nvidia/Kimi-K2.5-NVFP4/kv_a_proj_with_mqa", + "nvidia/Kimi-K2.6-NVFP4/kv_a_proj_with_mqa" + ] }, { "type": "gemm", "args": { - "m": 845, - "n": 7168, - "k": 4608, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 12, + "n": 576, + "k": 7168, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 32 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 1, + 32 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "amd/DeepSeek-R1-0528-MXFP4-Preview/kv_a_proj_with_mqa", + "amd/DeepSeek-R1-0528-MXFP4-v2/kv_a_proj_with_mqa", + "amd/DeepSeek-R1-0528-MXFP4/kv_a_proj_with_mqa" + ] }, { "type": "gemm", "args": { - "m": 1122304, - "n": 2304, + "m": 12, + "n": 576, "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/kv_a_proj_with_mqa" + ] }, { "type": "gemm", "args": { - "m": 1122304, - "n": 7168, - "k": 2304, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 12, + "n": 1152, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/gate_up_proj" + ] }, { "type": "gemm", "args": { - "m": 1731584, + "m": 12, "n": 1536, - "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/q_b_proj" + ] }, { "type": "gemm", "args": { - "m": 1731584, - "n": 24576, - "k": 1536, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 12, + "n": 1536, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/Kimi-K2.5-MXFP4/q_a_proj", + "moonshotai/Kimi-K2.5/q_a_proj", + "nvidia/DeepSeek-R1-0528-FP4-V2/q_a_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/q_a_proj", + "nvidia/Kimi-K2.5-NVFP4/q_a_proj", + "nvidia/Kimi-K2.6-NVFP4/q_a_proj" + ] }, { "type": "gemm", "args": { - "m": 1731584, - "n": 576, + "m": 12, + "n": 1536, "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 32 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 1, + 32 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "amd/DeepSeek-R1-0528-MXFP4-Preview/q_a_proj", + "amd/DeepSeek-R1-0528-MXFP4-v2/q_a_proj", + "amd/DeepSeek-R1-0528-MXFP4/q_a_proj" + ] }, { "type": "gemm", "args": { - "m": 1731584, - "n": 32768, - "k": 512, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 12, + "n": 1536, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/gate_up_proj", + "deepseek-ai/DeepSeek-R1-0528/q_a_proj", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w3" + ] }, { "type": "gemm", "args": { - "m": 1731584, - "n": 7168, - "k": 16384, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 12, + "n": 2048, + "k": 512, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/kv_b_proj" + ] }, { "type": "gemm", "args": { - "m": 1731584, + "m": 12, "n": 2304, "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/gate_up_proj" + ] }, { "type": "gemm", "args": { - "m": 1731584, - "n": 7168, - "k": 2304, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 12, + "n": 3072, + "k": 1536, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/Kimi-K2.5-MXFP4/q_b_proj", + "amd/MiniMax-M2.5-MXFP4/o_proj", + "moonshotai/Kimi-K2.5/q_b_proj", + "nvidia/DeepSeek-R1-0528-FP4-V2/q_b_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/q_b_proj", + "nvidia/Kimi-K2.5-NVFP4/q_b_proj", + "nvidia/Kimi-K2.6-NVFP4/q_b_proj", + "nvidia/MiniMax-M2.5-NVFP4/o_proj" + ] }, { "type": "gemm", "args": { - "m": 422, - "n": 1536, - "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 12, + "n": 3072, + "k": 1536, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 32 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 1, + 32 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "amd/DeepSeek-R1-0528-MXFP4-Preview/q_b_proj", + "amd/DeepSeek-R1-0528-MXFP4-v2/q_b_proj", + "amd/DeepSeek-R1-0528-MXFP4/q_b_proj", + "amd/MiniMax-M2.5-MXFP4/experts.down_proj" + ] }, { "type": "gemm", "args": { - "m": 422, - "n": 24576, + "m": 12, + "n": 3072, "k": 1536, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "MiniMaxAI/MiniMax-M2.5/experts.down_proj", + "MiniMaxAI/MiniMax-M2.5/o_proj", + "deepseek-ai/DeepSeek-R1-0528/q_b_proj" + ] }, { "type": "gemm", "args": { - "m": 422, - "n": 576, - "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 12, + "n": 4096, + "k": 512, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "Qwen/Qwen3.5-397B-A17B/o_proj", + "Qwen/Qwen3.5-397B-A17B/shared_expert.down_proj", + "amd/Kimi-K2.5-MXFP4/kv_b_proj", + "amd/Qwen3.5-397B-A17B-MXFP4/o_proj", + "amd/Qwen3.5-397B-A17B-MXFP4/shared_expert.down_proj", + "moonshotai/Kimi-K2.5/kv_b_proj", + "nvidia/DeepSeek-R1-0528-FP4-V2/kv_b_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/kv_b_proj", + "nvidia/Kimi-K2.5-NVFP4/kv_b_proj", + "nvidia/Kimi-K2.6-NVFP4/kv_b_proj", + "nvidia/Qwen3.5-397B-A17B-NVFP4/o_proj", + "nvidia/Qwen3.5-397B-A17B-NVFP4/shared_expert.down_proj" + ] }, { "type": "gemm", "args": { - "m": 422, - "n": 32768, + "m": 12, + "n": 4096, "k": 512, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 32 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 1, + 32 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "amd/DeepSeek-R1-0528-MXFP4-Preview/kv_b_proj", + "amd/DeepSeek-R1-0528-MXFP4-v2/kv_b_proj", + "amd/DeepSeek-R1-0528-MXFP4/kv_b_proj", + "amd/Qwen3.5-397B-A17B-MXFP4-AttnFP8-V2/shared_expert.down_proj" + ] }, { "type": "gemm", "args": { - "m": 422, - "n": 7168, - "k": 16384, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 12, + "n": 4096, + "k": 512, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "Qwen/Qwen3.5-397B-A17B-FP8/o_proj", + "Qwen/Qwen3.5-397B-A17B-FP8/shared_expert.down_proj", + "deepseek-ai/DeepSeek-R1-0528/kv_b_proj" + ] }, { "type": "gemm", "args": { - "m": 422, - "n": 2304, + "m": 12, + "n": 4608, "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/gate_up_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/gate_up_proj", + "nvidia/Kimi-K2.5-NVFP4/gate_up_proj" + ] }, { "type": "gemm", "args": { - "m": 422, - "n": 7168, - "k": 2304, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 12, + "n": 4608, + "k": 7168, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 32 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 1, + 32 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "amd/DeepSeek-R1-0528-MXFP4-Preview/gate_up_proj", + "amd/DeepSeek-R1-0528-MXFP4-v2/gate_up_proj", + "amd/DeepSeek-R1-0528-MXFP4/gate_up_proj", + "amd/Kimi-K2.5-MXFP4/gate_up_proj" + ] }, { "type": "gemm", "args": { - "m": 1347584, - "n": 1536, + "m": 12, + "n": 4608, "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/gate_up_proj" + ] }, { "type": "gemm", "args": { - "m": 1347584, - "n": 24576, + "m": 12, + "n": 6144, "k": 1536, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 1347584, - "n": 576, - "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/GLM-5.1-MXFP4/down_proj", + "amd/GLM-5.2-MXFP4/down_proj", + "amd/Kimi-K2.5-MXFP4/q_b_proj", + "amd/MiniMax-M3-MXFP4/down_proj", + "moonshotai/Kimi-K2.5/q_b_proj", + "nvidia/DeepSeek-R1-0528-FP4-V2/q_b_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/q_b_proj", + "nvidia/GLM-5.1-NVFP4/down_proj", + "nvidia/GLM-5.2-NVFP4/down_proj", + "nvidia/Kimi-K2.5-NVFP4/q_b_proj", + "nvidia/Kimi-K2.6-NVFP4/q_b_proj" + ] }, { "type": "gemm", "args": { - "m": 1347584, - "n": 32768, - "k": 512, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 12, + "n": 7168, + "k": 1024, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/o_proj", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] }, { "type": "gemm", "args": { - "m": 1347584, + "m": 12, "n": 7168, - "k": 16384, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "k": 1152, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/down_proj" + ] }, { "type": "gemm", "args": { - "m": 1347584, - "n": 4608, - "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 12, + "n": 7168, + "k": 2048, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/Kimi-K2.5-MXFP4/o_proj", + "moonshotai/Kimi-K2.5/o_proj", + "nvidia/Kimi-K2.5-NVFP4/o_proj", + "nvidia/Kimi-K2.6-NVFP4/o_proj" + ] }, { "type": "gemm", "args": { - "m": 1347584, + "m": 12, "n": 7168, - "k": 4608, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "k": 2048, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/o_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/o_proj" + ] }, { "type": "gemm", "args": { - "m": 329, - "n": 1536, - "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 12, + "n": 7168, + "k": 2048, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 32 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 1, + 32 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "amd/DeepSeek-R1-0528-MXFP4-Preview/o_proj", + "amd/DeepSeek-R1-0528-MXFP4-v2/o_proj", + "amd/DeepSeek-R1-0528-MXFP4/o_proj" + ] }, { "type": "gemm", "args": { - "m": 329, - "n": 24576, - "k": 1536, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 12, + "n": 7168, + "k": 2048, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/o_proj", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] }, { "type": "gemm", "args": { - "m": 329, - "n": 576, - "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 12, + "n": 7168, + "k": 2304, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/down_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/down_proj", + "nvidia/Kimi-K2.5-NVFP4/down_proj" + ] }, { "type": "gemm", "args": { - "m": 329, - "n": 32768, - "k": 512, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 12, + "n": 7168, + "k": 2304, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 32 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 1, + 32 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "amd/DeepSeek-R1-0528-MXFP4-Preview/down_proj", + "amd/DeepSeek-R1-0528-MXFP4-v2/down_proj", + "amd/DeepSeek-R1-0528-MXFP4/down_proj", + "amd/Kimi-K2.5-MXFP4/down_proj" + ] }, { "type": "gemm", "args": { - "m": 329, + "m": 12, "n": 7168, - "k": 16384, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "k": 2304, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/down_proj" + ] }, { "type": "gemm", "args": { - "m": 329, - "n": 4608, - "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 12, + "n": 7168, + "k": 4096, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/o_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/o_proj" + ] }, { "type": "gemm", "args": { - "m": 329, + "m": 12, "n": 7168, "k": 4608, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/down_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/down_proj", + "nvidia/Kimi-K2.5-NVFP4/down_proj" + ] }, { "type": "gemm", "args": { - "m": 673792, - "n": 1536, - "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 12, + "n": 7168, + "k": 9216, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/down_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/down_proj", + "nvidia/Kimi-K2.5-NVFP4/down_proj" + ] }, { "type": "gemm", "args": { - "m": 673792, - "n": 24576, - "k": 1536, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 12, + "n": 7168, + "k": 16384, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/o_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/o_proj" + ] }, { "type": "gemm", "args": { - "m": 673792, - "n": 576, - "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 12, + "n": 7168, + "k": 16384, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 32 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 1, + 32 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "amd/DeepSeek-R1-0528-MXFP4-Preview/o_proj", + "amd/DeepSeek-R1-0528-MXFP4-v2/o_proj", + "amd/DeepSeek-R1-0528-MXFP4/o_proj" + ] }, { "type": "gemm", "args": { - "m": 673792, - "n": 32768, - "k": 512, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 12, + "n": 7168, + "k": 16384, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/o_proj", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] }, { "type": "gemm", "args": { - "m": 673792, + "m": 12, "n": 7168, "k": 16384, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] }, { "type": "gemm", "args": { - "m": 673792, - "n": 2304, - "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 12, + "n": 8192, + "k": 512, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/Kimi-K2.5-MXFP4/kv_b_proj", + "moonshotai/Kimi-K2.5/kv_b_proj", + "nvidia/DeepSeek-R1-0528-FP4-V2/kv_b_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/kv_b_proj", + "nvidia/Kimi-K2.5-NVFP4/kv_b_proj", + "nvidia/Kimi-K2.6-NVFP4/kv_b_proj" + ] }, { "type": "gemm", "args": { - "m": 673792, - "n": 7168, - "k": 2304, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 12, + "n": 9216, + "k": 7168, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/gate_up_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/gate_up_proj", + "nvidia/Kimi-K2.5-NVFP4/gate_up_proj" + ] }, { "type": "gemm", "args": { - "m": 164, - "n": 1536, - "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 12, + "n": 24576, + "k": 1536, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/q_b_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/q_b_proj" + ] }, { "type": "gemm", "args": { - "m": 164, + "m": 12, "n": 24576, "k": 1536, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 32 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 1, + 32 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "amd/DeepSeek-R1-0528-MXFP4-Preview/q_b_proj", + "amd/DeepSeek-R1-0528-MXFP4-v2/q_b_proj", + "amd/DeepSeek-R1-0528-MXFP4/q_b_proj" + ] }, { "type": "gemm", "args": { - "m": 164, - "n": 576, - "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 12, + "n": 24576, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/q_b_proj" + ] }, { "type": "gemm", "args": { - "m": 164, + "m": 12, "n": 32768, "k": 512, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/kv_b_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/kv_b_proj" + ] }, { "type": "gemm", "args": { - "m": 164, - "n": 7168, - "k": 16384, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 12, + "n": 32768, + "k": 512, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 32 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 1, + 32 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "amd/DeepSeek-R1-0528-MXFP4-Preview/kv_b_proj", + "amd/DeepSeek-R1-0528-MXFP4-v2/kv_b_proj", + "amd/DeepSeek-R1-0528-MXFP4/kv_b_proj" + ] }, { "type": "gemm", "args": { - "m": 164, - "n": 2304, - "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 12, + "n": 32768, + "k": 512, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/kv_b_proj" + ] }, { "type": "gemm", "args": { - "m": 164, - "n": 7168, - "k": 2304, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 13, + "n": 576, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/Kimi-K2.5-MXFP4/kv_a_proj_with_mqa", + "moonshotai/Kimi-K2.5/kv_a_proj_with_mqa", + "nvidia/DeepSeek-R1-0528-FP4-V2/kv_a_proj_with_mqa", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/kv_a_proj_with_mqa", + "nvidia/Kimi-K2.5-NVFP4/kv_a_proj_with_mqa", + "nvidia/Kimi-K2.6-NVFP4/kv_a_proj_with_mqa" + ] }, { "type": "gemm", "args": { - "m": 51200, - "n": 1536, + "m": 13, + "n": 576, "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 32 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 1, + 32 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "amd/DeepSeek-R1-0528-MXFP4-Preview/kv_a_proj_with_mqa", + "amd/DeepSeek-R1-0528-MXFP4-v2/kv_a_proj_with_mqa", + "amd/DeepSeek-R1-0528-MXFP4/kv_a_proj_with_mqa" + ] }, { "type": "gemm", "args": { - "m": 51200, - "n": 24576, - "k": 1536, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 13, + "n": 1536, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/Kimi-K2.5-MXFP4/q_a_proj", + "moonshotai/Kimi-K2.5/q_a_proj", + "nvidia/DeepSeek-R1-0528-FP4-V2/q_a_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/q_a_proj", + "nvidia/Kimi-K2.5-NVFP4/q_a_proj", + "nvidia/Kimi-K2.6-NVFP4/q_a_proj" + ] }, { "type": "gemm", "args": { - "m": 51200, - "n": 576, + "m": 13, + "n": 1536, "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 32 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 1, + 32 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "amd/DeepSeek-R1-0528-MXFP4-Preview/q_a_proj", + "amd/DeepSeek-R1-0528-MXFP4-v2/q_a_proj", + "amd/DeepSeek-R1-0528-MXFP4/q_a_proj" + ] }, { "type": "gemm", "args": { - "m": 51200, - "n": 32768, - "k": 512, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 13, + "n": 1536, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w3" + ] }, { "type": "gemm", "args": { - "m": 51200, - "n": 7168, - "k": 16384, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 13, + "n": 4608, + "k": 7168, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/gate_up_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/gate_up_proj", + "nvidia/Kimi-K2.5-NVFP4/gate_up_proj" + ] }, { "type": "gemm", "args": { - "m": 51200, + "m": 13, "n": 4608, "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 32 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 1, + 32 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "amd/DeepSeek-R1-0528-MXFP4-Preview/gate_up_proj", + "amd/DeepSeek-R1-0528-MXFP4-v2/gate_up_proj", + "amd/DeepSeek-R1-0528-MXFP4/gate_up_proj", + "amd/Kimi-K2.5-MXFP4/gate_up_proj" + ] }, { "type": "gemm", "args": { - "m": 51200, + "m": 13, "n": 7168, "k": 4608, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/down_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/down_proj", + "nvidia/Kimi-K2.5-NVFP4/down_proj" + ] }, { "type": "gemm", "args": { - "m": 102400, - "n": 1536, - "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 13, + "n": 7168, + "k": 4608, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 32 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 1, + 32 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "amd/DeepSeek-R1-0528-MXFP4-Preview/down_proj", + "amd/DeepSeek-R1-0528-MXFP4-v2/down_proj", + "amd/DeepSeek-R1-0528-MXFP4/down_proj", + "amd/Kimi-K2.5-MXFP4/down_proj" + ] }, { "type": "gemm", "args": { - "m": 102400, - "n": 24576, - "k": 1536, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 13, + "n": 7168, + "k": 8192, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/Kimi-K2.5-MXFP4/o_proj", + "moonshotai/Kimi-K2.5/o_proj", + "nvidia/Kimi-K2.5-NVFP4/o_proj", + "nvidia/Kimi-K2.6-NVFP4/o_proj" + ] }, { "type": "gemm", "args": { - "m": 102400, - "n": 576, - "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 13, + "n": 7168, + "k": 8192, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] }, { "type": "gemm", "args": { - "m": 102400, - "n": 32768, - "k": 512, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 13, + "n": 12288, + "k": 1536, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/Kimi-K2.5-MXFP4/q_b_proj", + "moonshotai/Kimi-K2.5/q_b_proj", + "nvidia/DeepSeek-R1-0528-FP4-V2/q_b_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/q_b_proj", + "nvidia/Kimi-K2.5-NVFP4/q_b_proj", + "nvidia/Kimi-K2.6-NVFP4/q_b_proj" + ] }, { "type": "gemm", "args": { - "m": 102400, - "n": 7168, - "k": 16384, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 13, + "n": 16384, + "k": 512, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/Kimi-K2.5-MXFP4/kv_b_proj", + "moonshotai/Kimi-K2.5/kv_b_proj", + "nvidia/DeepSeek-R1-0528-FP4-V2/kv_b_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/kv_b_proj", + "nvidia/Kimi-K2.5-NVFP4/kv_b_proj", + "nvidia/Kimi-K2.6-NVFP4/kv_b_proj" + ] }, { "type": "gemm", "args": { - "m": 102400, - "n": 4608, + "m": 14, + "n": 576, "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/kv_a_proj_with_mqa" + ] }, { "type": "gemm", "args": { - "m": 102400, - "n": 7168, - "k": 4608, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 14, + "n": 1152, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/gate_up_proj" + ] }, { "type": "gemm", "args": { - "m": 204800, + "m": 14, "n": 1536, "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 204800, - "n": 24576, - "k": 1536, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/gate_up_proj", + "deepseek-ai/DeepSeek-R1-0528/q_a_proj", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w3" + ] }, { "type": "gemm", "args": { - "m": 204800, - "n": 576, + "m": 14, + "n": 1536, "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 204800, - "n": 32768, - "k": 512, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w3" + ] }, { "type": "gemm", "args": { - "m": 204800, + "m": 14, "n": 7168, - "k": 16384, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 204800, - "n": 4608, - "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "k": 1152, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/down_proj" + ] }, { "type": "gemm", "args": { - "m": 204800, + "m": 14, "n": 7168, - "k": 4608, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "k": 16384, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/o_proj", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] }, { "type": "gemm", "args": { - "m": 25, + "m": 14, "n": 24576, "k": 1536, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/q_b_proj" + ] }, { "type": "gemm", "args": { - "m": 25, + "m": 14, "n": 32768, "k": 512, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/kv_b_proj" + ] }, { "type": "gemm", "args": { - "m": 25, - "n": 7168, - "k": 16384, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 15, + "n": 576, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/Kimi-K2.5-MXFP4/kv_a_proj_with_mqa", + "moonshotai/Kimi-K2.5/kv_a_proj_with_mqa", + "nvidia/DeepSeek-R1-0528-FP4-V2/kv_a_proj_with_mqa", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/kv_a_proj_with_mqa", + "nvidia/Kimi-K2.5-NVFP4/kv_a_proj_with_mqa", + "nvidia/Kimi-K2.6-NVFP4/kv_a_proj_with_mqa" + ] }, { "type": "gemm", "args": { - "m": 25, - "n": 4608, + "m": 15, + "n": 1536, "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/Kimi-K2.5-MXFP4/q_a_proj", + "moonshotai/Kimi-K2.5/q_a_proj", + "nvidia/DeepSeek-R1-0528-FP4-V2/q_a_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/q_a_proj", + "nvidia/Kimi-K2.5-NVFP4/q_a_proj", + "nvidia/Kimi-K2.6-NVFP4/q_a_proj" + ] }, { "type": "gemm", "args": { - "m": 25, - "n": 7168, - "k": 4608, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 15, + "n": 2304, + "k": 7168, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/gate_up_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/gate_up_proj", + "nvidia/Kimi-K2.5-NVFP4/gate_up_proj" + ] }, { "type": "gemm", "args": { - "m": 204800, + "m": 15, "n": 3072, "k": 1536, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/Kimi-K2.5-MXFP4/q_b_proj", + "amd/MiniMax-M2.5-MXFP4/o_proj", + "moonshotai/Kimi-K2.5/q_b_proj", + "nvidia/DeepSeek-R1-0528-FP4-V2/q_b_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/q_b_proj", + "nvidia/Kimi-K2.5-NVFP4/q_b_proj", + "nvidia/Kimi-K2.6-NVFP4/q_b_proj", + "nvidia/MiniMax-M2.5-NVFP4/o_proj" + ] }, { "type": "gemm", "args": { - "m": 204800, + "m": 15, "n": 4096, "k": 512, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "Qwen/Qwen3.5-397B-A17B/o_proj", + "Qwen/Qwen3.5-397B-A17B/shared_expert.down_proj", + "amd/Kimi-K2.5-MXFP4/kv_b_proj", + "amd/Qwen3.5-397B-A17B-MXFP4/o_proj", + "amd/Qwen3.5-397B-A17B-MXFP4/shared_expert.down_proj", + "moonshotai/Kimi-K2.5/kv_b_proj", + "nvidia/DeepSeek-R1-0528-FP4-V2/kv_b_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/kv_b_proj", + "nvidia/Kimi-K2.5-NVFP4/kv_b_proj", + "nvidia/Kimi-K2.6-NVFP4/kv_b_proj", + "nvidia/Qwen3.5-397B-A17B-NVFP4/o_proj", + "nvidia/Qwen3.5-397B-A17B-NVFP4/shared_expert.down_proj" + ] }, { "type": "gemm", "args": { - "m": 204800, - "n": 7168, - "k": 2048, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 15, + "n": 4608, + "k": 7168, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/gate_up_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/gate_up_proj", + "nvidia/Kimi-K2.5-NVFP4/gate_up_proj" + ] }, { "type": "gemm", "args": { - "m": 204800, - "n": 2304, - "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 15, + "n": 6144, + "k": 1536, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/GLM-5.1-MXFP4/down_proj", + "amd/GLM-5.2-MXFP4/down_proj", + "amd/Kimi-K2.5-MXFP4/q_b_proj", + "amd/MiniMax-M3-MXFP4/down_proj", + "moonshotai/Kimi-K2.5/q_b_proj", + "nvidia/DeepSeek-R1-0528-FP4-V2/q_b_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/q_b_proj", + "nvidia/GLM-5.1-NVFP4/down_proj", + "nvidia/GLM-5.2-NVFP4/down_proj", + "nvidia/Kimi-K2.5-NVFP4/q_b_proj", + "nvidia/Kimi-K2.6-NVFP4/q_b_proj" + ] }, { "type": "gemm", "args": { - "m": 204800, + "m": 15, "n": 7168, - "k": 2304, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "k": 2048, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/Kimi-K2.5-MXFP4/o_proj", + "moonshotai/Kimi-K2.5/o_proj", + "nvidia/Kimi-K2.5-NVFP4/o_proj", + "nvidia/Kimi-K2.6-NVFP4/o_proj" + ] }, { "type": "gemm", "args": { - "m": 409600, - "n": 1536, - "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 15, + "n": 7168, + "k": 2048, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/o_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/o_proj" + ] }, { "type": "gemm", "args": { - "m": 409600, - "n": 3072, - "k": 1536, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 15, + "n": 7168, + "k": 2304, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/down_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/down_proj", + "nvidia/Kimi-K2.5-NVFP4/down_proj" + ] }, { "type": "gemm", "args": { - "m": 409600, - "n": 576, - "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 15, + "n": 7168, + "k": 4096, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/o_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/o_proj" + ] }, { "type": "gemm", "args": { - "m": 409600, - "n": 4096, - "k": 512, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 15, + "n": 7168, + "k": 4608, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/down_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/down_proj", + "nvidia/Kimi-K2.5-NVFP4/down_proj" + ] }, { "type": "gemm", "args": { - "m": 409600, + "m": 15, "n": 7168, - "k": 2048, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "k": 8192, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/Kimi-K2.5-MXFP4/o_proj", + "moonshotai/Kimi-K2.5/o_proj", + "nvidia/Kimi-K2.5-NVFP4/o_proj", + "nvidia/Kimi-K2.6-NVFP4/o_proj" + ] }, { "type": "gemm", "args": { - "m": 409600, - "n": 2304, - "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 15, + "n": 7168, + "k": 8192, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] }, { "type": "gemm", "args": { - "m": 409600, + "m": 15, "n": 7168, - "k": 2304, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "k": 9216, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/down_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/down_proj", + "nvidia/Kimi-K2.5-NVFP4/down_proj" + ] }, { "type": "gemm", "args": { - "m": 819200, - "n": 1536, - "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 15, + "n": 7168, + "k": 16384, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/o_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/o_proj" + ] }, { "type": "gemm", "args": { - "m": 819200, - "n": 3072, - "k": 1536, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 15, + "n": 7168, + "k": 16384, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 32 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 1, + 32 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "amd/DeepSeek-R1-0528-MXFP4-Preview/o_proj", + "amd/DeepSeek-R1-0528-MXFP4-v2/o_proj", + "amd/DeepSeek-R1-0528-MXFP4/o_proj" + ] }, { "type": "gemm", "args": { - "m": 819200, - "n": 576, - "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 15, + "n": 7168, + "k": 16384, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] }, { "type": "gemm", "args": { - "m": 819200, - "n": 4096, + "m": 15, + "n": 8192, "k": 512, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/Kimi-K2.5-MXFP4/kv_b_proj", + "moonshotai/Kimi-K2.5/kv_b_proj", + "nvidia/DeepSeek-R1-0528-FP4-V2/kv_b_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/kv_b_proj", + "nvidia/Kimi-K2.5-NVFP4/kv_b_proj", + "nvidia/Kimi-K2.6-NVFP4/kv_b_proj" + ] }, { "type": "gemm", "args": { - "m": 819200, - "n": 7168, - "k": 2048, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 819200, - "n": 2304, + "m": 15, + "n": 9216, "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/gate_up_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/gate_up_proj", + "nvidia/Kimi-K2.5-NVFP4/gate_up_proj" + ] }, { "type": "gemm", "args": { - "m": 819200, - "n": 7168, - "k": 2304, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 15, + "n": 12288, + "k": 1536, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/Kimi-K2.5-MXFP4/q_b_proj", + "moonshotai/Kimi-K2.5/q_b_proj", + "nvidia/DeepSeek-R1-0528-FP4-V2/q_b_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/q_b_proj", + "nvidia/Kimi-K2.5-NVFP4/q_b_proj", + "nvidia/Kimi-K2.6-NVFP4/q_b_proj" + ] }, { "type": "gemm", "args": { - "m": 100, - "n": 1536, - "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 15, + "n": 16384, + "k": 512, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/Kimi-K2.5-MXFP4/kv_b_proj", + "moonshotai/Kimi-K2.5/kv_b_proj", + "nvidia/DeepSeek-R1-0528-FP4-V2/kv_b_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/kv_b_proj", + "nvidia/Kimi-K2.5-NVFP4/kv_b_proj", + "nvidia/Kimi-K2.6-NVFP4/kv_b_proj" + ] }, { "type": "gemm", "args": { - "m": 100, - "n": 3072, + "m": 15, + "n": 24576, "k": 1536, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/q_b_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/q_b_proj" + ] }, { "type": "gemm", "args": { - "m": 100, - "n": 576, - "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 15, + "n": 24576, + "k": 1536, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 32 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 1, + 32 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "amd/DeepSeek-R1-0528-MXFP4-Preview/q_b_proj", + "amd/DeepSeek-R1-0528-MXFP4-v2/q_b_proj", + "amd/DeepSeek-R1-0528-MXFP4/q_b_proj" + ] }, { "type": "gemm", "args": { - "m": 100, - "n": 4096, + "m": 15, + "n": 32768, "k": 512, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/kv_b_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/kv_b_proj" + ] }, { "type": "gemm", "args": { - "m": 100, - "n": 7168, - "k": 2048, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 15, + "n": 32768, + "k": 512, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 32 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 1, + 32 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "amd/DeepSeek-R1-0528-MXFP4-Preview/kv_b_proj", + "amd/DeepSeek-R1-0528-MXFP4-v2/kv_b_proj", + "amd/DeepSeek-R1-0528-MXFP4/kv_b_proj" + ] }, { "type": "gemm", "args": { - "m": 100, - "n": 2304, - "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 16, + "n": 64, + "k": 2880, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/gpt-oss-120b-w-mxfp4-a-fp8/k_proj", + "amd/gpt-oss-120b-w-mxfp4-a-fp8/v_proj", + "openai/gpt-oss-120b/k_proj", + "openai/gpt-oss-120b/v_proj" + ] }, { "type": "gemm", "args": { - "m": 100, - "n": 7168, - "k": 2304, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 16, + "n": 128, + "k": 2880, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/gpt-oss-120b-w-mxfp4-a-fp8/k_proj", + "amd/gpt-oss-120b-w-mxfp4-a-fp8/v_proj", + "openai/gpt-oss-120b/k_proj", + "openai/gpt-oss-120b/v_proj" + ] }, { "type": "gemm", "args": { - "m": 757760, - "n": 1536, - "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 16, + "n": 128, + "k": 3072, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/MiniMax-M2.5-MXFP4/k_proj", + "amd/MiniMax-M2.5-MXFP4/v_proj", + "nvidia/MiniMax-M2.5-NVFP4/k_proj", + "nvidia/MiniMax-M2.5-NVFP4/v_proj" + ] }, { "type": "gemm", "args": { - "m": 757760, - "n": 24576, - "k": 1536, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 16, + "n": 128, + "k": 3072, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "MiniMaxAI/MiniMax-M2.5/k_proj", + "MiniMaxAI/MiniMax-M2.5/v_proj" + ] }, { "type": "gemm", "args": { - "m": 757760, - "n": 576, - "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 16, + "n": 128, + "k": 4096, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "Qwen/Qwen3.5-397B-A17B/shared_expert.gate_up_proj", + "amd/Qwen3.5-397B-A17B-MXFP4/shared_expert.gate_up_proj", + "nvidia/Qwen3.5-397B-A17B-NVFP4/shared_expert.gate_up_proj" + ] }, { "type": "gemm", "args": { - "m": 757760, - "n": 32768, - "k": 512, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 16, + "n": 128, + "k": 4096, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "Qwen/Qwen3.5-397B-A17B-FP8/shared_expert.gate_up_proj" + ] }, { "type": "gemm", "args": { - "m": 757760, - "n": 7168, - "k": 16384, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 16, + "n": 192, + "k": 3072, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/MiniMax-M2.5-NVFP4/experts.gate_up_proj" + ] }, { "type": "gemm", "args": { - "m": 757760, - "n": 4608, - "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 16, + "n": 192, + "k": 3072, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 32 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 1, + 32 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "amd/MiniMax-M2.5-MXFP4/experts.gate_up_proj" + ] }, { "type": "gemm", "args": { - "m": 757760, - "n": 7168, - "k": 4608, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 16, + "n": 192, + "k": 3072, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "MiniMaxAI/MiniMax-M2.5/experts.gate_up_proj" + ] }, { "type": "gemm", "args": { - "m": 92, - "n": 1536, - "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 16, + "n": 256, + "k": 2880, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/gpt-oss-120b-w-mxfp4-a-fp8/k_proj", + "amd/gpt-oss-120b-w-mxfp4-a-fp8/q_proj", + "amd/gpt-oss-120b-w-mxfp4-a-fp8/v_proj", + "openai/gpt-oss-120b/k_proj", + "openai/gpt-oss-120b/q_proj", + "openai/gpt-oss-120b/v_proj" + ] }, { "type": "gemm", "args": { - "m": 92, - "n": 24576, - "k": 1536, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 16, + "n": 256, + "k": 3072, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/MiniMax-M2.5-MXFP4/k_proj", + "amd/MiniMax-M2.5-MXFP4/v_proj", + "nvidia/MiniMax-M2.5-NVFP4/k_proj", + "nvidia/MiniMax-M2.5-NVFP4/v_proj" + ] }, { "type": "gemm", "args": { - "m": 92, - "n": 576, - "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 16, + "n": 256, + "k": 3072, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "MiniMaxAI/MiniMax-M2.5/k_proj", + "MiniMaxAI/MiniMax-M2.5/v_proj" + ] }, { "type": "gemm", "args": { - "m": 92, - "n": 32768, - "k": 512, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 16, + "n": 256, + "k": 4096, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "Qwen/Qwen3.5-397B-A17B/k_proj", + "Qwen/Qwen3.5-397B-A17B/shared_expert.gate_up_proj", + "Qwen/Qwen3.5-397B-A17B/v_proj", + "amd/Qwen3.5-397B-A17B-MXFP4/k_proj", + "amd/Qwen3.5-397B-A17B-MXFP4/shared_expert.gate_up_proj", + "amd/Qwen3.5-397B-A17B-MXFP4/v_proj", + "nvidia/Qwen3.5-397B-A17B-NVFP4/k_proj", + "nvidia/Qwen3.5-397B-A17B-NVFP4/shared_expert.gate_up_proj", + "nvidia/Qwen3.5-397B-A17B-NVFP4/v_proj" + ] }, { "type": "gemm", "args": { - "m": 92, - "n": 7168, - "k": 16384, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 16, + "n": 256, + "k": 4096, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "Qwen/Qwen3.5-397B-A17B-FP8/k_proj", + "Qwen/Qwen3.5-397B-A17B-FP8/shared_expert.gate_up_proj", + "Qwen/Qwen3.5-397B-A17B-FP8/v_proj" + ] }, { "type": "gemm", "args": { - "m": 92, - "n": 4608, - "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 16, + "n": 384, + "k": 3072, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 32 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 1, + 32 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "amd/MiniMax-M2.5-MXFP4/experts.gate_up_proj" + ] }, { "type": "gemm", "args": { - "m": 92, - "n": 7168, - "k": 4608, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 16, + "n": 384, + "k": 3072, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "MiniMaxAI/MiniMax-M2.5/experts.gate_up_proj", + "MiniMaxAI/MiniMax-M2.5/q_proj" + ] }, { "type": "gemm", "args": { - "m": 3031040, - "n": 1536, - "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 16, + "n": 512, + "k": 2880, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/gpt-oss-120b-w-mxfp4-a-fp8/k_proj", + "amd/gpt-oss-120b-w-mxfp4-a-fp8/q_proj", + "amd/gpt-oss-120b-w-mxfp4-a-fp8/v_proj", + "openai/gpt-oss-120b/k_proj", + "openai/gpt-oss-120b/q_proj", + "openai/gpt-oss-120b/v_proj" + ] }, { "type": "gemm", "args": { - "m": 3031040, - "n": 3072, - "k": 1536, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 16, + "n": 512, + "k": 3072, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/MiniMax-M2.5-MXFP4/k_proj", + "amd/MiniMax-M2.5-MXFP4/v_proj", + "nvidia/MiniMax-M2.5-NVFP4/k_proj", + "nvidia/MiniMax-M2.5-NVFP4/v_proj" + ] }, { "type": "gemm", "args": { - "m": 3031040, - "n": 576, - "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 16, + "n": 512, + "k": 3072, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "MiniMaxAI/MiniMax-M2.5/k_proj", + "MiniMaxAI/MiniMax-M2.5/v_proj" + ] }, { "type": "gemm", "args": { - "m": 3031040, - "n": 4096, - "k": 512, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 16, + "n": 512, + "k": 4096, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "Qwen/Qwen3.5-397B-A17B/k_proj", + "Qwen/Qwen3.5-397B-A17B/q_proj", + "Qwen/Qwen3.5-397B-A17B/shared_expert.gate_up_proj", + "Qwen/Qwen3.5-397B-A17B/v_proj", + "amd/Qwen3.5-397B-A17B-MXFP4/k_proj", + "amd/Qwen3.5-397B-A17B-MXFP4/q_proj", + "amd/Qwen3.5-397B-A17B-MXFP4/shared_expert.gate_up_proj", + "amd/Qwen3.5-397B-A17B-MXFP4/v_proj", + "nvidia/Qwen3.5-397B-A17B-NVFP4/k_proj", + "nvidia/Qwen3.5-397B-A17B-NVFP4/q_proj", + "nvidia/Qwen3.5-397B-A17B-NVFP4/shared_expert.gate_up_proj", + "nvidia/Qwen3.5-397B-A17B-NVFP4/v_proj" + ] }, { "type": "gemm", "args": { - "m": 3031040, - "n": 7168, - "k": 2048, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 16, + "n": 512, + "k": 4096, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "Qwen/Qwen3.5-397B-A17B-FP8/k_proj", + "Qwen/Qwen3.5-397B-A17B-FP8/q_proj", + "Qwen/Qwen3.5-397B-A17B-FP8/shared_expert.gate_up_proj", + "Qwen/Qwen3.5-397B-A17B-FP8/v_proj", + "deepseek-ai/DeepSeek-V4.1-Flash/attn.wo_a" + ] }, { "type": "gemm", "args": { - "m": 3031040, - "n": 2304, - "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 16, + "n": 576, + "k": 6144, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/GLM-5.1-MXFP4/kv_a_proj_with_mqa", + "amd/GLM-5.2-MXFP4/kv_a_proj_with_mqa", + "nvidia/GLM-5-NVFP4/kv_a_proj_with_mqa", + "nvidia/GLM-5.1-NVFP4/kv_a_proj_with_mqa", + "nvidia/GLM-5.2-NVFP4/kv_a_proj_with_mqa" + ] }, { "type": "gemm", "args": { - "m": 3031040, - "n": 7168, - "k": 2304, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 16, + "n": 576, + "k": 6144, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "zai-org/GLM-5-FP8/kv_a_proj_with_mqa", + "zai-org/GLM-5.1-FP8/kv_a_proj_with_mqa", + "zai-org/GLM-5.2-FP8/kv_a_proj_with_mqa" + ] }, { "type": "gemm", "args": { - "m": 370, - "n": 1536, + "m": 16, + "n": 576, "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/Kimi-K2.5-MXFP4/kv_a_proj_with_mqa", + "moonshotai/Kimi-K2.5/kv_a_proj_with_mqa", + "nvidia/DeepSeek-R1-0528-FP4-V2/kv_a_proj_with_mqa", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/kv_a_proj_with_mqa", + "nvidia/Kimi-K2.5-NVFP4/kv_a_proj_with_mqa", + "nvidia/Kimi-K2.6-NVFP4/kv_a_proj_with_mqa" + ] }, { "type": "gemm", "args": { - "m": 370, - "n": 3072, - "k": 1536, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 16, + "n": 576, + "k": 7168, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 32 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 1, + 32 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "amd/DeepSeek-R1-0528-MXFP4-Preview/kv_a_proj_with_mqa", + "amd/DeepSeek-R1-0528-MXFP4-v2/kv_a_proj_with_mqa", + "amd/DeepSeek-R1-0528-MXFP4/kv_a_proj_with_mqa" + ] }, { "type": "gemm", "args": { - "m": 370, + "m": 16, "n": 576, "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/kv_a_proj_with_mqa" + ] }, { "type": "gemm", "args": { - "m": 370, - "n": 4096, - "k": 512, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 16, + "n": 768, + "k": 3072, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/MiniMax-M2.5-NVFP4/experts.gate_up_proj" + ] }, { "type": "gemm", "args": { - "m": 370, - "n": 7168, - "k": 2048, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 16, + "n": 768, + "k": 3072, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 32 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 1, + 32 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "amd/MiniMax-M2.5-MXFP4/experts.gate_up_proj" + ] }, { "type": "gemm", "args": { - "m": 370, - "n": 2304, - "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 16, + "n": 768, + "k": 3072, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "MiniMaxAI/MiniMax-M2.5/experts.gate_up_proj", + "MiniMaxAI/MiniMax-M2.5/q_proj" + ] }, { "type": "gemm", "args": { - "m": 370, - "n": 7168, - "k": 2304, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 16, + "n": 1024, + "k": 2880, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/gpt-oss-120b-w-mxfp4-a-fp8/q_proj", + "openai/gpt-oss-120b/q_proj" + ] }, { "type": "gemm", "args": { - "m": 3289088, - "n": 1536, - "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 16, + "n": 1024, + "k": 3072, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/MiniMax-M2.5-MXFP4/k_proj", + "amd/MiniMax-M2.5-MXFP4/qkv_proj", + "amd/MiniMax-M2.5-MXFP4/v_proj", + "nvidia/MiniMax-M2.5-NVFP4/k_proj", + "nvidia/MiniMax-M2.5-NVFP4/qkv_proj", + "nvidia/MiniMax-M2.5-NVFP4/v_proj" + ] }, { "type": "gemm", "args": { - "m": 3289088, - "n": 24576, - "k": 1536, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 16, + "n": 1024, + "k": 3072, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "MiniMaxAI/MiniMax-M2.5/k_proj", + "MiniMaxAI/MiniMax-M2.5/qkv_proj", + "MiniMaxAI/MiniMax-M2.5/v_proj" + ] }, { "type": "gemm", "args": { - "m": 3289088, - "n": 576, - "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 16, + "n": 1024, + "k": 4096, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "Qwen/Qwen3.5-397B-A17B/q_proj", + "Qwen/Qwen3.5-397B-A17B/shared_expert.gate_up_proj", + "amd/Qwen3.5-397B-A17B-MXFP4/q_proj", + "amd/Qwen3.5-397B-A17B-MXFP4/shared_expert.gate_up_proj", + "nvidia/Qwen3.5-397B-A17B-NVFP4/q_proj", + "nvidia/Qwen3.5-397B-A17B-NVFP4/shared_expert.gate_up_proj" + ] }, { "type": "gemm", "args": { - "m": 3289088, - "n": 32768, - "k": 512, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 16, + "n": 1024, + "k": 4096, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "Qwen/Qwen3.5-397B-A17B-FP8/q_proj", + "Qwen/Qwen3.5-397B-A17B-FP8/shared_expert.gate_up_proj", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_a", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_a", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_a" + ] }, { "type": "gemm", "args": { - "m": 3289088, - "n": 7168, - "k": 16384, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 16, + "n": 1088, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] }, { "type": "gemm", "args": { - "m": 3289088, - "n": 4608, + "m": 16, + "n": 1152, "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-NVFP4-v2/gate_up_proj" + ] }, { "type": "gemm", "args": { - "m": 3289088, - "n": 7168, - "k": 4608, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 16, + "n": 1152, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/gate_up_proj" + ] }, { "type": "gemm", "args": { - "m": 401, + "m": 16, "n": 1536, - "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "k": 1536, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/Kimi-K2.5-MXFP4/q_b_proj", + "moonshotai/Kimi-K2.5/q_b_proj", + "nvidia/DeepSeek-R1-0528-FP4-V2/q_b_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/q_b_proj", + "nvidia/Kimi-K2.5-NVFP4/q_b_proj", + "nvidia/Kimi-K2.6-NVFP4/q_b_proj" + ] }, { "type": "gemm", "args": { - "m": 401, - "n": 24576, - "k": 1536, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 16, + "n": 1536, + "k": 3072, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/MiniMax-M2.5-NVFP4/experts.gate_up_proj" + ] }, { "type": "gemm", "args": { - "m": 401, - "n": 576, - "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 16, + "n": 1536, + "k": 3072, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 32 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 1, + 32 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "amd/MiniMax-M2.5-MXFP4/experts.gate_up_proj" + ] }, { "type": "gemm", "args": { - "m": 401, - "n": 32768, - "k": 512, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 16, + "n": 1536, + "k": 3072, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "MiniMaxAI/MiniMax-M2.5/experts.gate_up_proj", + "MiniMaxAI/MiniMax-M2.5/q_proj" + ] }, { "type": "gemm", "args": { - "m": 401, - "n": 7168, - "k": 16384, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 16, + "n": 1536, + "k": 6144, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "zai-org/GLM-5-FP8/gate_up_proj", + "zai-org/GLM-5.1-FP8/gate_up_proj", + "zai-org/GLM-5.2-FP8/gate_up_proj" + ] }, { "type": "gemm", "args": { - "m": 401, - "n": 4608, + "m": 16, + "n": 1536, "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/Kimi-K2.5-MXFP4/q_a_proj", + "moonshotai/Kimi-K2.5/q_a_proj", + "nvidia/DeepSeek-R1-0528-FP4-V2/q_a_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/q_a_proj", + "nvidia/Kimi-K2.5-NVFP4/q_a_proj", + "nvidia/Kimi-K2.6-NVFP4/q_a_proj" + ] }, { "type": "gemm", "args": { - "m": 401, - "n": 7168, - "k": 4608, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 16, + "n": 1536, + "k": 7168, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 32 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 1, + 32 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "amd/DeepSeek-R1-0528-MXFP4-Preview/q_a_proj", + "amd/DeepSeek-R1-0528-MXFP4-v2/q_a_proj", + "amd/DeepSeek-R1-0528-MXFP4/q_a_proj" + ] }, { "type": "gemm", "args": { - "m": 1644544, + "m": 16, "n": 1536, "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/gate_up_proj", + "deepseek-ai/DeepSeek-R1-0528/q_a_proj", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w3" + ] }, { "type": "gemm", "args": { - "m": 1644544, - "n": 24576, - "k": 1536, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 16, + "n": 1536, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w3" + ] }, { "type": "gemm", "args": { - "m": 1644544, - "n": 576, - "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 16, + "n": 2048, + "k": 512, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/Kimi-K2.5-MXFP4/kv_b_proj", + "moonshotai/Kimi-K2.5/kv_b_proj", + "nvidia/DeepSeek-R1-0528-FP4-V2/kv_b_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/kv_b_proj", + "nvidia/Kimi-K2.5-NVFP4/kv_b_proj", + "nvidia/Kimi-K2.6-NVFP4/kv_b_proj" + ] }, { "type": "gemm", "args": { - "m": 1644544, - "n": 32768, - "k": 512, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 16, + "n": 2048, + "k": 2048, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "zai-org/GLM-5-FP8/q_b_proj", + "zai-org/GLM-5.1-FP8/q_b_proj", + "zai-org/GLM-5.2-FP8/q_b_proj" + ] }, { "type": "gemm", "args": { - "m": 1644544, - "n": 7168, - "k": 16384, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 16, + "n": 2048, + "k": 2880, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/gpt-oss-120b-w-mxfp4-a-fp8/q_proj", + "openai/gpt-oss-120b/q_proj" + ] }, { "type": "gemm", "args": { - "m": 1644544, - "n": 2304, - "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 16, + "n": 2048, + "k": 4096, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "Qwen/Qwen3.5-397B-A17B/q_proj", + "Qwen/Qwen3.5-397B-A17B/shared_expert.gate_up_proj", + "amd/Qwen3.5-397B-A17B-MXFP4/q_proj", + "amd/Qwen3.5-397B-A17B-MXFP4/shared_expert.gate_up_proj", + "nvidia/Qwen3.5-397B-A17B-NVFP4/q_proj", + "nvidia/Qwen3.5-397B-A17B-NVFP4/shared_expert.gate_up_proj" + ] }, { "type": "gemm", "args": { - "m": 1644544, - "n": 7168, - "k": 2304, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 16, + "n": 2048, + "k": 4096, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "Qwen/Qwen3.5-397B-A17B-FP8/q_proj", + "Qwen/Qwen3.5-397B-A17B-FP8/shared_expert.gate_up_proj" + ] }, { "type": "gemm", "args": { - "m": 200, - "n": 1536, - "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 16, + "n": 2048, + "k": 6144, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/GLM-5.1-MXFP4/q_a_proj", + "amd/GLM-5.2-MXFP4/q_a_proj", + "amd/MiniMax-M3-MXFP4/q_proj", + "nvidia/GLM-5-NVFP4/q_a_proj", + "nvidia/GLM-5.1-NVFP4/q_a_proj", + "nvidia/GLM-5.2-NVFP4/q_a_proj" + ] }, { "type": "gemm", "args": { - "m": 200, - "n": 24576, - "k": 1536, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 16, + "n": 2048, + "k": 6144, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "zai-org/GLM-5-FP8/q_a_proj", + "zai-org/GLM-5.1-FP8/q_a_proj", + "zai-org/GLM-5.2-FP8/q_a_proj" + ] }, { "type": "gemm", "args": { - "m": 200, - "n": 576, + "m": 16, + "n": 2304, "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/gate_up_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/gate_up_proj", + "nvidia/Kimi-K2.5-NVFP4/gate_up_proj" + ] }, { "type": "gemm", "args": { - "m": 200, - "n": 32768, + "m": 16, + "n": 2880, "k": 512, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/gpt-oss-120b-w-mxfp4-a-fp8/o_proj", + "openai/gpt-oss-120b/o_proj" + ] }, { "type": "gemm", "args": { - "m": 200, - "n": 7168, - "k": 16384, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 16, + "n": 2880, + "k": 1024, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/gpt-oss-120b-w-mxfp4-a-fp8/o_proj", + "openai/gpt-oss-120b/o_proj" + ] }, { "type": "gemm", "args": { - "m": 200, - "n": 2304, - "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 16, + "n": 2880, + "k": 2048, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/gpt-oss-120b-w-mxfp4-a-fp8/o_proj", + "openai/gpt-oss-120b/o_proj" + ] }, { "type": "gemm", "args": { - "m": 200, - "n": 7168, - "k": 2304, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 16, + "n": 2880, + "k": 4096, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/gpt-oss-120b-w-mxfp4-a-fp8/o_proj", + "openai/gpt-oss-120b/o_proj" + ] }, { "type": "gemm", "args": { - "m": 1714176, - "n": 1536, - "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 16, + "n": 3072, + "k": 192, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 32 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 1, + 32 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "amd/MiniMax-M2.5-MXFP4/experts.down_proj" + ] }, { "type": "gemm", "args": { - "m": 1714176, - "n": 24576, - "k": 1536, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 16, + "n": 3072, + "k": 192, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "MiniMaxAI/MiniMax-M2.5/experts.down_proj" + ] }, { "type": "gemm", "args": { - "m": 1714176, - "n": 576, - "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 16, + "n": 3072, + "k": 384, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/MiniMax-M2.5-NVFP4/experts.down_proj" + ] }, { "type": "gemm", "args": { - "m": 1714176, - "n": 32768, - "k": 512, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 16, + "n": 3072, + "k": 384, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 32 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 1, + 32 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "amd/MiniMax-M2.5-MXFP4/experts.down_proj" + ] }, { "type": "gemm", "args": { - "m": 1714176, - "n": 7168, - "k": 16384, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 16, + "n": 3072, + "k": 384, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "MiniMaxAI/MiniMax-M2.5/experts.down_proj", + "MiniMaxAI/MiniMax-M2.5/o_proj" + ] }, { "type": "gemm", "args": { - "m": 1714176, - "n": 4608, - "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 16, + "n": 3072, + "k": 768, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/MiniMax-M2.5-MXFP4/o_proj", + "nvidia/MiniMax-M2.5-NVFP4/o_proj" + ] }, { "type": "gemm", "args": { - "m": 1714176, - "n": 7168, - "k": 4608, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 16, + "n": 3072, + "k": 768, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/MiniMax-M2.5-NVFP4/experts.down_proj" + ] }, { "type": "gemm", "args": { - "m": 209, - "n": 1536, - "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 16, + "n": 3072, + "k": 768, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 32 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 1, + 32 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "amd/MiniMax-M2.5-MXFP4/experts.down_proj" + ] }, { "type": "gemm", "args": { - "m": 209, - "n": 24576, - "k": 1536, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 16, + "n": 3072, + "k": 768, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "MiniMaxAI/MiniMax-M2.5/experts.down_proj", + "MiniMaxAI/MiniMax-M2.5/o_proj" + ] }, { "type": "gemm", "args": { - "m": 209, - "n": 576, - "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 16, + "n": 3072, + "k": 1536, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/Kimi-K2.5-MXFP4/q_b_proj", + "amd/MiniMax-M2.5-MXFP4/o_proj", + "moonshotai/Kimi-K2.5/q_b_proj", + "nvidia/DeepSeek-R1-0528-FP4-V2/q_b_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/q_b_proj", + "nvidia/Kimi-K2.5-NVFP4/q_b_proj", + "nvidia/Kimi-K2.6-NVFP4/q_b_proj", + "nvidia/MiniMax-M2.5-NVFP4/o_proj" + ] }, { "type": "gemm", "args": { - "m": 209, - "n": 32768, - "k": 512, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 16, + "n": 3072, + "k": 1536, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/MiniMax-M2.5-NVFP4/experts.down_proj" + ] }, { "type": "gemm", "args": { - "m": 209, - "n": 7168, - "k": 16384, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 16, + "n": 3072, + "k": 1536, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 32 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 1, + 32 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "amd/DeepSeek-R1-0528-MXFP4-Preview/q_b_proj", + "amd/DeepSeek-R1-0528-MXFP4-v2/q_b_proj", + "amd/DeepSeek-R1-0528-MXFP4/q_b_proj", + "amd/MiniMax-M2.5-MXFP4/experts.down_proj" + ] }, { "type": "gemm", "args": { - "m": 209, - "n": 4608, - "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 16, + "n": 3072, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "MiniMaxAI/MiniMax-M2.5/experts.down_proj", + "MiniMaxAI/MiniMax-M2.5/o_proj", + "deepseek-ai/DeepSeek-R1-0528/q_b_proj" + ] }, { "type": "gemm", "args": { - "m": 209, - "n": 7168, - "k": 4608, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 16, + "n": 3072, + "k": 3072, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/MiniMax-M2.5-MXFP4/o_proj", + "amd/MiniMax-M2.5-MXFP4/q_proj", + "nvidia/MiniMax-M2.5-NVFP4/o_proj", + "nvidia/MiniMax-M2.5-NVFP4/q_proj" + ] }, { "type": "gemm", "args": { - "m": 857088, - "n": 1536, - "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 16, + "n": 3072, + "k": 3072, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/MiniMax-M2.5-NVFP4/experts.gate_up_proj" + ] }, { "type": "gemm", "args": { - "m": 857088, - "n": 24576, - "k": 1536, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 16, + "n": 3072, + "k": 3072, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 32 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 1, + 32 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "amd/MiniMax-M2.5-MXFP4/experts.gate_up_proj" + ] }, { "type": "gemm", "args": { - "m": 857088, - "n": 576, - "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 16, + "n": 3072, + "k": 3072, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "MiniMaxAI/MiniMax-M2.5/experts.gate_up_proj", + "MiniMaxAI/MiniMax-M2.5/o_proj", + "MiniMaxAI/MiniMax-M2.5/q_proj" + ] }, { "type": "gemm", "args": { - "m": 857088, - "n": 32768, - "k": 512, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 16, + "n": 3072, + "k": 6144, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/GLM-5.1-MXFP4/gate_up_proj", + "amd/GLM-5.2-MXFP4/gate_up_proj", + "amd/MiniMax-M2.5-MXFP4/o_proj", + "amd/MiniMax-M3-MXFP4/gate_up_proj", + "nvidia/GLM-5.1-NVFP4/gate_up_proj", + "nvidia/GLM-5.2-NVFP4/gate_up_proj", + "nvidia/MiniMax-M2.5-NVFP4/o_proj" + ] }, { "type": "gemm", "args": { - "m": 857088, - "n": 7168, - "k": 16384, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 16, + "n": 3072, + "k": 6144, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "MiniMaxAI/MiniMax-M2.5/o_proj", + "zai-org/GLM-5-FP8/gate_up_proj", + "zai-org/GLM-5.1-FP8/gate_up_proj", + "zai-org/GLM-5.2-FP8/gate_up_proj" + ] }, { "type": "gemm", "args": { - "m": 857088, - "n": 2304, - "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 16, + "n": 3584, + "k": 512, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "zai-org/GLM-5-FP8/kv_b_proj", + "zai-org/GLM-5.1-FP8/kv_b_proj", + "zai-org/GLM-5.2-FP8/kv_b_proj" + ] }, { "type": "gemm", "args": { - "m": 857088, - "n": 7168, - "k": 2304, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 16, + "n": 4096, + "k": 128, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "Qwen/Qwen3.5-397B-A17B/shared_expert.down_proj", + "amd/Qwen3.5-397B-A17B-MXFP4/shared_expert.down_proj", + "nvidia/Qwen3.5-397B-A17B-NVFP4/shared_expert.down_proj" + ] }, { "type": "gemm", "args": { - "m": 104, - "n": 1536, - "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 16, + "n": 4096, + "k": 128, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "Qwen/Qwen3.5-397B-A17B-FP8/shared_expert.down_proj" + ] }, { "type": "gemm", "args": { - "m": 104, - "n": 24576, - "k": 1536, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 16, + "n": 4096, + "k": 256, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "Qwen/Qwen3.5-397B-A17B/shared_expert.down_proj", + "amd/Qwen3.5-397B-A17B-MXFP4/shared_expert.down_proj", + "nvidia/Qwen3.5-397B-A17B-NVFP4/shared_expert.down_proj" + ] }, { "type": "gemm", "args": { - "m": 104, - "n": 576, - "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 16, + "n": 4096, + "k": 256, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "Qwen/Qwen3.5-397B-A17B-FP8/shared_expert.down_proj" + ] }, { "type": "gemm", "args": { - "m": 104, - "n": 32768, + "m": 16, + "n": 4096, "k": 512, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "Qwen/Qwen3.5-397B-A17B/o_proj", + "Qwen/Qwen3.5-397B-A17B/shared_expert.down_proj", + "amd/Kimi-K2.5-MXFP4/kv_b_proj", + "amd/Qwen3.5-397B-A17B-MXFP4/o_proj", + "amd/Qwen3.5-397B-A17B-MXFP4/shared_expert.down_proj", + "moonshotai/Kimi-K2.5/kv_b_proj", + "nvidia/DeepSeek-R1-0528-FP4-V2/kv_b_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/kv_b_proj", + "nvidia/Kimi-K2.5-NVFP4/kv_b_proj", + "nvidia/Kimi-K2.6-NVFP4/kv_b_proj", + "nvidia/Qwen3.5-397B-A17B-NVFP4/o_proj", + "nvidia/Qwen3.5-397B-A17B-NVFP4/shared_expert.down_proj" + ] }, { "type": "gemm", "args": { - "m": 104, - "n": 7168, - "k": 16384, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 16, + "n": 4096, + "k": 512, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 32 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 1, + 32 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "amd/DeepSeek-R1-0528-MXFP4-Preview/kv_b_proj", + "amd/DeepSeek-R1-0528-MXFP4-v2/kv_b_proj", + "amd/DeepSeek-R1-0528-MXFP4/kv_b_proj", + "amd/Qwen3.5-397B-A17B-MXFP4-AttnFP8-V2/shared_expert.down_proj" + ] }, { "type": "gemm", "args": { - "m": 104, - "n": 2304, - "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 16, + "n": 4096, + "k": 512, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "Qwen/Qwen3.5-397B-A17B-FP8/o_proj", + "Qwen/Qwen3.5-397B-A17B-FP8/shared_expert.down_proj", + "deepseek-ai/DeepSeek-R1-0528/kv_b_proj" + ] }, { "type": "gemm", "args": { - "m": 104, - "n": 7168, - "k": 2304, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 16, + "n": 4096, + "k": 1024, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "Qwen/Qwen3.5-397B-A17B/o_proj", + "Qwen/Qwen3.5-397B-A17B/shared_expert.down_proj", + "amd/Qwen3.5-397B-A17B-MXFP4/o_proj", + "amd/Qwen3.5-397B-A17B-MXFP4/shared_expert.down_proj", + "nvidia/Qwen3.5-397B-A17B-NVFP4/o_proj", + "nvidia/Qwen3.5-397B-A17B-NVFP4/shared_expert.down_proj" + ] }, { "type": "gemm", "args": { - "m": 4550656, - "n": 1536, - "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 16, + "n": 4096, + "k": 1024, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "Qwen/Qwen3.5-397B-A17B-FP8/o_proj", + "Qwen/Qwen3.5-397B-A17B-FP8/shared_expert.down_proj" + ] }, { "type": "gemm", "args": { - "m": 4550656, - "n": 24576, - "k": 1536, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 16, + "n": 4096, + "k": 2048, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "Qwen/Qwen3.5-397B-A17B/o_proj", + "amd/GLM-5.1-MXFP4/q_b_proj", + "amd/GLM-5.2-MXFP4/q_b_proj", + "amd/Qwen3.5-397B-A17B-MXFP4/o_proj", + "nvidia/GLM-5-NVFP4/q_b_proj", + "nvidia/GLM-5.1-NVFP4/q_b_proj", + "nvidia/GLM-5.2-NVFP4/q_b_proj", + "nvidia/Qwen3.5-397B-A17B-NVFP4/o_proj" + ] }, { "type": "gemm", "args": { - "m": 4550656, - "n": 576, - "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 16, + "n": 4096, + "k": 2048, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "Qwen/Qwen3.5-397B-A17B-FP8/o_proj", + "zai-org/GLM-5-FP8/q_b_proj", + "zai-org/GLM-5.1-FP8/q_b_proj", + "zai-org/GLM-5.2-FP8/q_b_proj" + ] }, { "type": "gemm", "args": { - "m": 4550656, - "n": 32768, - "k": 512, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 16, + "n": 4096, + "k": 2880, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/gpt-oss-120b-w-mxfp4-a-fp8/q_proj", + "openai/gpt-oss-120b/q_proj" + ] }, { "type": "gemm", "args": { - "m": 4550656, - "n": 7168, - "k": 16384, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 16, + "n": 4096, + "k": 4096, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "Qwen/Qwen3.5-397B-A17B/o_proj", + "Qwen/Qwen3.5-397B-A17B/q_proj", + "amd/Qwen3.5-397B-A17B-MXFP4/o_proj", + "amd/Qwen3.5-397B-A17B-MXFP4/q_proj", + "nvidia/Qwen3.5-397B-A17B-NVFP4/o_proj", + "nvidia/Qwen3.5-397B-A17B-NVFP4/q_proj" + ] }, { "type": "gemm", "args": { - "m": 4550656, - "n": 4608, - "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 16, + "n": 4096, + "k": 4096, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "Qwen/Qwen3.5-397B-A17B-FP8/o_proj", + "Qwen/Qwen3.5-397B-A17B-FP8/q_proj" + ] }, { "type": "gemm", "args": { - "m": 4550656, - "n": 7168, - "k": 4608, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 16, + "n": 4608, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "moonshotai/Kimi-K2.5/gate_up_proj", + "nvidia/Kimi-K2.6-NVFP4/gate_up_proj" + ] }, { "type": "gemm", "args": { - "m": 555, - "n": 1536, + "m": 16, + "n": 4608, "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/gate_up_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/gate_up_proj", + "nvidia/Kimi-K2.5-NVFP4/gate_up_proj" + ] }, { "type": "gemm", "args": { - "m": 555, - "n": 24576, - "k": 1536, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 16, + "n": 4608, + "k": 7168, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 32 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 1, + 32 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "amd/DeepSeek-R1-0528-MXFP4-Preview/gate_up_proj", + "amd/DeepSeek-R1-0528-MXFP4-v2/gate_up_proj", + "amd/DeepSeek-R1-0528-MXFP4/gate_up_proj", + "amd/Kimi-K2.5-MXFP4/gate_up_proj" + ] }, { "type": "gemm", "args": { - "m": 555, - "n": 576, + "m": 16, + "n": 4608, "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/gate_up_proj" + ] }, { "type": "gemm", "args": { - "m": 555, - "n": 32768, - "k": 512, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 16, + "n": 6144, + "k": 1536, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/GLM-5.1-MXFP4/down_proj", + "amd/GLM-5.2-MXFP4/down_proj", + "amd/Kimi-K2.5-MXFP4/q_b_proj", + "amd/MiniMax-M3-MXFP4/down_proj", + "moonshotai/Kimi-K2.5/q_b_proj", + "nvidia/DeepSeek-R1-0528-FP4-V2/q_b_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/q_b_proj", + "nvidia/GLM-5.1-NVFP4/down_proj", + "nvidia/GLM-5.2-NVFP4/down_proj", + "nvidia/Kimi-K2.5-NVFP4/q_b_proj", + "nvidia/Kimi-K2.6-NVFP4/q_b_proj" + ] }, { "type": "gemm", "args": { - "m": 555, - "n": 7168, - "k": 16384, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 16, + "n": 6144, + "k": 1536, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 32 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 1, + 32 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "amd/DeepSeek-R1-0528-MXFP4-Preview/q_b_proj", + "amd/DeepSeek-R1-0528-MXFP4-v2/q_b_proj", + "amd/DeepSeek-R1-0528-MXFP4/q_b_proj" + ] }, { "type": "gemm", "args": { - "m": 555, - "n": 4608, - "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 16, + "n": 6144, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/q_b_proj", + "zai-org/GLM-5-FP8/down_proj", + "zai-org/GLM-5.1-FP8/down_proj", + "zai-org/GLM-5.2-FP8/down_proj" + ] }, { "type": "gemm", "args": { - "m": 555, - "n": 7168, - "k": 4608, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 16, + "n": 6144, + "k": 2048, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "zai-org/GLM-5-FP8/o_proj", + "zai-org/GLM-5.1-FP8/o_proj", + "zai-org/GLM-5.2-FP8/o_proj" + ] }, { "type": "gemm", "args": { - "m": 2275328, - "n": 1536, - "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 16, + "n": 6144, + "k": 3072, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/GLM-5.1-MXFP4/down_proj", + "amd/GLM-5.2-MXFP4/down_proj", + "amd/MiniMax-M2.5-MXFP4/q_proj", + "amd/MiniMax-M3-MXFP4/down_proj", + "nvidia/GLM-5.1-NVFP4/down_proj", + "nvidia/GLM-5.2-NVFP4/down_proj", + "nvidia/MiniMax-M2.5-NVFP4/q_proj" + ] }, { "type": "gemm", "args": { - "m": 2275328, - "n": 24576, - "k": 1536, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 16, + "n": 6144, + "k": 3072, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/GLM-5-NVFP4/down_proj" + ] }, { "type": "gemm", "args": { - "m": 2275328, - "n": 576, - "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 16, + "n": 6144, + "k": 3072, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "MiniMaxAI/MiniMax-M2.5/q_proj", + "zai-org/GLM-5-FP8/down_proj", + "zai-org/GLM-5.1-FP8/down_proj", + "zai-org/GLM-5.2-FP8/down_proj" + ] }, { "type": "gemm", "args": { - "m": 2275328, - "n": 32768, - "k": 512, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 16, + "n": 6144, + "k": 4096, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/GLM-5.1-MXFP4/o_proj", + "amd/GLM-5.2-MXFP4/o_proj", + "amd/MiniMax-M3-MXFP4/o_proj", + "nvidia/GLM-5-NVFP4/o_proj", + "nvidia/GLM-5.1-NVFP4/o_proj", + "nvidia/GLM-5.2-NVFP4/o_proj" + ] }, { "type": "gemm", "args": { - "m": 2275328, - "n": 7168, - "k": 16384, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 16, + "n": 6144, + "k": 4096, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "zai-org/GLM-5-FP8/o_proj", + "zai-org/GLM-5.1-FP8/o_proj", + "zai-org/GLM-5.2-FP8/o_proj" + ] }, { "type": "gemm", "args": { - "m": 2275328, - "n": 2304, - "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 16, + "n": 6144, + "k": 6144, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/GLM-5.1-MXFP4/down_proj", + "amd/GLM-5.1-MXFP4/gate_up_proj", + "amd/GLM-5.2-MXFP4/down_proj", + "amd/GLM-5.2-MXFP4/gate_up_proj", + "amd/MiniMax-M3-MXFP4/gate_up_proj", + "nvidia/GLM-5.1-NVFP4/down_proj", + "nvidia/GLM-5.1-NVFP4/gate_up_proj", + "nvidia/GLM-5.2-NVFP4/down_proj", + "nvidia/GLM-5.2-NVFP4/gate_up_proj" + ] }, { "type": "gemm", "args": { - "m": 2275328, - "n": 7168, - "k": 2304, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 16, + "n": 6144, + "k": 6144, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/GLM-5-NVFP4/down_proj", + "nvidia/GLM-5-NVFP4/gate_up_proj" + ] }, { "type": "gemm", "args": { - "m": 277, - "n": 1536, - "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 16, + "n": 6144, + "k": 6144, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "zai-org/GLM-5-FP8/down_proj", + "zai-org/GLM-5-FP8/gate_up_proj", + "zai-org/GLM-5.1-FP8/down_proj", + "zai-org/GLM-5.1-FP8/gate_up_proj", + "zai-org/GLM-5.2-FP8/down_proj", + "zai-org/GLM-5.2-FP8/gate_up_proj" + ] }, { "type": "gemm", "args": { - "m": 277, - "n": 24576, - "k": 1536, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 16, + "n": 6144, + "k": 8192, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/GLM-5.1-MXFP4/o_proj", + "amd/GLM-5.2-MXFP4/o_proj", + "amd/MiniMax-M3-MXFP4/o_proj", + "nvidia/GLM-5-NVFP4/o_proj", + "nvidia/GLM-5.1-NVFP4/o_proj", + "nvidia/GLM-5.2-NVFP4/o_proj" + ] }, { "type": "gemm", "args": { - "m": 277, - "n": 576, - "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 16, + "n": 7168, + "k": 512, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/GLM-5.1-MXFP4/kv_b_proj", + "amd/GLM-5.2-MXFP4/kv_b_proj", + "amd/Kimi-K2.5-MXFP4/o_proj", + "moonshotai/Kimi-K2.5/o_proj", + "nvidia/GLM-5-NVFP4/kv_b_proj", + "nvidia/GLM-5.1-NVFP4/kv_b_proj", + "nvidia/GLM-5.2-NVFP4/kv_b_proj", + "nvidia/Kimi-K2.5-NVFP4/o_proj", + "nvidia/Kimi-K2.6-NVFP4/o_proj" + ] }, { "type": "gemm", "args": { - "m": 277, - "n": 32768, + "m": 16, + "n": 7168, "k": 512, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "zai-org/GLM-5-FP8/kv_b_proj", + "zai-org/GLM-5.1-FP8/kv_b_proj", + "zai-org/GLM-5.2-FP8/kv_b_proj" + ] }, { "type": "gemm", "args": { - "m": 277, + "m": 16, "n": 7168, - "k": 16384, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "k": 576, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-NVFP4-v2/down_proj" + ] }, { "type": "gemm", "args": { - "m": 277, - "n": 2304, - "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 16, + "n": 7168, + "k": 576, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/down_proj" + ] }, { "type": "gemm", "args": { - "m": 277, + "m": 16, "n": 7168, - "k": 2304, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "k": 1024, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/Kimi-K2.5-MXFP4/o_proj", + "moonshotai/Kimi-K2.5/o_proj", + "nvidia/Kimi-K2.5-NVFP4/o_proj", + "nvidia/Kimi-K2.6-NVFP4/o_proj" + ] }, { "type": "gemm", "args": { - "m": 2048, - "n": 1536, - "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 16, + "n": 7168, + "k": 1152, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/down_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/down_proj", + "nvidia/Kimi-K2.5-NVFP4/down_proj" + ] }, { "type": "gemm", "args": { - "m": 2048, - "n": 24576, - "k": 1536, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 16, + "n": 7168, + "k": 2048, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/Kimi-K2.5-MXFP4/o_proj", + "moonshotai/Kimi-K2.5/o_proj", + "nvidia/Kimi-K2.5-NVFP4/o_proj", + "nvidia/Kimi-K2.6-NVFP4/o_proj" + ] }, { "type": "gemm", "args": { - "m": 2048, - "n": 576, - "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 16, + "n": 7168, + "k": 2048, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/o_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/o_proj" + ] }, { "type": "gemm", "args": { - "m": 2048, - "n": 32768, - "k": 512, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 16, + "n": 7168, + "k": 2048, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 32 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 1, + 32 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "amd/DeepSeek-R1-0528-MXFP4-Preview/o_proj", + "amd/DeepSeek-R1-0528-MXFP4-v2/o_proj", + "amd/DeepSeek-R1-0528-MXFP4/o_proj" + ] }, { "type": "gemm", "args": { - "m": 2048, + "m": 16, "n": 7168, - "k": 16384, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "k": 2048, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/o_proj", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] }, { "type": "gemm", "args": { - "m": 2048, - "n": 4608, - "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 16, + "n": 7168, + "k": 2304, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "moonshotai/Kimi-K2.5/down_proj", + "nvidia/Kimi-K2.6-NVFP4/down_proj" + ] }, { "type": "gemm", "args": { - "m": 2048, + "m": 16, "n": 7168, - "k": 4608, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "k": 2304, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/down_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/down_proj", + "nvidia/Kimi-K2.5-NVFP4/down_proj" + ] }, { "type": "gemm", "args": { - "m": 0, - "n": 1536, - "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 16, + "n": 7168, + "k": 2304, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 32 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 1, + 32 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "amd/DeepSeek-R1-0528-MXFP4-Preview/down_proj", + "amd/DeepSeek-R1-0528-MXFP4-v2/down_proj", + "amd/DeepSeek-R1-0528-MXFP4/down_proj", + "amd/Kimi-K2.5-MXFP4/down_proj" + ] }, { "type": "gemm", "args": { - "m": 0, - "n": 24576, - "k": 1536, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 16, + "n": 7168, + "k": 2304, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/down_proj" + ] }, { "type": "gemm", "args": { - "m": 0, - "n": 576, - "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 16, + "n": 7168, + "k": 4096, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/o_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/o_proj" + ] }, { "type": "gemm", "args": { - "m": 0, - "n": 32768, - "k": 512, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 16, + "n": 7168, + "k": 4096, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 32 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 1, + 32 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "amd/DeepSeek-R1-0528-MXFP4-Preview/o_proj", + "amd/DeepSeek-R1-0528-MXFP4-v2/o_proj", + "amd/DeepSeek-R1-0528-MXFP4/o_proj" + ] }, { "type": "gemm", "args": { - "m": 0, + "m": 16, "n": 7168, - "k": 16384, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "k": 4096, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/o_proj", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] }, { "type": "gemm", "args": { - "m": 0, - "n": 4608, - "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 16, + "n": 7168, + "k": 4608, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "moonshotai/Kimi-K2.5/down_proj", + "nvidia/Kimi-K2.6-NVFP4/down_proj" + ] }, { "type": "gemm", "args": { - "m": 0, + "m": 16, "n": 7168, "k": 4608, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/down_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/down_proj", + "nvidia/Kimi-K2.5-NVFP4/down_proj" + ] }, { "type": "gemm", "args": { - "m": 8192, - "n": 1536, - "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 16, + "n": 7168, + "k": 4608, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 32 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 1, + 32 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "amd/DeepSeek-R1-0528-MXFP4-Preview/down_proj", + "amd/DeepSeek-R1-0528-MXFP4-v2/down_proj", + "amd/DeepSeek-R1-0528-MXFP4/down_proj", + "amd/Kimi-K2.5-MXFP4/down_proj" + ] }, { "type": "gemm", "args": { - "m": 8192, - "n": 3072, - "k": 1536, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 16, + "n": 7168, + "k": 4608, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/down_proj" + ] }, { "type": "gemm", "args": { - "m": 8192, - "n": 576, - "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 16, + "n": 7168, + "k": 8192, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/Kimi-K2.5-MXFP4/o_proj", + "moonshotai/Kimi-K2.5/o_proj", + "nvidia/Kimi-K2.5-NVFP4/o_proj", + "nvidia/Kimi-K2.6-NVFP4/o_proj" + ] }, { "type": "gemm", "args": { - "m": 8192, - "n": 4096, - "k": 512, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 16, + "n": 7168, + "k": 8192, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] }, { "type": "gemm", "args": { - "m": 8192, + "m": 16, "n": 7168, - "k": 2048, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "k": 16384, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/o_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/o_proj" + ] }, { "type": "gemm", "args": { - "m": 8192, - "n": 2304, - "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 16, + "n": 7168, + "k": 16384, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/o_proj", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] }, { "type": "gemm", "args": { - "m": 8192, + "m": 16, "n": 7168, - "k": 2304, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "k": 16384, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] }, { "type": "gemm", "args": { - "m": 1, - "n": 3072, - "k": 1536, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 16, + "n": 7168, + "k": 65536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] }, { "type": "gemm", "args": { - "m": 1, - "n": 4096, + "m": 16, + "n": 8192, "k": 512, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/Kimi-K2.5-MXFP4/kv_b_proj", + "moonshotai/Kimi-K2.5/kv_b_proj", + "nvidia/DeepSeek-R1-0528-FP4-V2/kv_b_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/kv_b_proj", + "nvidia/Kimi-K2.5-NVFP4/kv_b_proj", + "nvidia/Kimi-K2.6-NVFP4/kv_b_proj" + ] }, { "type": "gemm", "args": { - "m": 1, - "n": 7168, - "k": 2048, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 16, + "n": 8192, + "k": 512, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 32 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 1, + 32 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "amd/DeepSeek-R1-0528-MXFP4-Preview/kv_b_proj", + "amd/DeepSeek-R1-0528-MXFP4-v2/kv_b_proj", + "amd/DeepSeek-R1-0528-MXFP4/kv_b_proj" + ] }, { "type": "gemm", "args": { - "m": 1, - "n": 2304, - "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 16, + "n": 8192, + "k": 512, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/kv_b_proj" + ] }, { "type": "gemm", "args": { - "m": 1, - "n": 7168, - "k": 2304, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 16, + "n": 8192, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_b" + ] }, { "type": "gemm", "args": { - "m": 4096, - "n": 1536, - "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 16, + "n": 8192, + "k": 2048, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/GLM-5.1-MXFP4/q_b_proj", + "amd/GLM-5.2-MXFP4/q_b_proj", + "nvidia/GLM-5-NVFP4/q_b_proj", + "nvidia/GLM-5.1-NVFP4/q_b_proj", + "nvidia/GLM-5.2-NVFP4/q_b_proj" + ] }, { "type": "gemm", "args": { - "m": 4096, - "n": 24576, + "m": 16, + "n": 12288, "k": 1536, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 4096, - "n": 576, - "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/Kimi-K2.5-MXFP4/q_b_proj", + "moonshotai/Kimi-K2.5/q_b_proj", + "nvidia/DeepSeek-R1-0528-FP4-V2/q_b_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/q_b_proj", + "nvidia/Kimi-K2.5-NVFP4/q_b_proj", + "nvidia/Kimi-K2.6-NVFP4/q_b_proj" + ] }, { "type": "gemm", "args": { - "m": 4096, - "n": 32768, + "m": 16, + "n": 14336, "k": 512, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/GLM-5.1-MXFP4/kv_b_proj", + "amd/GLM-5.2-MXFP4/kv_b_proj", + "nvidia/GLM-5-NVFP4/kv_b_proj", + "nvidia/GLM-5.1-NVFP4/kv_b_proj", + "nvidia/GLM-5.2-NVFP4/kv_b_proj" + ] }, { "type": "gemm", "args": { - "m": 4096, - "n": 7168, - "k": 16384, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 16, + "n": 15360, + "k": 1024, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] }, { "type": "gemm", "args": { - "m": 4096, - "n": 4608, - "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 16, + "n": 16384, + "k": 512, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/Kimi-K2.5-MXFP4/kv_b_proj", + "moonshotai/Kimi-K2.5/kv_b_proj", + "nvidia/DeepSeek-R1-0528-FP4-V2/kv_b_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/kv_b_proj", + "nvidia/Kimi-K2.5-NVFP4/kv_b_proj", + "nvidia/Kimi-K2.6-NVFP4/kv_b_proj" + ] }, { "type": "gemm", "args": { - "m": 4096, - "n": 7168, - "k": 4608, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 16, + "n": 16384, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_b" + ] }, { "type": "gemm", "args": { - "m": 16384, - "n": 1536, - "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 16, + "n": 24576, + "k": 1536, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/q_b_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/q_b_proj" + ] }, { "type": "gemm", "args": { - "m": 16384, - "n": 3072, + "m": 16, + "n": 24576, "k": 1536, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/q_b_proj" + ] }, { "type": "gemm", "args": { - "m": 16384, - "n": 576, - "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 16, + "n": 30720, + "k": 1024, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] }, { "type": "gemm", "args": { - "m": 16384, - "n": 4096, + "m": 16, + "n": 32768, "k": 512, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/kv_b_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/kv_b_proj" + ] }, { "type": "gemm", "args": { - "m": 16384, - "n": 7168, - "k": 2048, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 16, + "n": 32768, + "k": 512, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/kv_b_proj" + ] }, { "type": "gemm", "args": { - "m": 16384, - "n": 2304, - "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 16, + "n": 65536, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_b" + ] }, { "type": "gemm", "args": { - "m": 16384, - "n": 7168, - "k": 2304, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 16, + "n": 122880, + "k": 1024, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] }, { "type": "gemm", "args": { - "m": 2, - "n": 3072, - "k": 1536, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 18, + "n": 576, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/Kimi-K2.5-MXFP4/kv_a_proj_with_mqa", + "moonshotai/Kimi-K2.5/kv_a_proj_with_mqa", + "nvidia/DeepSeek-R1-0528-FP4-V2/kv_a_proj_with_mqa", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/kv_a_proj_with_mqa", + "nvidia/Kimi-K2.5-NVFP4/kv_a_proj_with_mqa", + "nvidia/Kimi-K2.6-NVFP4/kv_a_proj_with_mqa" + ] }, { "type": "gemm", "args": { - "m": 2, - "n": 4096, - "k": 512, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 18, + "n": 576, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/kv_a_proj_with_mqa" + ] }, { "type": "gemm", "args": { - "m": 2, - "n": 7168, - "k": 2048, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 18, + "n": 1088, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] }, { "type": "gemm", "args": { - "m": 2, - "n": 2304, + "m": 18, + "n": 1152, "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/gate_up_proj" + ] }, { "type": "gemm", "args": { - "m": 2, - "n": 7168, - "k": 2304, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 18, + "n": 1536, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/q_b_proj" + ] }, { "type": "gemm", "args": { - "m": 8192, - "n": 24576, - "k": 1536, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 18, + "n": 1536, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/Kimi-K2.5-MXFP4/q_a_proj", + "moonshotai/Kimi-K2.5/q_a_proj", + "nvidia/DeepSeek-R1-0528-FP4-V2/q_a_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/q_a_proj", + "nvidia/Kimi-K2.5-NVFP4/q_a_proj", + "nvidia/Kimi-K2.6-NVFP4/q_a_proj" + ] }, { "type": "gemm", "args": { - "m": 8192, - "n": 32768, - "k": 512, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 18, + "n": 1536, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/gate_up_proj", + "deepseek-ai/DeepSeek-R1-0528/q_a_proj", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w3" + ] }, { "type": "gemm", "args": { - "m": 8192, - "n": 7168, - "k": 16384, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 18, + "n": 2048, + "k": 512, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/kv_b_proj" + ] }, { "type": "gemm", "args": { - "m": 8192, - "n": 4608, + "m": 18, + "n": 2304, "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/gate_up_proj" + ] }, { "type": "gemm", "args": { - "m": 8192, - "n": 7168, - "k": 4608, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 18, + "n": 3072, + "k": 1536, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/Kimi-K2.5-MXFP4/q_b_proj", + "amd/MiniMax-M2.5-MXFP4/o_proj", + "moonshotai/Kimi-K2.5/q_b_proj", + "nvidia/DeepSeek-R1-0528-FP4-V2/q_b_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/q_b_proj", + "nvidia/Kimi-K2.5-NVFP4/q_b_proj", + "nvidia/Kimi-K2.6-NVFP4/q_b_proj", + "nvidia/MiniMax-M2.5-NVFP4/o_proj" + ] }, { "type": "gemm", "args": { - "m": 32768, - "n": 1536, - "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 18, + "n": 4096, + "k": 512, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "Qwen/Qwen3.5-397B-A17B/o_proj", + "Qwen/Qwen3.5-397B-A17B/shared_expert.down_proj", + "amd/Kimi-K2.5-MXFP4/kv_b_proj", + "amd/Qwen3.5-397B-A17B-MXFP4/o_proj", + "amd/Qwen3.5-397B-A17B-MXFP4/shared_expert.down_proj", + "moonshotai/Kimi-K2.5/kv_b_proj", + "nvidia/DeepSeek-R1-0528-FP4-V2/kv_b_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/kv_b_proj", + "nvidia/Kimi-K2.5-NVFP4/kv_b_proj", + "nvidia/Kimi-K2.6-NVFP4/kv_b_proj", + "nvidia/Qwen3.5-397B-A17B-NVFP4/o_proj", + "nvidia/Qwen3.5-397B-A17B-NVFP4/shared_expert.down_proj" + ] }, { "type": "gemm", "args": { - "m": 32768, - "n": 3072, - "k": 1536, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 18, + "n": 7168, + "k": 1024, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/o_proj", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] }, { "type": "gemm", "args": { - "m": 32768, - "n": 576, - "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 18, + "n": 7168, + "k": 1152, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/down_proj" + ] }, { "type": "gemm", "args": { - "m": 32768, - "n": 4096, - "k": 512, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 18, + "n": 7168, + "k": 2048, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/o_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/o_proj" + ] }, { "type": "gemm", "args": { - "m": 32768, + "m": 18, "n": 7168, - "k": 2048, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "k": 2304, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/down_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/down_proj", + "nvidia/Kimi-K2.5-NVFP4/down_proj" + ] }, { "type": "gemm", "args": { - "m": 32768, - "n": 2304, - "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 18, + "n": 7168, + "k": 2304, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/down_proj" + ] }, { "type": "gemm", "args": { - "m": 32768, + "m": 18, "n": 7168, - "k": 2304, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "k": 16384, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/o_proj", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] }, { "type": "gemm", "args": { - "m": 4, - "n": 1536, - "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 18, + "n": 16384, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_b" + ] }, { "type": "gemm", "args": { - "m": 4, - "n": 3072, + "m": 18, + "n": 24576, "k": 1536, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/q_b_proj" + ] }, { "type": "gemm", "args": { - "m": 4, - "n": 576, - "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 18, + "n": 30720, + "k": 1024, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] }, { "type": "gemm", "args": { - "m": 4, - "n": 4096, + "m": 18, + "n": 32768, "k": 512, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/kv_b_proj" + ] }, { "type": "gemm", "args": { - "m": 4, - "n": 7168, - "k": 2048, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 19, + "n": 576, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/Kimi-K2.5-MXFP4/kv_a_proj_with_mqa", + "moonshotai/Kimi-K2.5/kv_a_proj_with_mqa", + "nvidia/DeepSeek-R1-0528-FP4-V2/kv_a_proj_with_mqa", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/kv_a_proj_with_mqa", + "nvidia/Kimi-K2.5-NVFP4/kv_a_proj_with_mqa", + "nvidia/Kimi-K2.6-NVFP4/kv_a_proj_with_mqa" + ] }, { "type": "gemm", "args": { - "m": 4, - "n": 2304, + "m": 19, + "n": 1152, "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-NVFP4-v2/gate_up_proj" + ] }, { "type": "gemm", "args": { - "m": 4, - "n": 7168, - "k": 2304, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 19, + "n": 1536, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/Kimi-K2.5-MXFP4/q_a_proj", + "moonshotai/Kimi-K2.5/q_a_proj", + "nvidia/DeepSeek-R1-0528-FP4-V2/q_a_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/q_a_proj", + "nvidia/Kimi-K2.5-NVFP4/q_a_proj", + "nvidia/Kimi-K2.6-NVFP4/q_a_proj" + ] }, { "type": "gemm", "args": { - "m": 16384, - "n": 24576, - "k": 1536, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 19, + "n": 4608, + "k": 7168, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/gate_up_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/gate_up_proj", + "nvidia/Kimi-K2.5-NVFP4/gate_up_proj" + ] }, { "type": "gemm", "args": { - "m": 16384, - "n": 32768, - "k": 512, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 19, + "n": 7168, + "k": 576, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-NVFP4-v2/down_proj" + ] }, { "type": "gemm", "args": { - "m": 16384, + "m": 19, "n": 7168, - "k": 16384, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "k": 1152, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/down_proj" + ] }, { "type": "gemm", "args": { - "m": 16384, - "n": 4608, - "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 19, + "n": 7168, + "k": 4608, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/down_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/down_proj", + "nvidia/Kimi-K2.5-NVFP4/down_proj" + ] }, { "type": "gemm", "args": { - "m": 16384, + "m": 19, "n": 7168, "k": 4608, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 32 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 1, + 32 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "amd/DeepSeek-R1-0528-MXFP4-Preview/down_proj", + "amd/DeepSeek-R1-0528-MXFP4-v2/down_proj", + "amd/DeepSeek-R1-0528-MXFP4/down_proj", + "amd/Kimi-K2.5-MXFP4/down_proj" + ] }, { "type": "gemm", "args": { - "m": 65536, - "n": 1536, - "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 19, + "n": 7168, + "k": 16384, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/o_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/o_proj" + ] }, { "type": "gemm", "args": { - "m": 65536, - "n": 3072, + "m": 19, + "n": 24576, "k": 1536, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/q_b_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/q_b_proj" + ] }, { "type": "gemm", "args": { - "m": 65536, + "m": 19, + "n": 32768, + "k": 512, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/kv_b_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/kv_b_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 20, "n": 576, "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/Kimi-K2.5-MXFP4/kv_a_proj_with_mqa", + "moonshotai/Kimi-K2.5/kv_a_proj_with_mqa", + "nvidia/DeepSeek-R1-0528-FP4-V2/kv_a_proj_with_mqa", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/kv_a_proj_with_mqa", + "nvidia/Kimi-K2.5-NVFP4/kv_a_proj_with_mqa", + "nvidia/Kimi-K2.6-NVFP4/kv_a_proj_with_mqa" + ] }, { "type": "gemm", "args": { - "m": 65536, - "n": 4096, - "k": 512, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 20, + "n": 576, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/kv_a_proj_with_mqa" + ] }, { "type": "gemm", "args": { - "m": 65536, - "n": 7168, - "k": 2048, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 20, + "n": 1152, + "k": 7168, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-NVFP4-v2/gate_up_proj" + ] }, { "type": "gemm", "args": { - "m": 65536, - "n": 2304, + "m": 20, + "n": 1152, "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/gate_up_proj" + ] }, { "type": "gemm", "args": { - "m": 65536, - "n": 7168, - "k": 2304, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 20, + "n": 1536, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/Kimi-K2.5-MXFP4/q_a_proj", + "moonshotai/Kimi-K2.5/q_a_proj", + "nvidia/DeepSeek-R1-0528-FP4-V2/q_a_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/q_a_proj", + "nvidia/Kimi-K2.5-NVFP4/q_a_proj", + "nvidia/Kimi-K2.6-NVFP4/q_a_proj" + ] }, { "type": "gemm", "args": { - "m": 8, + "m": 20, "n": 1536, "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/gate_up_proj", + "deepseek-ai/DeepSeek-R1-0528/q_a_proj", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w3" + ] }, { "type": "gemm", "args": { - "m": 8, - "n": 3072, - "k": 1536, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 20, + "n": 2304, + "k": 7168, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/gate_up_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/gate_up_proj", + "nvidia/Kimi-K2.5-NVFP4/gate_up_proj" + ] }, { "type": "gemm", "args": { - "m": 8, - "n": 576, + "m": 20, + "n": 2304, "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/gate_up_proj" + ] }, { "type": "gemm", "args": { - "m": 8, + "m": 20, + "n": 3072, + "k": 1536, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/Kimi-K2.5-MXFP4/q_b_proj", + "amd/MiniMax-M2.5-MXFP4/o_proj", + "moonshotai/Kimi-K2.5/q_b_proj", + "nvidia/DeepSeek-R1-0528-FP4-V2/q_b_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/q_b_proj", + "nvidia/Kimi-K2.5-NVFP4/q_b_proj", + "nvidia/Kimi-K2.6-NVFP4/q_b_proj", + "nvidia/MiniMax-M2.5-NVFP4/o_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 20, "n": 4096, "k": 512, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "Qwen/Qwen3.5-397B-A17B/o_proj", + "Qwen/Qwen3.5-397B-A17B/shared_expert.down_proj", + "amd/Kimi-K2.5-MXFP4/kv_b_proj", + "amd/Qwen3.5-397B-A17B-MXFP4/o_proj", + "amd/Qwen3.5-397B-A17B-MXFP4/shared_expert.down_proj", + "moonshotai/Kimi-K2.5/kv_b_proj", + "nvidia/DeepSeek-R1-0528-FP4-V2/kv_b_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/kv_b_proj", + "nvidia/Kimi-K2.5-NVFP4/kv_b_proj", + "nvidia/Kimi-K2.6-NVFP4/kv_b_proj", + "nvidia/Qwen3.5-397B-A17B-NVFP4/o_proj", + "nvidia/Qwen3.5-397B-A17B-NVFP4/shared_expert.down_proj" + ] }, { "type": "gemm", "args": { - "m": 8, - "n": 7168, - "k": 2048, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 20, + "n": 4608, + "k": 7168, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/gate_up_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/gate_up_proj", + "nvidia/Kimi-K2.5-NVFP4/gate_up_proj" + ] }, { "type": "gemm", "args": { - "m": 8, - "n": 2304, + "m": 20, + "n": 4608, "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/gate_up_proj" + ] }, { "type": "gemm", "args": { - "m": 8, + "m": 20, "n": 7168, - "k": 2304, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "k": 576, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-NVFP4-v2/down_proj" + ] }, { "type": "gemm", "args": { - "m": 32768, - "n": 24576, - "k": 1536, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 20, + "n": 7168, + "k": 576, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/down_proj" + ] }, { "type": "gemm", "args": { - "m": 32768, - "n": 32768, - "k": 512, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 20, + "n": 7168, + "k": 1152, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/down_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/down_proj", + "nvidia/Kimi-K2.5-NVFP4/down_proj" + ] }, { "type": "gemm", "args": { - "m": 32768, + "m": 20, "n": 7168, - "k": 16384, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "k": 1152, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/down_proj" + ] }, { "type": "gemm", "args": { - "m": 32768, - "n": 4608, - "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 20, + "n": 7168, + "k": 2048, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/o_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/o_proj" + ] }, { "type": "gemm", "args": { - "m": 32768, + "m": 20, + "n": 7168, + "k": 2304, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/down_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/down_proj", + "nvidia/Kimi-K2.5-NVFP4/down_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 20, + "n": 7168, + "k": 2304, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/down_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 20, "n": 7168, "k": 4608, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/down_proj" + ] }, { "type": "gemm", "args": { - "m": 4, - "n": 24576, - "k": 1536, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 20, + "n": 7168, + "k": 8192, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/Kimi-K2.5-MXFP4/o_proj", + "moonshotai/Kimi-K2.5/o_proj", + "nvidia/Kimi-K2.5-NVFP4/o_proj", + "nvidia/Kimi-K2.6-NVFP4/o_proj" + ] }, { "type": "gemm", "args": { - "m": 4, - "n": 32768, - "k": 512, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 20, + "n": 7168, + "k": 9216, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/down_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/down_proj", + "nvidia/Kimi-K2.5-NVFP4/down_proj" + ] }, { "type": "gemm", "args": { - "m": 4, + "m": 20, "n": 7168, "k": 16384, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/o_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/o_proj" + ] }, { "type": "gemm", "args": { - "m": 4, - "n": 4608, - "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 20, + "n": 7168, + "k": 16384, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/o_proj", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] }, { "type": "gemm", "args": { - "m": 4, - "n": 7168, - "k": 4608, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 20, + "n": 9216, + "k": 7168, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/gate_up_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/gate_up_proj", + "nvidia/Kimi-K2.5-NVFP4/gate_up_proj" + ] }, { "type": "gemm", "args": { - "m": 131072, - "n": 1536, + "m": 20, + "n": 9216, "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 32 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 1, + 32 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "amd/DeepSeek-R1-0528-MXFP4-Preview/gate_up_proj", + "amd/DeepSeek-R1-0528-MXFP4-v2/gate_up_proj", + "amd/DeepSeek-R1-0528-MXFP4/gate_up_proj", + "amd/Kimi-K2.5-MXFP4/gate_up_proj" + ] }, { "type": "gemm", "args": { - "m": 131072, - "n": 3072, + "m": 20, + "n": 12288, "k": 1536, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/Kimi-K2.5-MXFP4/q_b_proj", + "moonshotai/Kimi-K2.5/q_b_proj", + "nvidia/DeepSeek-R1-0528-FP4-V2/q_b_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/q_b_proj", + "nvidia/Kimi-K2.5-NVFP4/q_b_proj", + "nvidia/Kimi-K2.6-NVFP4/q_b_proj" + ] }, { "type": "gemm", "args": { - "m": 131072, - "n": 576, - "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 20, + "n": 16384, + "k": 512, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/Kimi-K2.5-MXFP4/kv_b_proj", + "moonshotai/Kimi-K2.5/kv_b_proj", + "nvidia/DeepSeek-R1-0528-FP4-V2/kv_b_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/kv_b_proj", + "nvidia/Kimi-K2.5-NVFP4/kv_b_proj", + "nvidia/Kimi-K2.6-NVFP4/kv_b_proj" + ] }, { "type": "gemm", "args": { - "m": 131072, - "n": 4096, - "k": 512, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 20, + "n": 24576, + "k": 1536, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/q_b_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/q_b_proj" + ] }, { "type": "gemm", "args": { - "m": 131072, - "n": 7168, - "k": 2048, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 20, + "n": 24576, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/q_b_proj" + ] }, { "type": "gemm", "args": { - "m": 131072, - "n": 2304, - "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 20, + "n": 32768, + "k": 512, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/kv_b_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/kv_b_proj" + ] }, { "type": "gemm", "args": { - "m": 131072, - "n": 7168, - "k": 2304, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 20, + "n": 32768, + "k": 512, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/kv_b_proj" + ] }, { "type": "gemm", "args": { - "m": 16, - "n": 1536, + "m": 21, + "n": 384, "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w3" + ] }, { "type": "gemm", "args": { - "m": 16, - "n": 3072, - "k": 1536, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 21, + "n": 576, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/Kimi-K2.5-MXFP4/kv_a_proj_with_mqa", + "moonshotai/Kimi-K2.5/kv_a_proj_with_mqa", + "nvidia/DeepSeek-R1-0528-FP4-V2/kv_a_proj_with_mqa", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/kv_a_proj_with_mqa", + "nvidia/Kimi-K2.5-NVFP4/kv_a_proj_with_mqa", + "nvidia/Kimi-K2.6-NVFP4/kv_a_proj_with_mqa" + ] }, { "type": "gemm", "args": { - "m": 16, + "m": 21, "n": 576, "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/kv_a_proj_with_mqa" + ] }, { "type": "gemm", "args": { - "m": 16, - "n": 4096, - "k": 512, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 21, + "n": 1536, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/Kimi-K2.5-MXFP4/q_a_proj", + "moonshotai/Kimi-K2.5/q_a_proj", + "nvidia/DeepSeek-R1-0528-FP4-V2/q_a_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/q_a_proj", + "nvidia/Kimi-K2.5-NVFP4/q_a_proj", + "nvidia/Kimi-K2.6-NVFP4/q_a_proj" + ] }, { "type": "gemm", "args": { - "m": 16, - "n": 7168, - "k": 2048, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 21, + "n": 1536, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/gate_up_proj", + "deepseek-ai/DeepSeek-R1-0528/q_a_proj", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w3" + ] }, { "type": "gemm", "args": { - "m": 16, - "n": 2304, + "m": 21, + "n": 1536, "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w3" + ] }, { "type": "gemm", "args": { - "m": 16, - "n": 7168, - "k": 2304, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 21, + "n": 4608, + "k": 7168, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/gate_up_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/gate_up_proj", + "nvidia/Kimi-K2.5-NVFP4/gate_up_proj" + ] }, { "type": "gemm", "args": { - "m": 65536, - "n": 24576, - "k": 1536, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 21, + "n": 4608, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/gate_up_proj" + ] }, { "type": "gemm", "args": { - "m": 65536, - "n": 32768, - "k": 512, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 21, + "n": 7168, + "k": 384, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w2" + ] }, { "type": "gemm", "args": { - "m": 65536, + "m": 21, "n": 7168, - "k": 16384, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "k": 4608, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/down_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/down_proj", + "nvidia/Kimi-K2.5-NVFP4/down_proj" + ] }, { "type": "gemm", "args": { - "m": 65536, - "n": 4608, - "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 21, + "n": 7168, + "k": 16384, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/o_proj", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] }, { "type": "gemm", "args": { - "m": 65536, + "m": 21, "n": 7168, - "k": 4608, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "k": 16384, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] }, { "type": "gemm", "args": { - "m": 8, + "m": 21, "n": 24576, "k": 1536, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/q_b_proj" + ] }, { "type": "gemm", "args": { - "m": 8, + "m": 21, "n": 32768, "k": 512, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/kv_b_proj" + ] }, { "type": "gemm", "args": { - "m": 8, - "n": 7168, - "k": 16384, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 22, + "n": 576, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/Kimi-K2.5-MXFP4/kv_a_proj_with_mqa", + "moonshotai/Kimi-K2.5/kv_a_proj_with_mqa", + "nvidia/DeepSeek-R1-0528-FP4-V2/kv_a_proj_with_mqa", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/kv_a_proj_with_mqa", + "nvidia/Kimi-K2.5-NVFP4/kv_a_proj_with_mqa", + "nvidia/Kimi-K2.6-NVFP4/kv_a_proj_with_mqa" + ] }, { "type": "gemm", "args": { - "m": 8, - "n": 4608, + "m": 22, + "n": 1536, "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/Kimi-K2.5-MXFP4/q_a_proj", + "moonshotai/Kimi-K2.5/q_a_proj", + "nvidia/DeepSeek-R1-0528-FP4-V2/q_a_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/q_a_proj", + "nvidia/Kimi-K2.5-NVFP4/q_a_proj", + "nvidia/Kimi-K2.6-NVFP4/q_a_proj" + ] }, { "type": "gemm", "args": { - "m": 8, - "n": 7168, - "k": 4608, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 22, + "n": 2304, + "k": 7168, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/gate_up_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/gate_up_proj", + "nvidia/Kimi-K2.5-NVFP4/gate_up_proj" + ] }, { "type": "gemm", "args": { - "m": 262144, - "n": 1536, + "m": 22, + "n": 2304, "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/gate_up_proj" + ] }, { "type": "gemm", "args": { - "m": 262144, + "m": 22, "n": 3072, "k": 1536, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/Kimi-K2.5-MXFP4/q_b_proj", + "amd/MiniMax-M2.5-MXFP4/o_proj", + "moonshotai/Kimi-K2.5/q_b_proj", + "nvidia/DeepSeek-R1-0528-FP4-V2/q_b_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/q_b_proj", + "nvidia/Kimi-K2.5-NVFP4/q_b_proj", + "nvidia/Kimi-K2.6-NVFP4/q_b_proj", + "nvidia/MiniMax-M2.5-NVFP4/o_proj" + ] }, { "type": "gemm", "args": { - "m": 262144, - "n": 576, - "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 22, + "n": 4096, + "k": 512, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "Qwen/Qwen3.5-397B-A17B/o_proj", + "Qwen/Qwen3.5-397B-A17B/shared_expert.down_proj", + "amd/Kimi-K2.5-MXFP4/kv_b_proj", + "amd/Qwen3.5-397B-A17B-MXFP4/o_proj", + "amd/Qwen3.5-397B-A17B-MXFP4/shared_expert.down_proj", + "moonshotai/Kimi-K2.5/kv_b_proj", + "nvidia/DeepSeek-R1-0528-FP4-V2/kv_b_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/kv_b_proj", + "nvidia/Kimi-K2.5-NVFP4/kv_b_proj", + "nvidia/Kimi-K2.6-NVFP4/kv_b_proj", + "nvidia/Qwen3.5-397B-A17B-NVFP4/o_proj", + "nvidia/Qwen3.5-397B-A17B-NVFP4/shared_expert.down_proj" + ] }, { "type": "gemm", "args": { - "m": 262144, - "n": 4096, - "k": 512, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 22, + "n": 4608, + "k": 7168, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/gate_up_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/gate_up_proj", + "nvidia/Kimi-K2.5-NVFP4/gate_up_proj" + ] }, { "type": "gemm", "args": { - "m": 262144, + "m": 22, "n": 7168, "k": 2048, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/o_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/o_proj" + ] }, { "type": "gemm", "args": { - "m": 262144, - "n": 2304, - "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 22, + "n": 7168, + "k": 2304, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/down_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/down_proj", + "nvidia/Kimi-K2.5-NVFP4/down_proj" + ] }, { "type": "gemm", "args": { - "m": 262144, + "m": 22, "n": 7168, - "k": 2304, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "k": 4608, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/down_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/down_proj", + "nvidia/Kimi-K2.5-NVFP4/down_proj" + ] }, { "type": "gemm", "args": { - "m": 32, - "n": 1536, - "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 22, + "n": 7168, + "k": 4608, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 32 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 1, + 32 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "amd/DeepSeek-R1-0528-MXFP4-Preview/down_proj", + "amd/DeepSeek-R1-0528-MXFP4-v2/down_proj", + "amd/DeepSeek-R1-0528-MXFP4/down_proj", + "amd/Kimi-K2.5-MXFP4/down_proj" + ] }, { "type": "gemm", "args": { - "m": 32, - "n": 3072, - "k": 1536, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 22, + "n": 7168, + "k": 16384, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/o_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/o_proj" + ] }, { "type": "gemm", "args": { - "m": 32, - "n": 576, - "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 22, + "n": 24576, + "k": 1536, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/q_b_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/q_b_proj" + ] }, { "type": "gemm", "args": { - "m": 32, - "n": 4096, + "m": 22, + "n": 32768, "k": 512, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/kv_b_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/kv_b_proj" + ] }, { "type": "gemm", "args": { - "m": 32, - "n": 7168, - "k": 2048, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 24, + "n": 576, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/Kimi-K2.5-MXFP4/kv_a_proj_with_mqa", + "moonshotai/Kimi-K2.5/kv_a_proj_with_mqa", + "nvidia/DeepSeek-R1-0528-FP4-V2/kv_a_proj_with_mqa", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/kv_a_proj_with_mqa", + "nvidia/Kimi-K2.5-NVFP4/kv_a_proj_with_mqa", + "nvidia/Kimi-K2.6-NVFP4/kv_a_proj_with_mqa" + ] }, { "type": "gemm", "args": { - "m": 32, - "n": 2304, + "m": 24, + "n": 576, "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/kv_a_proj_with_mqa" + ] }, { "type": "gemm", "args": { - "m": 32, - "n": 7168, - "k": 2304, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 24, + "n": 1536, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/Kimi-K2.5-MXFP4/q_a_proj", + "moonshotai/Kimi-K2.5/q_a_proj", + "nvidia/DeepSeek-R1-0528-FP4-V2/q_a_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/q_a_proj", + "nvidia/Kimi-K2.5-NVFP4/q_a_proj", + "nvidia/Kimi-K2.6-NVFP4/q_a_proj" + ] }, { "type": "gemm", "args": { - "m": 1024, + "m": 24, "n": 1536, "k": 7168, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/gate_up_proj", + "deepseek-ai/DeepSeek-R1-0528/q_a_proj", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w3" + ] }, { "type": "gemm", "args": { - "m": 1024, - "n": 24576, - "k": 1536, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 24, + "n": 2304, + "k": 7168, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/gate_up_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/gate_up_proj", + "nvidia/Kimi-K2.5-NVFP4/gate_up_proj" + ] }, { "type": "gemm", "args": { - "m": 1024, - "n": 576, + "m": 24, + "n": 2304, "k": 7168, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/gate_up_proj" + ] }, { "type": "gemm", "args": { - "m": 1024, - "n": 32768, - "k": 512, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 24, + "n": 3072, + "k": 1536, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/Kimi-K2.5-MXFP4/q_b_proj", + "amd/MiniMax-M2.5-MXFP4/o_proj", + "moonshotai/Kimi-K2.5/q_b_proj", + "nvidia/DeepSeek-R1-0528-FP4-V2/q_b_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/q_b_proj", + "nvidia/Kimi-K2.5-NVFP4/q_b_proj", + "nvidia/Kimi-K2.6-NVFP4/q_b_proj", + "nvidia/MiniMax-M2.5-NVFP4/o_proj" + ] }, { "type": "gemm", "args": { - "m": 1024, - "n": 7168, - "k": 16384, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 24, + "n": 3072, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "MiniMaxAI/MiniMax-M2.5/experts.down_proj", + "MiniMaxAI/MiniMax-M2.5/o_proj", + "deepseek-ai/DeepSeek-R1-0528/q_b_proj" + ] }, { "type": "gemm", "args": { - "m": 1024, - "n": 2304, - "k": 7168, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 24, + "n": 4096, + "k": 512, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "Qwen/Qwen3.5-397B-A17B/o_proj", + "Qwen/Qwen3.5-397B-A17B/shared_expert.down_proj", + "amd/Kimi-K2.5-MXFP4/kv_b_proj", + "amd/Qwen3.5-397B-A17B-MXFP4/o_proj", + "amd/Qwen3.5-397B-A17B-MXFP4/shared_expert.down_proj", + "moonshotai/Kimi-K2.5/kv_b_proj", + "nvidia/DeepSeek-R1-0528-FP4-V2/kv_b_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/kv_b_proj", + "nvidia/Kimi-K2.5-NVFP4/kv_b_proj", + "nvidia/Kimi-K2.6-NVFP4/kv_b_proj", + "nvidia/Qwen3.5-397B-A17B-NVFP4/o_proj", + "nvidia/Qwen3.5-397B-A17B-NVFP4/shared_expert.down_proj" + ] }, { "type": "gemm", "args": { - "m": 1024, - "n": 7168, - "k": 2304, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 24, + "n": 4096, + "k": 512, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "Qwen/Qwen3.5-397B-A17B-FP8/o_proj", + "Qwen/Qwen3.5-397B-A17B-FP8/shared_expert.down_proj", + "deepseek-ai/DeepSeek-R1-0528/kv_b_proj" + ] }, { "type": "gemm", "args": { - "m": 2, - "n": 1536, + "m": 24, + "n": 4608, "k": 7168, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/gate_up_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/gate_up_proj", + "nvidia/Kimi-K2.5-NVFP4/gate_up_proj" + ] }, { "type": "gemm", "args": { - "m": 2, - "n": 24576, - "k": 1536, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 24, + "n": 4608, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/gate_up_proj" + ] }, { "type": "gemm", "args": { - "m": 2, - "n": 576, - "k": 7168, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 24, + "n": 6144, + "k": 1536, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/GLM-5.1-MXFP4/down_proj", + "amd/GLM-5.2-MXFP4/down_proj", + "amd/Kimi-K2.5-MXFP4/q_b_proj", + "amd/MiniMax-M3-MXFP4/down_proj", + "moonshotai/Kimi-K2.5/q_b_proj", + "nvidia/DeepSeek-R1-0528-FP4-V2/q_b_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/q_b_proj", + "nvidia/GLM-5.1-NVFP4/down_proj", + "nvidia/GLM-5.2-NVFP4/down_proj", + "nvidia/Kimi-K2.5-NVFP4/q_b_proj", + "nvidia/Kimi-K2.6-NVFP4/q_b_proj" + ] }, { "type": "gemm", "args": { - "m": 2, - "n": 32768, - "k": 512, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 24, + "n": 7168, + "k": 2048, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/o_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/o_proj" + ] }, { "type": "gemm", "args": { - "m": 2, + "m": 24, "n": 7168, - "k": 16384, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "k": 2048, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/o_proj", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] }, { "type": "gemm", "args": { - "m": 2, - "n": 2304, - "k": 7168, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 24, + "n": 7168, + "k": 2304, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/down_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/down_proj", + "nvidia/Kimi-K2.5-NVFP4/down_proj" + ] }, { "type": "gemm", "args": { - "m": 2, + "m": 24, "n": 7168, "k": 2304, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/down_proj" + ] }, { "type": "gemm", "args": { - "m": 8192, - "n": 1536, - "k": 7168, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 24, + "n": 7168, + "k": 4096, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/o_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/o_proj" + ] }, { "type": "gemm", "args": { - "m": 8192, - "n": 3072, - "k": 1536, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 24, + "n": 7168, + "k": 4608, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/down_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/down_proj", + "nvidia/Kimi-K2.5-NVFP4/down_proj" + ] }, { "type": "gemm", "args": { - "m": 8192, - "n": 576, - "k": 7168, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 24, + "n": 7168, + "k": 9216, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/down_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/down_proj", + "nvidia/Kimi-K2.5-NVFP4/down_proj" + ] }, { "type": "gemm", "args": { - "m": 8192, - "n": 4096, - "k": 512, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 24, + "n": 7168, + "k": 16384, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/o_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/o_proj" + ] }, { "type": "gemm", "args": { - "m": 8192, + "m": 24, "n": 7168, - "k": 2048, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "k": 16384, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 32 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 1, + 32 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "amd/DeepSeek-R1-0528-MXFP4-Preview/o_proj", + "amd/DeepSeek-R1-0528-MXFP4-v2/o_proj", + "amd/DeepSeek-R1-0528-MXFP4/o_proj" + ] }, { "type": "gemm", "args": { - "m": 8192, - "n": 2304, - "k": 7168, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 24, + "n": 7168, + "k": 16384, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] }, { "type": "gemm", "args": { - "m": 8192, - "n": 7168, - "k": 2304, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 24, + "n": 8192, + "k": 512, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/Kimi-K2.5-MXFP4/kv_b_proj", + "moonshotai/Kimi-K2.5/kv_b_proj", + "nvidia/DeepSeek-R1-0528-FP4-V2/kv_b_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/kv_b_proj", + "nvidia/Kimi-K2.5-NVFP4/kv_b_proj", + "nvidia/Kimi-K2.6-NVFP4/kv_b_proj" + ] }, { "type": "gemm", "args": { - "m": 16, - "n": 1536, + "m": 24, + "n": 9216, "k": 7168, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/gate_up_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/gate_up_proj", + "nvidia/Kimi-K2.5-NVFP4/gate_up_proj" + ] }, { "type": "gemm", "args": { - "m": 16, - "n": 3072, + "m": 24, + "n": 24576, "k": 1536, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/q_b_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/q_b_proj" + ] }, { "type": "gemm", "args": { - "m": 16, - "n": 576, - "k": 7168, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 24, + "n": 24576, + "k": 1536, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 32 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 1, + 32 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "amd/DeepSeek-R1-0528-MXFP4-Preview/q_b_proj", + "amd/DeepSeek-R1-0528-MXFP4-v2/q_b_proj", + "amd/DeepSeek-R1-0528-MXFP4/q_b_proj" + ] }, { "type": "gemm", "args": { - "m": 16, - "n": 4096, + "m": 24, + "n": 32768, "k": 512, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/kv_b_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/kv_b_proj" + ] }, { "type": "gemm", "args": { - "m": 16, - "n": 7168, - "k": 2048, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 24, + "n": 32768, + "k": 512, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 32 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 1, + 32 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "amd/DeepSeek-R1-0528-MXFP4-Preview/kv_b_proj", + "amd/DeepSeek-R1-0528-MXFP4-v2/kv_b_proj", + "amd/DeepSeek-R1-0528-MXFP4/kv_b_proj" + ] }, { "type": "gemm", "args": { - "m": 16, - "n": 2304, + "m": 25, + "n": 576, "k": 7168, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/Kimi-K2.5-MXFP4/kv_a_proj_with_mqa", + "moonshotai/Kimi-K2.5/kv_a_proj_with_mqa", + "nvidia/DeepSeek-R1-0528-FP4-V2/kv_a_proj_with_mqa", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/kv_a_proj_with_mqa", + "nvidia/Kimi-K2.5-NVFP4/kv_a_proj_with_mqa", + "nvidia/Kimi-K2.6-NVFP4/kv_a_proj_with_mqa" + ] }, { "type": "gemm", "args": { - "m": 16, - "n": 7168, - "k": 2304, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 25, + "n": 1536, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/Kimi-K2.5-MXFP4/q_a_proj", + "moonshotai/Kimi-K2.5/q_a_proj", + "nvidia/DeepSeek-R1-0528-FP4-V2/q_a_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/q_a_proj", + "nvidia/Kimi-K2.5-NVFP4/q_a_proj", + "nvidia/Kimi-K2.6-NVFP4/q_a_proj" + ] }, { "type": "gemm", "args": { - "m": 4096, - "n": 1536, + "m": 25, + "n": 2304, "k": 7168, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/gate_up_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/gate_up_proj", + "nvidia/Kimi-K2.5-NVFP4/gate_up_proj" + ] }, { "type": "gemm", "args": { - "m": 4096, - "n": 24576, + "m": 25, + "n": 3072, "k": 1536, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/Kimi-K2.5-MXFP4/q_b_proj", + "amd/MiniMax-M2.5-MXFP4/o_proj", + "moonshotai/Kimi-K2.5/q_b_proj", + "nvidia/DeepSeek-R1-0528-FP4-V2/q_b_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/q_b_proj", + "nvidia/Kimi-K2.5-NVFP4/q_b_proj", + "nvidia/Kimi-K2.6-NVFP4/q_b_proj", + "nvidia/MiniMax-M2.5-NVFP4/o_proj" + ] }, { "type": "gemm", "args": { - "m": 4096, - "n": 576, - "k": 7168, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 25, + "n": 4096, + "k": 512, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "Qwen/Qwen3.5-397B-A17B/o_proj", + "Qwen/Qwen3.5-397B-A17B/shared_expert.down_proj", + "amd/Kimi-K2.5-MXFP4/kv_b_proj", + "amd/Qwen3.5-397B-A17B-MXFP4/o_proj", + "amd/Qwen3.5-397B-A17B-MXFP4/shared_expert.down_proj", + "moonshotai/Kimi-K2.5/kv_b_proj", + "nvidia/DeepSeek-R1-0528-FP4-V2/kv_b_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/kv_b_proj", + "nvidia/Kimi-K2.5-NVFP4/kv_b_proj", + "nvidia/Kimi-K2.6-NVFP4/kv_b_proj", + "nvidia/Qwen3.5-397B-A17B-NVFP4/o_proj", + "nvidia/Qwen3.5-397B-A17B-NVFP4/shared_expert.down_proj" + ] }, { "type": "gemm", "args": { - "m": 4096, - "n": 32768, - "k": 512, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 25, + "n": 4608, + "k": 7168, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/gate_up_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/gate_up_proj", + "nvidia/Kimi-K2.5-NVFP4/gate_up_proj" + ] }, { "type": "gemm", "args": { - "m": 4096, - "n": 7168, - "k": 16384, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 25, + "n": 6144, + "k": 1536, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/GLM-5.1-MXFP4/down_proj", + "amd/GLM-5.2-MXFP4/down_proj", + "amd/Kimi-K2.5-MXFP4/q_b_proj", + "amd/MiniMax-M3-MXFP4/down_proj", + "moonshotai/Kimi-K2.5/q_b_proj", + "nvidia/DeepSeek-R1-0528-FP4-V2/q_b_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/q_b_proj", + "nvidia/GLM-5.1-NVFP4/down_proj", + "nvidia/GLM-5.2-NVFP4/down_proj", + "nvidia/Kimi-K2.5-NVFP4/q_b_proj", + "nvidia/Kimi-K2.6-NVFP4/q_b_proj" + ] }, { "type": "gemm", "args": { - "m": 4096, - "n": 2304, - "k": 7168, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 25, + "n": 7168, + "k": 2048, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/o_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/o_proj" + ] }, { "type": "gemm", "args": { - "m": 4096, + "m": 25, "n": 7168, "k": 2304, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/down_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/down_proj", + "nvidia/Kimi-K2.5-NVFP4/down_proj" + ] }, { "type": "gemm", "args": { - "m": 8, - "n": 1536, - "k": 7168, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 25, + "n": 7168, + "k": 4096, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/o_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/o_proj" + ] }, { "type": "gemm", "args": { - "m": 8, - "n": 24576, - "k": 1536, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 25, + "n": 7168, + "k": 4608, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/down_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/down_proj", + "nvidia/Kimi-K2.5-NVFP4/down_proj" + ] }, { "type": "gemm", "args": { - "m": 8, - "n": 576, - "k": 7168, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 25, + "n": 7168, + "k": 9216, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/down_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/down_proj", + "nvidia/Kimi-K2.5-NVFP4/down_proj" + ] }, { "type": "gemm", "args": { - "m": 8, - "n": 32768, - "k": 512, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, + "m": 25, + "n": 7168, + "k": 16384, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/o_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/o_proj" + ] + }, { "type": "gemm", "args": { - "m": 8, + "m": 25, "n": 7168, "k": 16384, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 32 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 1, + 32 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "amd/DeepSeek-R1-0528-MXFP4-Preview/o_proj", + "amd/DeepSeek-R1-0528-MXFP4-v2/o_proj", + "amd/DeepSeek-R1-0528-MXFP4/o_proj" + ] }, { "type": "gemm", "args": { - "m": 8, - "n": 2304, - "k": 7168, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 25, + "n": 7168, + "k": 16384, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] }, { "type": "gemm", "args": { - "m": 8, - "n": 7168, - "k": 2304, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 25, + "n": 8192, + "k": 512, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/Kimi-K2.5-MXFP4/kv_b_proj", + "moonshotai/Kimi-K2.5/kv_b_proj", + "nvidia/DeepSeek-R1-0528-FP4-V2/kv_b_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/kv_b_proj", + "nvidia/Kimi-K2.5-NVFP4/kv_b_proj", + "nvidia/Kimi-K2.6-NVFP4/kv_b_proj" + ] }, { "type": "gemm", "args": { - "m": 32768, - "n": 1536, + "m": 25, + "n": 9216, "k": 7168, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/gate_up_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/gate_up_proj", + "nvidia/Kimi-K2.5-NVFP4/gate_up_proj" + ] }, { "type": "gemm", "args": { - "m": 32768, - "n": 3072, + "m": 25, + "n": 24576, "k": 1536, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/q_b_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/q_b_proj" + ] }, { "type": "gemm", "args": { - "m": 32768, - "n": 576, - "k": 7168, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 25, + "n": 24576, + "k": 1536, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 32 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 1, + 32 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "amd/DeepSeek-R1-0528-MXFP4-Preview/q_b_proj", + "amd/DeepSeek-R1-0528-MXFP4-v2/q_b_proj", + "amd/DeepSeek-R1-0528-MXFP4/q_b_proj" + ] }, { "type": "gemm", "args": { - "m": 32768, - "n": 4096, + "m": 25, + "n": 32768, "k": 512, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/kv_b_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/kv_b_proj" + ] }, { "type": "gemm", "args": { - "m": 32768, - "n": 7168, - "k": 2048, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 25, + "n": 32768, + "k": 512, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 32 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 1, + 32 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "amd/DeepSeek-R1-0528-MXFP4-Preview/kv_b_proj", + "amd/DeepSeek-R1-0528-MXFP4-v2/kv_b_proj", + "amd/DeepSeek-R1-0528-MXFP4/kv_b_proj" + ] }, { "type": "gemm", "args": { - "m": 32768, - "n": 2304, + "m": 26, + "n": 576, "k": 7168, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 32768, - "n": 7168, - "k": 2304, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/Kimi-K2.5-MXFP4/kv_a_proj_with_mqa", + "moonshotai/Kimi-K2.5/kv_a_proj_with_mqa", + "nvidia/DeepSeek-R1-0528-FP4-V2/kv_a_proj_with_mqa", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/kv_a_proj_with_mqa", + "nvidia/Kimi-K2.5-NVFP4/kv_a_proj_with_mqa", + "nvidia/Kimi-K2.6-NVFP4/kv_a_proj_with_mqa" + ] }, { "type": "gemm", "args": { - "m": 64, + "m": 26, "n": 1536, "k": 7168, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 64, - "n": 3072, - "k": 1536, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/Kimi-K2.5-MXFP4/q_a_proj", + "moonshotai/Kimi-K2.5/q_a_proj", + "nvidia/DeepSeek-R1-0528-FP4-V2/q_a_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/q_a_proj", + "nvidia/Kimi-K2.5-NVFP4/q_a_proj", + "nvidia/Kimi-K2.6-NVFP4/q_a_proj" + ] }, { "type": "gemm", "args": { - "m": 64, - "n": 576, + "m": 26, + "n": 4608, "k": 7168, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 64, - "n": 4096, - "k": 512, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 64, - "n": 7168, - "k": 2048, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/gate_up_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/gate_up_proj", + "nvidia/Kimi-K2.5-NVFP4/gate_up_proj" + ] }, { "type": "gemm", "args": { - "m": 64, - "n": 2304, + "m": 26, + "n": 4608, "k": 7168, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 32 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 1, + 32 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "amd/DeepSeek-R1-0528-MXFP4-Preview/gate_up_proj", + "amd/DeepSeek-R1-0528-MXFP4-v2/gate_up_proj", + "amd/DeepSeek-R1-0528-MXFP4/gate_up_proj", + "amd/Kimi-K2.5-MXFP4/gate_up_proj" + ] }, { "type": "gemm", "args": { - "m": 64, + "m": 26, "n": 7168, - "k": 2304, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 8192, - "n": 24576, - "k": 1536, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "k": 4608, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/down_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/down_proj", + "nvidia/Kimi-K2.5-NVFP4/down_proj" + ] }, { "type": "gemm", "args": { - "m": 8192, - "n": 32768, - "k": 512, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 26, + "n": 7168, + "k": 8192, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/Kimi-K2.5-MXFP4/o_proj", + "moonshotai/Kimi-K2.5/o_proj", + "nvidia/Kimi-K2.5-NVFP4/o_proj", + "nvidia/Kimi-K2.6-NVFP4/o_proj" + ] }, { "type": "gemm", "args": { - "m": 8192, + "m": 26, "n": 7168, - "k": 16384, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "k": 8192, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] }, { "type": "gemm", "args": { - "m": 16, - "n": 24576, + "m": 26, + "n": 12288, "k": 1536, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/Kimi-K2.5-MXFP4/q_b_proj", + "moonshotai/Kimi-K2.5/q_b_proj", + "nvidia/DeepSeek-R1-0528-FP4-V2/q_b_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/q_b_proj", + "nvidia/Kimi-K2.5-NVFP4/q_b_proj", + "nvidia/Kimi-K2.6-NVFP4/q_b_proj" + ] }, { "type": "gemm", "args": { - "m": 16, - "n": 32768, + "m": 26, + "n": 16384, "k": 512, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/Kimi-K2.5-MXFP4/kv_b_proj", + "moonshotai/Kimi-K2.5/kv_b_proj", + "nvidia/DeepSeek-R1-0528-FP4-V2/kv_b_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/kv_b_proj", + "nvidia/Kimi-K2.5-NVFP4/kv_b_proj", + "nvidia/Kimi-K2.6-NVFP4/kv_b_proj" + ] }, { "type": "gemm", "args": { - "m": 16, - "n": 7168, - "k": 16384, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 27, + "n": 576, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/kv_a_proj_with_mqa" + ] }, { "type": "gemm", "args": { - "m": 65536, + "m": 27, "n": 1536, "k": 7168, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/gate_up_proj", + "deepseek-ai/DeepSeek-R1-0528/q_a_proj", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w3" + ] }, { "type": "gemm", "args": { - "m": 65536, - "n": 3072, - "k": 1536, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 27, + "n": 2304, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/gate_up_proj" + ] }, { "type": "gemm", "args": { - "m": 65536, - "n": 576, - "k": 7168, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 27, + "n": 3072, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "MiniMaxAI/MiniMax-M2.5/experts.down_proj", + "MiniMaxAI/MiniMax-M2.5/o_proj", + "deepseek-ai/DeepSeek-R1-0528/q_b_proj" + ] }, { "type": "gemm", "args": { - "m": 65536, + "m": 27, "n": 4096, "k": 512, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "Qwen/Qwen3.5-397B-A17B-FP8/o_proj", + "Qwen/Qwen3.5-397B-A17B-FP8/shared_expert.down_proj", + "deepseek-ai/DeepSeek-R1-0528/kv_b_proj" + ] }, { "type": "gemm", "args": { - "m": 65536, + "m": 27, "n": 7168, "k": 2048, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 65536, - "n": 2304, - "k": 7168, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/o_proj", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] }, { "type": "gemm", "args": { - "m": 65536, + "m": 27, "n": 7168, "k": 2304, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/down_proj" + ] }, { "type": "gemm", "args": { - "m": 128, - "n": 1536, + "m": 28, + "n": 576, "k": 7168, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/kv_a_proj_with_mqa" + ] }, { "type": "gemm", "args": { - "m": 128, - "n": 3072, - "k": 1536, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 28, + "n": 1152, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/gate_up_proj" + ] }, { "type": "gemm", "args": { - "m": 128, - "n": 576, + "m": 28, + "n": 1536, "k": 7168, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/gate_up_proj", + "deepseek-ai/DeepSeek-R1-0528/q_a_proj", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w3" + ] }, { "type": "gemm", "args": { - "m": 128, - "n": 4096, - "k": 512, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 28, + "n": 1536, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w3" + ] }, { "type": "gemm", "args": { - "m": 128, + "m": 28, "n": 7168, - "k": 2048, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 128, - "n": 2304, - "k": 7168, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "k": 1152, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/down_proj" + ] }, { "type": "gemm", "args": { - "m": 128, + "m": 28, "n": 7168, - "k": 2304, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "k": 16384, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/o_proj", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] }, { "type": "gemm", "args": { - "m": 32768, + "m": 28, "n": 24576, "k": 1536, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/q_b_proj" + ] }, { "type": "gemm", "args": { - "m": 32768, + "m": 28, "n": 32768, "k": 512, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/kv_b_proj" + ] }, { "type": "gemm", "args": { - "m": 32768, - "n": 7168, - "k": 16384, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 30, + "n": 576, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/Kimi-K2.5-MXFP4/kv_a_proj_with_mqa", + "moonshotai/Kimi-K2.5/kv_a_proj_with_mqa", + "nvidia/DeepSeek-R1-0528-FP4-V2/kv_a_proj_with_mqa", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/kv_a_proj_with_mqa", + "nvidia/Kimi-K2.5-NVFP4/kv_a_proj_with_mqa", + "nvidia/Kimi-K2.6-NVFP4/kv_a_proj_with_mqa" + ] }, { "type": "gemm", "args": { - "m": 64, - "n": 24576, - "k": 1536, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 30, + "n": 576, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/kv_a_proj_with_mqa" + ] }, { "type": "gemm", "args": { - "m": 64, - "n": 32768, - "k": 512, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 30, + "n": 1152, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/gate_up_proj" + ] }, { "type": "gemm", "args": { - "m": 64, - "n": 7168, - "k": 16384, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 30, + "n": 1536, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/q_b_proj" + ] }, { "type": "gemm", "args": { - "m": 262144, + "m": 30, "n": 1536, "k": 7168, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 262144, - "n": 3072, - "k": 1536, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/Kimi-K2.5-MXFP4/q_a_proj", + "moonshotai/Kimi-K2.5/q_a_proj", + "nvidia/DeepSeek-R1-0528-FP4-V2/q_a_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/q_a_proj", + "nvidia/Kimi-K2.5-NVFP4/q_a_proj", + "nvidia/Kimi-K2.6-NVFP4/q_a_proj" + ] }, { "type": "gemm", "args": { - "m": 262144, - "n": 576, + "m": 30, + "n": 1536, "k": 7168, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/gate_up_proj", + "deepseek-ai/DeepSeek-R1-0528/q_a_proj", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w3" + ] }, { "type": "gemm", "args": { - "m": 262144, - "n": 4096, + "m": 30, + "n": 2048, "k": 512, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 262144, - "n": 7168, - "k": 2048, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/kv_b_proj" + ] }, { "type": "gemm", "args": { - "m": 262144, + "m": 30, "n": 2304, "k": 7168, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 262144, - "n": 7168, - "k": 2304, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/gate_up_proj" + ] }, { "type": "gemm", "args": { - "m": 512, - "n": 1536, - "k": 7168, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 30, + "n": 3072, + "k": 1536, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/Kimi-K2.5-MXFP4/q_b_proj", + "amd/MiniMax-M2.5-MXFP4/o_proj", + "moonshotai/Kimi-K2.5/q_b_proj", + "nvidia/DeepSeek-R1-0528-FP4-V2/q_b_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/q_b_proj", + "nvidia/Kimi-K2.5-NVFP4/q_b_proj", + "nvidia/Kimi-K2.6-NVFP4/q_b_proj", + "nvidia/MiniMax-M2.5-NVFP4/o_proj" + ] }, { "type": "gemm", "args": { - "m": 512, + "m": 30, "n": 3072, "k": 1536, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "MiniMaxAI/MiniMax-M2.5/experts.down_proj", + "MiniMaxAI/MiniMax-M2.5/o_proj", + "deepseek-ai/DeepSeek-R1-0528/q_b_proj" + ] }, { "type": "gemm", "args": { - "m": 512, - "n": 576, - "k": 7168, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 30, + "n": 4096, + "k": 512, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "Qwen/Qwen3.5-397B-A17B/o_proj", + "Qwen/Qwen3.5-397B-A17B/shared_expert.down_proj", + "amd/Kimi-K2.5-MXFP4/kv_b_proj", + "amd/Qwen3.5-397B-A17B-MXFP4/o_proj", + "amd/Qwen3.5-397B-A17B-MXFP4/shared_expert.down_proj", + "moonshotai/Kimi-K2.5/kv_b_proj", + "nvidia/DeepSeek-R1-0528-FP4-V2/kv_b_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/kv_b_proj", + "nvidia/Kimi-K2.5-NVFP4/kv_b_proj", + "nvidia/Kimi-K2.6-NVFP4/kv_b_proj", + "nvidia/Qwen3.5-397B-A17B-NVFP4/o_proj", + "nvidia/Qwen3.5-397B-A17B-NVFP4/shared_expert.down_proj" + ] }, { "type": "gemm", "args": { - "m": 512, + "m": 30, "n": 4096, "k": 512, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "Qwen/Qwen3.5-397B-A17B-FP8/o_proj", + "Qwen/Qwen3.5-397B-A17B-FP8/shared_expert.down_proj", + "deepseek-ai/DeepSeek-R1-0528/kv_b_proj" + ] }, { "type": "gemm", "args": { - "m": 512, - "n": 7168, - "k": 2048, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 30, + "n": 4608, + "k": 7168, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/gate_up_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/gate_up_proj", + "nvidia/Kimi-K2.5-NVFP4/gate_up_proj" + ] }, { "type": "gemm", "args": { - "m": 512, - "n": 2304, + "m": 30, + "n": 4608, "k": 7168, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/gate_up_proj" + ] }, { "type": "gemm", "args": { - "m": 512, - "n": 7168, - "k": 2304, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 30, + "n": 6144, + "k": 1536, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/GLM-5.1-MXFP4/down_proj", + "amd/GLM-5.2-MXFP4/down_proj", + "amd/Kimi-K2.5-MXFP4/q_b_proj", + "amd/MiniMax-M3-MXFP4/down_proj", + "moonshotai/Kimi-K2.5/q_b_proj", + "nvidia/DeepSeek-R1-0528-FP4-V2/q_b_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/q_b_proj", + "nvidia/GLM-5.1-NVFP4/down_proj", + "nvidia/GLM-5.2-NVFP4/down_proj", + "nvidia/Kimi-K2.5-NVFP4/q_b_proj", + "nvidia/Kimi-K2.6-NVFP4/q_b_proj" + ] }, { "type": "gemm", "args": { - "m": 114688, - "n": 1536, - "k": 7168, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 30, + "n": 7168, + "k": 1024, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/o_proj", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] }, { "type": "gemm", "args": { - "m": 114688, - "n": 24576, - "k": 1536, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 30, + "n": 7168, + "k": 1152, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/down_proj" + ] }, { "type": "gemm", "args": { - "m": 114688, - "n": 576, - "k": 7168, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 30, + "n": 7168, + "k": 2048, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/Kimi-K2.5-MXFP4/o_proj", + "moonshotai/Kimi-K2.5/o_proj", + "nvidia/Kimi-K2.5-NVFP4/o_proj", + "nvidia/Kimi-K2.6-NVFP4/o_proj" + ] }, { "type": "gemm", "args": { - "m": 114688, - "n": 32768, - "k": 512, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 30, + "n": 7168, + "k": 2048, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/o_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/o_proj" + ] }, { "type": "gemm", "args": { - "m": 114688, + "m": 30, "n": 7168, - "k": 16384, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "k": 2048, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/o_proj", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] }, { "type": "gemm", "args": { - "m": 114688, - "n": 2304, - "k": 7168, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 30, + "n": 7168, + "k": 2304, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/down_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/down_proj", + "nvidia/Kimi-K2.5-NVFP4/down_proj" + ] }, { "type": "gemm", "args": { - "m": 114688, + "m": 30, "n": 7168, "k": 2304, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/down_proj" + ] }, { "type": "gemm", "args": { - "m": 224, - "n": 1536, - "k": 7168, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 30, + "n": 7168, + "k": 4096, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/o_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/o_proj" + ] }, { "type": "gemm", "args": { - "m": 224, - "n": 24576, - "k": 1536, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 30, + "n": 7168, + "k": 4608, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/down_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/down_proj", + "nvidia/Kimi-K2.5-NVFP4/down_proj" + ] }, { "type": "gemm", "args": { - "m": 224, - "n": 576, - "k": 7168, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 30, + "n": 7168, + "k": 9216, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/down_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/down_proj", + "nvidia/Kimi-K2.5-NVFP4/down_proj" + ] }, { "type": "gemm", "args": { - "m": 224, - "n": 32768, - "k": 512, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 30, + "n": 7168, + "k": 16384, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/o_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/o_proj" + ] }, { "type": "gemm", "args": { - "m": 224, + "m": 30, "n": 7168, "k": 16384, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 32 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 1, + 32 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "amd/DeepSeek-R1-0528-MXFP4-Preview/o_proj", + "amd/DeepSeek-R1-0528-MXFP4-v2/o_proj", + "amd/DeepSeek-R1-0528-MXFP4/o_proj" + ] }, { "type": "gemm", "args": { - "m": 224, - "n": 2304, - "k": 7168, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 30, + "n": 7168, + "k": 16384, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] }, { "type": "gemm", "args": { - "m": 224, - "n": 7168, - "k": 2304, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 30, + "n": 8192, + "k": 512, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/Kimi-K2.5-MXFP4/kv_b_proj", + "moonshotai/Kimi-K2.5/kv_b_proj", + "nvidia/DeepSeek-R1-0528-FP4-V2/kv_b_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/kv_b_proj", + "nvidia/Kimi-K2.5-NVFP4/kv_b_proj", + "nvidia/Kimi-K2.6-NVFP4/kv_b_proj" + ] }, { "type": "gemm", "args": { - "m": 131072, - "n": 1536, + "m": 30, + "n": 9216, "k": 7168, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/gate_up_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/gate_up_proj", + "nvidia/Kimi-K2.5-NVFP4/gate_up_proj" + ] }, { "type": "gemm", "args": { - "m": 131072, + "m": 30, "n": 24576, "k": 1536, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/q_b_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/q_b_proj" + ] }, { "type": "gemm", "args": { - "m": 131072, - "n": 576, - "k": 7168, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 30, + "n": 24576, + "k": 1536, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 32 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 1, + 32 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "amd/DeepSeek-R1-0528-MXFP4-Preview/q_b_proj", + "amd/DeepSeek-R1-0528-MXFP4-v2/q_b_proj", + "amd/DeepSeek-R1-0528-MXFP4/q_b_proj" + ] }, { "type": "gemm", "args": { - "m": 131072, + "m": 30, "n": 32768, "k": 512, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/kv_b_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/kv_b_proj" + ] }, { "type": "gemm", "args": { - "m": 131072, - "n": 7168, - "k": 16384, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 30, + "n": 32768, + "k": 512, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 32 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 1, + 32 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "amd/DeepSeek-R1-0528-MXFP4-Preview/kv_b_proj", + "amd/DeepSeek-R1-0528-MXFP4-v2/kv_b_proj", + "amd/DeepSeek-R1-0528-MXFP4/kv_b_proj" + ] }, { "type": "gemm", "args": { - "m": 131072, - "n": 2304, + "m": 31, + "n": 576, "k": 7168, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/Kimi-K2.5-MXFP4/kv_a_proj_with_mqa", + "moonshotai/Kimi-K2.5/kv_a_proj_with_mqa", + "nvidia/DeepSeek-R1-0528-FP4-V2/kv_a_proj_with_mqa", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/kv_a_proj_with_mqa", + "nvidia/Kimi-K2.5-NVFP4/kv_a_proj_with_mqa", + "nvidia/Kimi-K2.6-NVFP4/kv_a_proj_with_mqa" + ] }, { "type": "gemm", "args": { - "m": 131072, - "n": 7168, - "k": 2304, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 31, + "n": 576, + "k": 7168, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 32 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 1, + 32 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "amd/DeepSeek-R1-0528-MXFP4-Preview/kv_a_proj_with_mqa", + "amd/DeepSeek-R1-0528-MXFP4-v2/kv_a_proj_with_mqa", + "amd/DeepSeek-R1-0528-MXFP4/kv_a_proj_with_mqa" + ] }, { "type": "gemm", "args": { - "m": 256, + "m": 31, "n": 1536, "k": 7168, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/Kimi-K2.5-MXFP4/q_a_proj", + "moonshotai/Kimi-K2.5/q_a_proj", + "nvidia/DeepSeek-R1-0528-FP4-V2/q_a_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/q_a_proj", + "nvidia/Kimi-K2.5-NVFP4/q_a_proj", + "nvidia/Kimi-K2.6-NVFP4/q_a_proj" + ] }, { "type": "gemm", "args": { - "m": 256, - "n": 24576, - "k": 1536, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 31, + "n": 1536, + "k": 7168, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 32 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 1, + 32 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "amd/DeepSeek-R1-0528-MXFP4-Preview/q_a_proj", + "amd/DeepSeek-R1-0528-MXFP4-v2/q_a_proj", + "amd/DeepSeek-R1-0528-MXFP4/q_a_proj" + ] }, { "type": "gemm", "args": { - "m": 256, - "n": 576, + "m": 31, + "n": 1536, "k": 7168, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w3" + ] }, { "type": "gemm", "args": { - "m": 256, - "n": 32768, - "k": 512, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 31, + "n": 7168, + "k": 9216, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/down_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/down_proj", + "nvidia/Kimi-K2.5-NVFP4/down_proj" + ] }, { "type": "gemm", "args": { - "m": 256, + "m": 31, "n": 7168, "k": 16384, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/o_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/o_proj" + ] }, { "type": "gemm", "args": { - "m": 256, - "n": 2304, - "k": 7168, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 31, + "n": 7168, + "k": 16384, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 32 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 1, + 32 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "amd/DeepSeek-R1-0528-MXFP4-Preview/o_proj", + "amd/DeepSeek-R1-0528-MXFP4-v2/o_proj", + "amd/DeepSeek-R1-0528-MXFP4/o_proj" + ] }, { "type": "gemm", "args": { - "m": 256, + "m": 31, "n": 7168, - "k": 2304, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "k": 16384, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] }, { "type": "gemm", "args": { - "m": 151552, - "n": 1536, + "m": 31, + "n": 9216, + "k": 7168, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/gate_up_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/gate_up_proj", + "nvidia/Kimi-K2.5-NVFP4/gate_up_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 31, + "n": 9216, "k": 7168, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 32 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 1, + 32 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "amd/DeepSeek-R1-0528-MXFP4-Preview/gate_up_proj", + "amd/DeepSeek-R1-0528-MXFP4-v2/gate_up_proj", + "amd/DeepSeek-R1-0528-MXFP4/gate_up_proj", + "amd/Kimi-K2.5-MXFP4/gate_up_proj" + ] }, { "type": "gemm", "args": { - "m": 151552, + "m": 31, "n": 24576, "k": 1536, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/q_b_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/q_b_proj" + ] }, { "type": "gemm", "args": { - "m": 151552, - "n": 576, - "k": 7168, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 31, + "n": 24576, + "k": 1536, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 32 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 1, + 32 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "amd/DeepSeek-R1-0528-MXFP4-Preview/q_b_proj", + "amd/DeepSeek-R1-0528-MXFP4-v2/q_b_proj", + "amd/DeepSeek-R1-0528-MXFP4/q_b_proj" + ] }, { "type": "gemm", "args": { - "m": 151552, + "m": 31, "n": 32768, "k": 512, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/kv_b_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/kv_b_proj" + ] }, { "type": "gemm", "args": { - "m": 151552, - "n": 7168, - "k": 16384, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 31, + "n": 32768, + "k": 512, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 32 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 1, + 32 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "amd/DeepSeek-R1-0528-MXFP4-Preview/kv_b_proj", + "amd/DeepSeek-R1-0528-MXFP4-v2/kv_b_proj", + "amd/DeepSeek-R1-0528-MXFP4/kv_b_proj" + ] }, { "type": "gemm", "args": { - "m": 151552, - "n": 2304, - "k": 7168, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 32, + "n": 64, + "k": 2880, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/gpt-oss-120b-w-mxfp4-a-fp8/k_proj", + "amd/gpt-oss-120b-w-mxfp4-a-fp8/v_proj", + "openai/gpt-oss-120b/k_proj", + "openai/gpt-oss-120b/v_proj" + ] }, { "type": "gemm", "args": { - "m": 151552, - "n": 7168, - "k": 2304, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 32, + "n": 128, + "k": 2880, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/gpt-oss-120b-w-mxfp4-a-fp8/k_proj", + "amd/gpt-oss-120b-w-mxfp4-a-fp8/v_proj", + "openai/gpt-oss-120b/k_proj", + "openai/gpt-oss-120b/v_proj" + ] }, { "type": "gemm", "args": { - "m": 296, - "n": 1536, - "k": 7168, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 32, + "n": 128, + "k": 3072, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "MiniMaxAI/MiniMax-M2.5/k_proj", + "MiniMaxAI/MiniMax-M2.5/v_proj" + ] }, { "type": "gemm", "args": { - "m": 296, - "n": 24576, - "k": 1536, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 32, + "n": 128, + "k": 4096, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "Qwen/Qwen3.5-397B-A17B/shared_expert.gate_up_proj", + "amd/Qwen3.5-397B-A17B-MXFP4/shared_expert.gate_up_proj", + "nvidia/Qwen3.5-397B-A17B-NVFP4/shared_expert.gate_up_proj" + ] }, { "type": "gemm", "args": { - "m": 296, - "n": 576, - "k": 7168, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 32, + "n": 128, + "k": 4096, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "Qwen/Qwen3.5-397B-A17B-FP8/shared_expert.gate_up_proj" + ] }, { "type": "gemm", "args": { - "m": 296, - "n": 32768, - "k": 512, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 32, + "n": 192, + "k": 3072, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "MiniMaxAI/MiniMax-M2.5/experts.gate_up_proj" + ] }, { "type": "gemm", "args": { - "m": 296, - "n": 7168, - "k": 16384, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 32, + "n": 256, + "k": 2880, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/gpt-oss-120b-w-mxfp4-a-fp8/k_proj", + "amd/gpt-oss-120b-w-mxfp4-a-fp8/q_proj", + "amd/gpt-oss-120b-w-mxfp4-a-fp8/v_proj", + "openai/gpt-oss-120b/k_proj", + "openai/gpt-oss-120b/q_proj", + "openai/gpt-oss-120b/v_proj" + ] }, { "type": "gemm", "args": { - "m": 296, - "n": 2304, - "k": 7168, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 32, + "n": 256, + "k": 3072, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/MiniMax-M2.5-MXFP4/k_proj", + "amd/MiniMax-M2.5-MXFP4/v_proj", + "nvidia/MiniMax-M2.5-NVFP4/k_proj", + "nvidia/MiniMax-M2.5-NVFP4/v_proj" + ] }, { "type": "gemm", "args": { - "m": 296, - "n": 7168, - "k": 2304, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 32, + "n": 256, + "k": 3072, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "MiniMaxAI/MiniMax-M2.5/k_proj", + "MiniMaxAI/MiniMax-M2.5/v_proj" + ] }, { "type": "gemm", "args": { - "m": 204800, - "n": 1536, - "k": 7168, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 32, + "n": 256, + "k": 4096, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "Qwen/Qwen3.5-397B-A17B/k_proj", + "Qwen/Qwen3.5-397B-A17B/shared_expert.gate_up_proj", + "Qwen/Qwen3.5-397B-A17B/v_proj", + "amd/Qwen3.5-397B-A17B-MXFP4/k_proj", + "amd/Qwen3.5-397B-A17B-MXFP4/shared_expert.gate_up_proj", + "amd/Qwen3.5-397B-A17B-MXFP4/v_proj", + "nvidia/Qwen3.5-397B-A17B-NVFP4/k_proj", + "nvidia/Qwen3.5-397B-A17B-NVFP4/shared_expert.gate_up_proj", + "nvidia/Qwen3.5-397B-A17B-NVFP4/v_proj" + ] }, { "type": "gemm", "args": { - "m": 204800, - "n": 24576, - "k": 1536, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 32, + "n": 256, + "k": 4096, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "Qwen/Qwen3.5-397B-A17B-FP8/k_proj", + "Qwen/Qwen3.5-397B-A17B-FP8/shared_expert.gate_up_proj", + "Qwen/Qwen3.5-397B-A17B-FP8/v_proj" + ] }, { "type": "gemm", "args": { - "m": 204800, - "n": 576, - "k": 7168, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 32, + "n": 384, + "k": 3072, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "MiniMaxAI/MiniMax-M2.5/experts.gate_up_proj", + "MiniMaxAI/MiniMax-M2.5/q_proj" + ] }, { "type": "gemm", "args": { - "m": 204800, - "n": 32768, - "k": 512, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 32, + "n": 512, + "k": 2880, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/gpt-oss-120b-w-mxfp4-a-fp8/k_proj", + "amd/gpt-oss-120b-w-mxfp4-a-fp8/q_proj", + "amd/gpt-oss-120b-w-mxfp4-a-fp8/v_proj", + "openai/gpt-oss-120b/k_proj", + "openai/gpt-oss-120b/q_proj", + "openai/gpt-oss-120b/v_proj" + ] }, { "type": "gemm", "args": { - "m": 204800, - "n": 7168, - "k": 16384, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 32, + "n": 512, + "k": 3072, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/MiniMax-M2.5-MXFP4/k_proj", + "amd/MiniMax-M2.5-MXFP4/v_proj", + "nvidia/MiniMax-M2.5-NVFP4/k_proj", + "nvidia/MiniMax-M2.5-NVFP4/v_proj" + ] }, { "type": "gemm", "args": { - "m": 204800, - "n": 2304, - "k": 7168, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 32, + "n": 512, + "k": 3072, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "MiniMaxAI/MiniMax-M2.5/k_proj", + "MiniMaxAI/MiniMax-M2.5/v_proj" + ] }, { "type": "gemm", "args": { - "m": 204800, - "n": 7168, - "k": 2304, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 32, + "n": 512, + "k": 4096, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "Qwen/Qwen3.5-397B-A17B/k_proj", + "Qwen/Qwen3.5-397B-A17B/q_proj", + "Qwen/Qwen3.5-397B-A17B/shared_expert.gate_up_proj", + "Qwen/Qwen3.5-397B-A17B/v_proj", + "amd/Qwen3.5-397B-A17B-MXFP4/k_proj", + "amd/Qwen3.5-397B-A17B-MXFP4/q_proj", + "amd/Qwen3.5-397B-A17B-MXFP4/shared_expert.gate_up_proj", + "amd/Qwen3.5-397B-A17B-MXFP4/v_proj", + "nvidia/Qwen3.5-397B-A17B-NVFP4/k_proj", + "nvidia/Qwen3.5-397B-A17B-NVFP4/q_proj", + "nvidia/Qwen3.5-397B-A17B-NVFP4/shared_expert.gate_up_proj", + "nvidia/Qwen3.5-397B-A17B-NVFP4/v_proj" + ] }, { "type": "gemm", "args": { - "m": 400, - "n": 1536, - "k": 7168, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 32, + "n": 512, + "k": 4096, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "Qwen/Qwen3.5-397B-A17B-FP8/k_proj", + "Qwen/Qwen3.5-397B-A17B-FP8/q_proj", + "Qwen/Qwen3.5-397B-A17B-FP8/shared_expert.gate_up_proj", + "Qwen/Qwen3.5-397B-A17B-FP8/v_proj", + "deepseek-ai/DeepSeek-V4.1-Flash/attn.wo_a" + ] }, { "type": "gemm", "args": { - "m": 400, - "n": 24576, - "k": 1536, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 32, + "n": 576, + "k": 6144, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/GLM-5.1-MXFP4/kv_a_proj_with_mqa", + "amd/GLM-5.2-MXFP4/kv_a_proj_with_mqa", + "nvidia/GLM-5-NVFP4/kv_a_proj_with_mqa", + "nvidia/GLM-5.1-NVFP4/kv_a_proj_with_mqa", + "nvidia/GLM-5.2-NVFP4/kv_a_proj_with_mqa" + ] }, { "type": "gemm", "args": { - "m": 400, + "m": 32, "n": 576, - "k": 7168, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "k": 6144, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "zai-org/GLM-5-FP8/kv_a_proj_with_mqa", + "zai-org/GLM-5.1-FP8/kv_a_proj_with_mqa", + "zai-org/GLM-5.2-FP8/kv_a_proj_with_mqa" + ] }, { "type": "gemm", "args": { - "m": 400, - "n": 32768, - "k": 512, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 32, + "n": 576, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/Kimi-K2.5-MXFP4/kv_a_proj_with_mqa", + "moonshotai/Kimi-K2.5/kv_a_proj_with_mqa", + "nvidia/DeepSeek-R1-0528-FP4-V2/kv_a_proj_with_mqa", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/kv_a_proj_with_mqa", + "nvidia/Kimi-K2.5-NVFP4/kv_a_proj_with_mqa", + "nvidia/Kimi-K2.6-NVFP4/kv_a_proj_with_mqa" + ] }, { "type": "gemm", "args": { - "m": 400, - "n": 7168, - "k": 16384, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 32, + "n": 576, + "k": 7168, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 32 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 1, + 32 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "amd/DeepSeek-R1-0528-MXFP4-Preview/kv_a_proj_with_mqa", + "amd/DeepSeek-R1-0528-MXFP4-v2/kv_a_proj_with_mqa", + "amd/DeepSeek-R1-0528-MXFP4/kv_a_proj_with_mqa" + ] }, { "type": "gemm", "args": { - "m": 400, - "n": 2304, + "m": 32, + "n": 576, "k": 7168, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/kv_a_proj_with_mqa" + ] }, { "type": "gemm", "args": { - "m": 400, - "n": 7168, - "k": 2304, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 32, + "n": 768, + "k": 3072, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 32 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 1, + 32 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "amd/MiniMax-M2.5-MXFP4/experts.gate_up_proj" + ] }, { "type": "gemm", "args": { - "m": 512, - "n": 24576, - "k": 1536, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 32, + "n": 768, + "k": 3072, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "MiniMaxAI/MiniMax-M2.5/experts.gate_up_proj", + "MiniMaxAI/MiniMax-M2.5/q_proj" + ] }, { "type": "gemm", "args": { - "m": 512, - "n": 32768, - "k": 512, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 32, + "n": 1024, + "k": 2880, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/gpt-oss-120b-w-mxfp4-a-fp8/q_proj", + "openai/gpt-oss-120b/q_proj" + ] }, { "type": "gemm", "args": { - "m": 512, - "n": 7168, - "k": 16384, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 32, + "n": 1024, + "k": 3072, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "MiniMaxAI/MiniMax-M2.5/k_proj", + "MiniMaxAI/MiniMax-M2.5/qkv_proj", + "MiniMaxAI/MiniMax-M2.5/v_proj" + ] }, { "type": "gemm", "args": { - "m": 0, - "n": 1536, - "k": 7168, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 32, + "n": 1024, + "k": 4096, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "Qwen/Qwen3.5-397B-A17B/q_proj", + "Qwen/Qwen3.5-397B-A17B/shared_expert.gate_up_proj", + "amd/Qwen3.5-397B-A17B-MXFP4/q_proj", + "amd/Qwen3.5-397B-A17B-MXFP4/shared_expert.gate_up_proj", + "nvidia/Qwen3.5-397B-A17B-NVFP4/q_proj", + "nvidia/Qwen3.5-397B-A17B-NVFP4/shared_expert.gate_up_proj" + ] }, { "type": "gemm", "args": { - "m": 0, - "n": 24576, - "k": 1536, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 32, + "n": 1024, + "k": 4096, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "Qwen/Qwen3.5-397B-A17B-FP8/q_proj", + "Qwen/Qwen3.5-397B-A17B-FP8/shared_expert.gate_up_proj", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_a", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_a", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_a" + ] }, { "type": "gemm", "args": { - "m": 0, - "n": 576, + "m": 32, + "n": 1088, "k": 7168, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 0, - "n": 32768, - "k": 512, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] }, { "type": "gemm", "args": { - "m": 0, - "n": 7168, - "k": 16384, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 32, + "n": 1152, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/gate_up_proj" + ] }, { "type": "gemm", "args": { - "m": 0, - "n": 2304, - "k": 7168, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 32, + "n": 1536, + "k": 1536, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/Kimi-K2.5-MXFP4/q_b_proj", + "moonshotai/Kimi-K2.5/q_b_proj", + "nvidia/DeepSeek-R1-0528-FP4-V2/q_b_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/q_b_proj", + "nvidia/Kimi-K2.5-NVFP4/q_b_proj", + "nvidia/Kimi-K2.6-NVFP4/q_b_proj" + ] }, { "type": "gemm", "args": { - "m": 0, - "n": 7168, - "k": 2304, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 32, + "n": 1536, + "k": 3072, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 32 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 1, + 32 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "amd/MiniMax-M2.5-MXFP4/experts.gate_up_proj" + ] }, { "type": "gemm", "args": { - "m": 4096, - "n": 3072, - "k": 1536, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 32, + "n": 1536, + "k": 3072, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "MiniMaxAI/MiniMax-M2.5/experts.gate_up_proj", + "MiniMaxAI/MiniMax-M2.5/q_proj" + ] }, { "type": "gemm", "args": { - "m": 4096, - "n": 4096, - "k": 512, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 32, + "n": 1536, + "k": 6144, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "zai-org/GLM-5-FP8/gate_up_proj", + "zai-org/GLM-5.1-FP8/gate_up_proj", + "zai-org/GLM-5.2-FP8/gate_up_proj" + ] }, { "type": "gemm", "args": { - "m": 4096, - "n": 7168, - "k": 2048, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 32, + "n": 1536, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/Kimi-K2.5-MXFP4/q_a_proj", + "moonshotai/Kimi-K2.5/q_a_proj", + "nvidia/DeepSeek-R1-0528-FP4-V2/q_a_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/q_a_proj", + "nvidia/Kimi-K2.5-NVFP4/q_a_proj", + "nvidia/Kimi-K2.6-NVFP4/q_a_proj" + ] }, { "type": "gemm", "args": { - "m": 4, + "m": 32, "n": 1536, "k": 7168, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 32 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 1, + 32 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "amd/DeepSeek-R1-0528-MXFP4-Preview/q_a_proj", + "amd/DeepSeek-R1-0528-MXFP4-v2/q_a_proj", + "amd/DeepSeek-R1-0528-MXFP4/q_a_proj" + ] }, { "type": "gemm", "args": { - "m": 4, - "n": 3072, - "k": 1536, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 32, + "n": 1536, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/gate_up_proj", + "deepseek-ai/DeepSeek-R1-0528/q_a_proj", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w3" + ] }, { "type": "gemm", "args": { - "m": 4, - "n": 576, + "m": 32, + "n": 1536, "k": 7168, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w3" + ] }, { "type": "gemm", "args": { - "m": 4, - "n": 4096, + "m": 32, + "n": 2048, "k": 512, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/Kimi-K2.5-MXFP4/kv_b_proj", + "moonshotai/Kimi-K2.5/kv_b_proj", + "nvidia/DeepSeek-R1-0528-FP4-V2/kv_b_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/kv_b_proj", + "nvidia/Kimi-K2.5-NVFP4/kv_b_proj", + "nvidia/Kimi-K2.6-NVFP4/kv_b_proj" + ] }, { "type": "gemm", "args": { - "m": 4, - "n": 7168, + "m": 32, + "n": 2048, "k": 2048, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "zai-org/GLM-5-FP8/q_b_proj", + "zai-org/GLM-5.1-FP8/q_b_proj", + "zai-org/GLM-5.2-FP8/q_b_proj" + ] }, { "type": "gemm", "args": { - "m": 4, - "n": 2304, - "k": 7168, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 32, + "n": 2048, + "k": 2880, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/gpt-oss-120b-w-mxfp4-a-fp8/q_proj", + "openai/gpt-oss-120b/q_proj" + ] }, { "type": "gemm", "args": { - "m": 4, - "n": 7168, - "k": 2304, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 32, + "n": 2048, + "k": 4096, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "Qwen/Qwen3.5-397B-A17B/q_proj", + "Qwen/Qwen3.5-397B-A17B/shared_expert.gate_up_proj", + "amd/Qwen3.5-397B-A17B-MXFP4/q_proj", + "amd/Qwen3.5-397B-A17B-MXFP4/shared_expert.gate_up_proj", + "nvidia/Qwen3.5-397B-A17B-NVFP4/q_proj", + "nvidia/Qwen3.5-397B-A17B-NVFP4/shared_expert.gate_up_proj" + ] }, { "type": "gemm", "args": { - "m": 4, - "n": 24576, - "k": 1536, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 32, + "n": 2048, + "k": 4096, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "Qwen/Qwen3.5-397B-A17B-FP8/q_proj", + "Qwen/Qwen3.5-397B-A17B-FP8/shared_expert.gate_up_proj" + ] }, { "type": "gemm", "args": { - "m": 4, - "n": 32768, - "k": 512, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 32, + "n": 2048, + "k": 6144, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/GLM-5.1-MXFP4/q_a_proj", + "amd/GLM-5.2-MXFP4/q_a_proj", + "amd/MiniMax-M3-MXFP4/q_proj", + "nvidia/GLM-5-NVFP4/q_a_proj", + "nvidia/GLM-5.1-NVFP4/q_a_proj", + "nvidia/GLM-5.2-NVFP4/q_a_proj" + ] }, { "type": "gemm", "args": { - "m": 4, - "n": 7168, - "k": 16384, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 32, + "n": 2048, + "k": 6144, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "zai-org/GLM-5-FP8/q_a_proj", + "zai-org/GLM-5.1-FP8/q_a_proj", + "zai-org/GLM-5.2-FP8/q_a_proj" + ] }, { "type": "gemm", "args": { "m": 32, - "n": 1536, + "n": 2304, "k": 7168, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/gate_up_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/gate_up_proj", + "nvidia/Kimi-K2.5-NVFP4/gate_up_proj" + ] }, { "type": "gemm", "args": { "m": 32, - "n": 3072, - "k": 1536, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "n": 2880, + "k": 512, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/gpt-oss-120b-w-mxfp4-a-fp8/o_proj", + "openai/gpt-oss-120b/o_proj" + ] }, { "type": "gemm", "args": { "m": 32, - "n": 576, - "k": 7168, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "n": 2880, + "k": 1024, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/gpt-oss-120b-w-mxfp4-a-fp8/o_proj", + "openai/gpt-oss-120b/o_proj" + ] }, { "type": "gemm", "args": { "m": 32, - "n": 4096, - "k": 512, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "n": 2880, + "k": 2048, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/gpt-oss-120b-w-mxfp4-a-fp8/o_proj", + "openai/gpt-oss-120b/o_proj" + ] }, { "type": "gemm", "args": { "m": 32, - "n": 7168, - "k": 2048, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "n": 2880, + "k": 4096, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/gpt-oss-120b-w-mxfp4-a-fp8/o_proj", + "openai/gpt-oss-120b/o_proj" + ] }, { "type": "gemm", "args": { "m": 32, - "n": 2304, - "k": 7168, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "n": 3072, + "k": 192, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "MiniMaxAI/MiniMax-M2.5/experts.down_proj" + ] }, { "type": "gemm", "args": { "m": 32, - "n": 7168, - "k": 2304, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "n": 3072, + "k": 384, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 32 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 1, + 32 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "amd/MiniMax-M2.5-MXFP4/experts.down_proj" + ] }, { "type": "gemm", "args": { - "m": 16384, - "n": 1536, - "k": 7168, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 32, + "n": 3072, + "k": 384, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "MiniMaxAI/MiniMax-M2.5/experts.down_proj", + "MiniMaxAI/MiniMax-M2.5/o_proj" + ] }, { "type": "gemm", "args": { - "m": 16384, - "n": 24576, - "k": 1536, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 32, + "n": 3072, + "k": 768, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 32 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 1, + 32 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "amd/MiniMax-M2.5-MXFP4/experts.down_proj" + ] }, { "type": "gemm", "args": { - "m": 16384, - "n": 576, - "k": 7168, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 32, + "n": 3072, + "k": 768, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "MiniMaxAI/MiniMax-M2.5/experts.down_proj", + "MiniMaxAI/MiniMax-M2.5/o_proj" + ] }, { "type": "gemm", "args": { - "m": 16384, - "n": 32768, - "k": 512, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 32, + "n": 3072, + "k": 1536, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/Kimi-K2.5-MXFP4/q_b_proj", + "amd/MiniMax-M2.5-MXFP4/o_proj", + "moonshotai/Kimi-K2.5/q_b_proj", + "nvidia/DeepSeek-R1-0528-FP4-V2/q_b_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/q_b_proj", + "nvidia/Kimi-K2.5-NVFP4/q_b_proj", + "nvidia/Kimi-K2.6-NVFP4/q_b_proj", + "nvidia/MiniMax-M2.5-NVFP4/o_proj" + ] }, { "type": "gemm", "args": { - "m": 16384, - "n": 7168, - "k": 16384, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 32, + "n": 3072, + "k": 1536, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/MiniMax-M2.5-NVFP4/experts.down_proj" + ] }, { "type": "gemm", "args": { - "m": 16384, - "n": 2304, - "k": 7168, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 32, + "n": 3072, + "k": 1536, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 32 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 1, + 32 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "amd/DeepSeek-R1-0528-MXFP4-Preview/q_b_proj", + "amd/DeepSeek-R1-0528-MXFP4-v2/q_b_proj", + "amd/DeepSeek-R1-0528-MXFP4/q_b_proj", + "amd/MiniMax-M2.5-MXFP4/experts.down_proj" + ] }, { "type": "gemm", "args": { - "m": 16384, - "n": 7168, - "k": 2304, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 32, + "n": 3072, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "MiniMaxAI/MiniMax-M2.5/experts.down_proj", + "MiniMaxAI/MiniMax-M2.5/o_proj", + "deepseek-ai/DeepSeek-R1-0528/q_b_proj" + ] }, { "type": "gemm", "args": { - "m": 131072, + "m": 32, "n": 3072, - "k": 1536, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "k": 3072, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/MiniMax-M2.5-MXFP4/o_proj", + "amd/MiniMax-M2.5-MXFP4/q_proj", + "nvidia/MiniMax-M2.5-NVFP4/o_proj", + "nvidia/MiniMax-M2.5-NVFP4/q_proj" + ] }, { "type": "gemm", "args": { - "m": 131072, - "n": 4096, - "k": 512, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 32, + "n": 3072, + "k": 3072, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/MiniMax-M2.5-NVFP4/experts.gate_up_proj" + ] }, { "type": "gemm", "args": { - "m": 131072, - "n": 7168, - "k": 2048, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 32, + "n": 3072, + "k": 3072, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 32 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 1, + 32 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "amd/MiniMax-M2.5-MXFP4/experts.gate_up_proj" + ] }, { "type": "gemm", "args": { - "m": 245760, - "n": 1536, - "k": 7168, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 32, + "n": 3072, + "k": 3072, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "MiniMaxAI/MiniMax-M2.5/experts.gate_up_proj", + "MiniMaxAI/MiniMax-M2.5/o_proj", + "MiniMaxAI/MiniMax-M2.5/q_proj" + ] }, { "type": "gemm", "args": { - "m": 245760, - "n": 24576, - "k": 1536, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 32, + "n": 3072, + "k": 6144, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/GLM-5.1-MXFP4/gate_up_proj", + "amd/GLM-5.2-MXFP4/gate_up_proj", + "amd/MiniMax-M2.5-MXFP4/o_proj", + "amd/MiniMax-M3-MXFP4/gate_up_proj", + "nvidia/GLM-5.1-NVFP4/gate_up_proj", + "nvidia/GLM-5.2-NVFP4/gate_up_proj", + "nvidia/MiniMax-M2.5-NVFP4/o_proj" + ] }, { "type": "gemm", "args": { - "m": 245760, - "n": 576, - "k": 7168, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 32, + "n": 3072, + "k": 6144, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "MiniMaxAI/MiniMax-M2.5/o_proj", + "zai-org/GLM-5-FP8/gate_up_proj", + "zai-org/GLM-5.1-FP8/gate_up_proj", + "zai-org/GLM-5.2-FP8/gate_up_proj" + ] }, { "type": "gemm", "args": { - "m": 245760, - "n": 32768, + "m": 32, + "n": 3584, "k": 512, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "zai-org/GLM-5-FP8/kv_b_proj", + "zai-org/GLM-5.1-FP8/kv_b_proj", + "zai-org/GLM-5.2-FP8/kv_b_proj" + ] }, { "type": "gemm", "args": { - "m": 245760, - "n": 7168, - "k": 16384, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 32, + "n": 4096, + "k": 128, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "Qwen/Qwen3.5-397B-A17B/shared_expert.down_proj", + "amd/Qwen3.5-397B-A17B-MXFP4/shared_expert.down_proj", + "nvidia/Qwen3.5-397B-A17B-NVFP4/shared_expert.down_proj" + ] }, { "type": "gemm", "args": { - "m": 245760, - "n": 2304, - "k": 7168, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 32, + "n": 4096, + "k": 128, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "Qwen/Qwen3.5-397B-A17B-FP8/shared_expert.down_proj" + ] }, { "type": "gemm", "args": { - "m": 245760, - "n": 7168, - "k": 2304, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 32, + "n": 4096, + "k": 256, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "Qwen/Qwen3.5-397B-A17B/shared_expert.down_proj", + "amd/Qwen3.5-397B-A17B-MXFP4/shared_expert.down_proj", + "nvidia/Qwen3.5-397B-A17B-NVFP4/shared_expert.down_proj" + ] }, { "type": "gemm", "args": { - "m": 240, - "n": 1536, - "k": 7168, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 32, + "n": 4096, + "k": 256, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "Qwen/Qwen3.5-397B-A17B-FP8/shared_expert.down_proj" + ] }, { "type": "gemm", "args": { - "m": 240, - "n": 24576, - "k": 1536, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 32, + "n": 4096, + "k": 512, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "Qwen/Qwen3.5-397B-A17B/o_proj", + "Qwen/Qwen3.5-397B-A17B/shared_expert.down_proj", + "amd/Kimi-K2.5-MXFP4/kv_b_proj", + "amd/Qwen3.5-397B-A17B-MXFP4/o_proj", + "amd/Qwen3.5-397B-A17B-MXFP4/shared_expert.down_proj", + "moonshotai/Kimi-K2.5/kv_b_proj", + "nvidia/DeepSeek-R1-0528-FP4-V2/kv_b_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/kv_b_proj", + "nvidia/Kimi-K2.5-NVFP4/kv_b_proj", + "nvidia/Kimi-K2.6-NVFP4/kv_b_proj", + "nvidia/Qwen3.5-397B-A17B-NVFP4/o_proj", + "nvidia/Qwen3.5-397B-A17B-NVFP4/shared_expert.down_proj" + ] }, { "type": "gemm", "args": { - "m": 240, - "n": 576, - "k": 7168, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 32, + "n": 4096, + "k": 512, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 32 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 1, + 32 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "amd/DeepSeek-R1-0528-MXFP4-Preview/kv_b_proj", + "amd/DeepSeek-R1-0528-MXFP4-v2/kv_b_proj", + "amd/DeepSeek-R1-0528-MXFP4/kv_b_proj", + "amd/Qwen3.5-397B-A17B-MXFP4-AttnFP8-V2/shared_expert.down_proj" + ] }, { "type": "gemm", "args": { - "m": 240, - "n": 32768, + "m": 32, + "n": 4096, "k": 512, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "Qwen/Qwen3.5-397B-A17B-FP8/o_proj", + "Qwen/Qwen3.5-397B-A17B-FP8/shared_expert.down_proj", + "deepseek-ai/DeepSeek-R1-0528/kv_b_proj" + ] }, { "type": "gemm", "args": { - "m": 240, - "n": 7168, - "k": 16384, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 32, + "n": 4096, + "k": 1024, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "Qwen/Qwen3.5-397B-A17B/o_proj", + "Qwen/Qwen3.5-397B-A17B/shared_expert.down_proj", + "amd/Qwen3.5-397B-A17B-MXFP4/o_proj", + "amd/Qwen3.5-397B-A17B-MXFP4/shared_expert.down_proj", + "nvidia/Qwen3.5-397B-A17B-NVFP4/o_proj", + "nvidia/Qwen3.5-397B-A17B-NVFP4/shared_expert.down_proj" + ] }, { "type": "gemm", "args": { - "m": 240, - "n": 2304, - "k": 7168, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 32, + "n": 4096, + "k": 1024, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "Qwen/Qwen3.5-397B-A17B-FP8/o_proj", + "Qwen/Qwen3.5-397B-A17B-FP8/shared_expert.down_proj" + ] }, { "type": "gemm", "args": { - "m": 240, - "n": 7168, - "k": 2304, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 32, + "n": 4096, + "k": 2048, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "Qwen/Qwen3.5-397B-A17B/o_proj", + "amd/GLM-5.1-MXFP4/q_b_proj", + "amd/GLM-5.2-MXFP4/q_b_proj", + "amd/Qwen3.5-397B-A17B-MXFP4/o_proj", + "nvidia/GLM-5-NVFP4/q_b_proj", + "nvidia/GLM-5.1-NVFP4/q_b_proj", + "nvidia/GLM-5.2-NVFP4/q_b_proj", + "nvidia/Qwen3.5-397B-A17B-NVFP4/o_proj" + ] }, { "type": "gemm", "args": { - "m": 524288, - "n": 1536, - "k": 7168, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 32, + "n": 4096, + "k": 2048, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "Qwen/Qwen3.5-397B-A17B-FP8/o_proj", + "zai-org/GLM-5-FP8/q_b_proj", + "zai-org/GLM-5.1-FP8/q_b_proj", + "zai-org/GLM-5.2-FP8/q_b_proj" + ] }, { "type": "gemm", "args": { - "m": 524288, - "n": 24576, - "k": 1536, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 32, + "n": 4096, + "k": 2880, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/gpt-oss-120b-w-mxfp4-a-fp8/q_proj", + "openai/gpt-oss-120b/q_proj" + ] }, { "type": "gemm", "args": { - "m": 524288, - "n": 576, - "k": 7168, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 32, + "n": 4096, + "k": 4096, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "Qwen/Qwen3.5-397B-A17B/o_proj", + "Qwen/Qwen3.5-397B-A17B/q_proj", + "amd/Qwen3.5-397B-A17B-MXFP4/o_proj", + "amd/Qwen3.5-397B-A17B-MXFP4/q_proj", + "nvidia/Qwen3.5-397B-A17B-NVFP4/o_proj", + "nvidia/Qwen3.5-397B-A17B-NVFP4/q_proj" + ] }, { "type": "gemm", "args": { - "m": 524288, - "n": 32768, - "k": 512, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 32, + "n": 4096, + "k": 4096, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "Qwen/Qwen3.5-397B-A17B-FP8/o_proj", + "Qwen/Qwen3.5-397B-A17B-FP8/q_proj" + ] }, { "type": "gemm", "args": { - "m": 524288, - "n": 7168, - "k": 16384, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 32, + "n": 4608, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "moonshotai/Kimi-K2.5/gate_up_proj", + "nvidia/Kimi-K2.6-NVFP4/gate_up_proj" + ] }, { "type": "gemm", "args": { - "m": 524288, - "n": 2304, + "m": 32, + "n": 4608, "k": 7168, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/gate_up_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/gate_up_proj", + "nvidia/Kimi-K2.5-NVFP4/gate_up_proj" + ] }, { "type": "gemm", "args": { - "m": 524288, - "n": 7168, - "k": 2304, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 32, + "n": 4608, + "k": 7168, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 32 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 1, + 32 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "amd/DeepSeek-R1-0528-MXFP4-Preview/gate_up_proj", + "amd/DeepSeek-R1-0528-MXFP4-v2/gate_up_proj", + "amd/DeepSeek-R1-0528-MXFP4/gate_up_proj", + "amd/Kimi-K2.5-MXFP4/gate_up_proj" + ] }, { "type": "gemm", "args": { - "m": 659456, - "n": 1536, + "m": 32, + "n": 4608, "k": 7168, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/gate_up_proj" + ] }, { "type": "gemm", "args": { - "m": 659456, - "n": 24576, + "m": 32, + "n": 6144, "k": 1536, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/GLM-5.1-MXFP4/down_proj", + "amd/GLM-5.2-MXFP4/down_proj", + "amd/Kimi-K2.5-MXFP4/q_b_proj", + "amd/MiniMax-M3-MXFP4/down_proj", + "moonshotai/Kimi-K2.5/q_b_proj", + "nvidia/DeepSeek-R1-0528-FP4-V2/q_b_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/q_b_proj", + "nvidia/GLM-5.1-NVFP4/down_proj", + "nvidia/GLM-5.2-NVFP4/down_proj", + "nvidia/Kimi-K2.5-NVFP4/q_b_proj", + "nvidia/Kimi-K2.6-NVFP4/q_b_proj" + ] }, { "type": "gemm", "args": { - "m": 659456, - "n": 576, - "k": 7168, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 32, + "n": 6144, + "k": 1536, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 32 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 1, + 32 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "amd/DeepSeek-R1-0528-MXFP4-Preview/q_b_proj", + "amd/DeepSeek-R1-0528-MXFP4-v2/q_b_proj", + "amd/DeepSeek-R1-0528-MXFP4/q_b_proj" + ] }, { "type": "gemm", "args": { - "m": 659456, - "n": 32768, - "k": 512, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 32, + "n": 6144, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/q_b_proj", + "zai-org/GLM-5-FP8/down_proj", + "zai-org/GLM-5.1-FP8/down_proj", + "zai-org/GLM-5.2-FP8/down_proj" + ] }, { "type": "gemm", "args": { - "m": 659456, - "n": 7168, - "k": 16384, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 32, + "n": 6144, + "k": 2048, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "zai-org/GLM-5-FP8/o_proj", + "zai-org/GLM-5.1-FP8/o_proj", + "zai-org/GLM-5.2-FP8/o_proj" + ] }, { "type": "gemm", "args": { - "m": 659456, - "n": 2304, - "k": 7168, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 32, + "n": 6144, + "k": 3072, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/GLM-5.1-MXFP4/down_proj", + "amd/GLM-5.2-MXFP4/down_proj", + "amd/MiniMax-M2.5-MXFP4/q_proj", + "amd/MiniMax-M3-MXFP4/down_proj", + "nvidia/GLM-5.1-NVFP4/down_proj", + "nvidia/GLM-5.2-NVFP4/down_proj", + "nvidia/MiniMax-M2.5-NVFP4/q_proj" + ] }, { "type": "gemm", "args": { - "m": 659456, - "n": 7168, - "k": 2304, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 32, + "n": 6144, + "k": 3072, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/GLM-5-NVFP4/down_proj" + ] }, { "type": "gemm", "args": { - "m": 644, - "n": 1536, - "k": 7168, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 32, + "n": 6144, + "k": 3072, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "MiniMaxAI/MiniMax-M2.5/q_proj", + "zai-org/GLM-5-FP8/down_proj", + "zai-org/GLM-5.1-FP8/down_proj", + "zai-org/GLM-5.2-FP8/down_proj" + ] }, { "type": "gemm", "args": { - "m": 644, - "n": 24576, - "k": 1536, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 32, + "n": 6144, + "k": 4096, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/GLM-5.1-MXFP4/o_proj", + "amd/GLM-5.2-MXFP4/o_proj", + "amd/MiniMax-M3-MXFP4/o_proj", + "nvidia/GLM-5-NVFP4/o_proj", + "nvidia/GLM-5.1-NVFP4/o_proj", + "nvidia/GLM-5.2-NVFP4/o_proj" + ] }, { "type": "gemm", "args": { - "m": 644, - "n": 576, - "k": 7168, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 32, + "n": 6144, + "k": 4096, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "zai-org/GLM-5-FP8/o_proj", + "zai-org/GLM-5.1-FP8/o_proj", + "zai-org/GLM-5.2-FP8/o_proj" + ] }, { "type": "gemm", "args": { - "m": 644, - "n": 32768, - "k": 512, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 32, + "n": 6144, + "k": 6144, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/GLM-5.1-MXFP4/down_proj", + "amd/GLM-5.1-MXFP4/gate_up_proj", + "amd/GLM-5.2-MXFP4/down_proj", + "amd/GLM-5.2-MXFP4/gate_up_proj", + "amd/MiniMax-M3-MXFP4/gate_up_proj", + "nvidia/GLM-5.1-NVFP4/down_proj", + "nvidia/GLM-5.1-NVFP4/gate_up_proj", + "nvidia/GLM-5.2-NVFP4/down_proj", + "nvidia/GLM-5.2-NVFP4/gate_up_proj" + ] }, { "type": "gemm", "args": { - "m": 644, - "n": 7168, - "k": 16384, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 32, + "n": 6144, + "k": 6144, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/GLM-5-NVFP4/down_proj", + "nvidia/GLM-5-NVFP4/gate_up_proj" + ] }, { "type": "gemm", "args": { - "m": 644, - "n": 2304, - "k": 7168, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 32, + "n": 6144, + "k": 6144, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "zai-org/GLM-5-FP8/down_proj", + "zai-org/GLM-5-FP8/gate_up_proj", + "zai-org/GLM-5.1-FP8/down_proj", + "zai-org/GLM-5.1-FP8/gate_up_proj", + "zai-org/GLM-5.2-FP8/down_proj", + "zai-org/GLM-5.2-FP8/gate_up_proj" + ] }, { "type": "gemm", "args": { - "m": 644, - "n": 7168, - "k": 2304, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 32, + "n": 6144, + "k": 8192, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/GLM-5.1-MXFP4/o_proj", + "amd/GLM-5.2-MXFP4/o_proj", + "amd/MiniMax-M3-MXFP4/o_proj", + "nvidia/GLM-5-NVFP4/o_proj", + "nvidia/GLM-5.1-NVFP4/o_proj", + "nvidia/GLM-5.2-NVFP4/o_proj" + ] }, { "type": "gemm", "args": { - "m": 49152, - "n": 1536, - "k": 7168, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 32, + "n": 7168, + "k": 512, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/GLM-5.1-MXFP4/kv_b_proj", + "amd/GLM-5.2-MXFP4/kv_b_proj", + "amd/Kimi-K2.5-MXFP4/o_proj", + "moonshotai/Kimi-K2.5/o_proj", + "nvidia/GLM-5-NVFP4/kv_b_proj", + "nvidia/GLM-5.1-NVFP4/kv_b_proj", + "nvidia/GLM-5.2-NVFP4/kv_b_proj", + "nvidia/Kimi-K2.5-NVFP4/o_proj", + "nvidia/Kimi-K2.6-NVFP4/o_proj" + ] }, { "type": "gemm", "args": { - "m": 49152, - "n": 24576, - "k": 1536, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 32, + "n": 7168, + "k": 512, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "zai-org/GLM-5-FP8/kv_b_proj", + "zai-org/GLM-5.1-FP8/kv_b_proj", + "zai-org/GLM-5.2-FP8/kv_b_proj" + ] }, { "type": "gemm", "args": { - "m": 49152, - "n": 576, - "k": 7168, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 32, + "n": 7168, + "k": 576, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/down_proj" + ] }, { "type": "gemm", "args": { - "m": 49152, - "n": 32768, - "k": 512, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 32, + "n": 7168, + "k": 1024, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/Kimi-K2.5-MXFP4/o_proj", + "moonshotai/Kimi-K2.5/o_proj", + "nvidia/Kimi-K2.5-NVFP4/o_proj", + "nvidia/Kimi-K2.6-NVFP4/o_proj" + ] }, { "type": "gemm", "args": { - "m": 49152, + "m": 32, "n": 7168, - "k": 16384, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "k": 1152, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/down_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/down_proj", + "nvidia/Kimi-K2.5-NVFP4/down_proj" + ] }, { "type": "gemm", "args": { - "m": 49152, - "n": 2304, - "k": 7168, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 32, + "n": 7168, + "k": 2048, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/Kimi-K2.5-MXFP4/o_proj", + "moonshotai/Kimi-K2.5/o_proj", + "nvidia/Kimi-K2.5-NVFP4/o_proj", + "nvidia/Kimi-K2.6-NVFP4/o_proj" + ] }, { "type": "gemm", "args": { - "m": 49152, + "m": 32, "n": 7168, - "k": 2304, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "k": 2048, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/o_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/o_proj" + ] }, { "type": "gemm", "args": { - "m": 12, - "n": 1536, - "k": 7168, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 32, + "n": 7168, + "k": 2048, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 32 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 1, + 32 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "amd/DeepSeek-R1-0528-MXFP4-Preview/o_proj", + "amd/DeepSeek-R1-0528-MXFP4-v2/o_proj", + "amd/DeepSeek-R1-0528-MXFP4/o_proj" + ] }, { "type": "gemm", "args": { - "m": 12, - "n": 24576, - "k": 1536, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 32, + "n": 7168, + "k": 2048, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/o_proj", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] }, { "type": "gemm", "args": { - "m": 12, - "n": 576, - "k": 7168, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 32, + "n": 7168, + "k": 2304, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "moonshotai/Kimi-K2.5/down_proj", + "nvidia/Kimi-K2.6-NVFP4/down_proj" + ] }, { "type": "gemm", "args": { - "m": 12, - "n": 32768, - "k": 512, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 32, + "n": 7168, + "k": 2304, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/down_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/down_proj", + "nvidia/Kimi-K2.5-NVFP4/down_proj" + ] }, { "type": "gemm", "args": { - "m": 12, + "m": 32, "n": 7168, - "k": 16384, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "k": 2304, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 32 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 1, + 32 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "amd/DeepSeek-R1-0528-MXFP4-Preview/down_proj", + "amd/DeepSeek-R1-0528-MXFP4-v2/down_proj", + "amd/DeepSeek-R1-0528-MXFP4/down_proj", + "amd/Kimi-K2.5-MXFP4/down_proj" + ] }, { "type": "gemm", "args": { - "m": 12, - "n": 2304, - "k": 7168, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 32, + "n": 7168, + "k": 2304, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/down_proj" + ] }, { "type": "gemm", "args": { - "m": 12, + "m": 32, "n": 7168, - "k": 2304, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "k": 4096, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/o_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/o_proj" + ] }, { "type": "gemm", "args": { - "m": 393216, - "n": 1536, - "k": 7168, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 32, + "n": 7168, + "k": 4096, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 32 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 1, + 32 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "amd/DeepSeek-R1-0528-MXFP4-Preview/o_proj", + "amd/DeepSeek-R1-0528-MXFP4-v2/o_proj", + "amd/DeepSeek-R1-0528-MXFP4/o_proj" + ] }, { "type": "gemm", "args": { - "m": 393216, - "n": 3072, - "k": 1536, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 32, + "n": 7168, + "k": 4096, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/o_proj", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] }, { "type": "gemm", "args": { - "m": 393216, - "n": 576, - "k": 7168, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 32, + "n": 7168, + "k": 4608, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "moonshotai/Kimi-K2.5/down_proj", + "nvidia/Kimi-K2.6-NVFP4/down_proj" + ] }, { "type": "gemm", "args": { - "m": 393216, - "n": 4096, - "k": 512, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 32, + "n": 7168, + "k": 4608, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/down_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/down_proj", + "nvidia/Kimi-K2.5-NVFP4/down_proj" + ] }, { "type": "gemm", "args": { - "m": 393216, + "m": 32, "n": 7168, - "k": 2048, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "k": 4608, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 32 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 1, + 32 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "amd/DeepSeek-R1-0528-MXFP4-Preview/down_proj", + "amd/DeepSeek-R1-0528-MXFP4-v2/down_proj", + "amd/DeepSeek-R1-0528-MXFP4/down_proj", + "amd/Kimi-K2.5-MXFP4/down_proj" + ] }, { "type": "gemm", "args": { - "m": 393216, - "n": 2304, - "k": 7168, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 32, + "n": 7168, + "k": 4608, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/down_proj" + ] }, { "type": "gemm", "args": { - "m": 393216, + "m": 32, "n": 7168, - "k": 2304, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "k": 8192, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/Kimi-K2.5-MXFP4/o_proj", + "moonshotai/Kimi-K2.5/o_proj", + "nvidia/Kimi-K2.5-NVFP4/o_proj", + "nvidia/Kimi-K2.6-NVFP4/o_proj" + ] }, { "type": "gemm", "args": { - "m": 96, - "n": 1536, - "k": 7168, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 32, + "n": 7168, + "k": 8192, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] }, { "type": "gemm", "args": { - "m": 96, - "n": 3072, - "k": 1536, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 32, + "n": 7168, + "k": 9216, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/down_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/down_proj", + "nvidia/Kimi-K2.5-NVFP4/down_proj" + ] }, { "type": "gemm", "args": { - "m": 96, - "n": 576, - "k": 7168, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 32, + "n": 7168, + "k": 16384, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/o_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/o_proj" + ] }, { "type": "gemm", "args": { - "m": 96, - "n": 4096, - "k": 512, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 32, + "n": 7168, + "k": 16384, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/o_proj", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] }, { "type": "gemm", "args": { - "m": 96, + "m": 32, "n": 7168, - "k": 2048, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "k": 16384, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] }, { "type": "gemm", "args": { - "m": 96, - "n": 2304, - "k": 7168, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 32, + "n": 7168, + "k": 65536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] }, { "type": "gemm", "args": { - "m": 96, - "n": 7168, - "k": 2304, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 32, + "n": 8192, + "k": 512, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/Kimi-K2.5-MXFP4/kv_b_proj", + "moonshotai/Kimi-K2.5/kv_b_proj", + "nvidia/DeepSeek-R1-0528-FP4-V2/kv_b_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/kv_b_proj", + "nvidia/Kimi-K2.5-NVFP4/kv_b_proj", + "nvidia/Kimi-K2.6-NVFP4/kv_b_proj" + ] }, { "type": "gemm", "args": { - "m": 65536, - "n": 24576, - "k": 1536, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 32, + "n": 8192, + "k": 512, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 32 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 1, + 32 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "amd/DeepSeek-R1-0528-MXFP4-Preview/kv_b_proj", + "amd/DeepSeek-R1-0528-MXFP4-v2/kv_b_proj", + "amd/DeepSeek-R1-0528-MXFP4/kv_b_proj" + ] }, { "type": "gemm", "args": { - "m": 65536, - "n": 32768, + "m": 32, + "n": 8192, "k": 512, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/kv_b_proj" + ] }, { "type": "gemm", "args": { - "m": 65536, - "n": 7168, - "k": 16384, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 32, + "n": 8192, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_b" + ] }, { "type": "gemm", "args": { - "m": 524288, - "n": 3072, - "k": 1536, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 32, + "n": 8192, + "k": 2048, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/GLM-5.1-MXFP4/q_b_proj", + "amd/GLM-5.2-MXFP4/q_b_proj", + "nvidia/GLM-5-NVFP4/q_b_proj", + "nvidia/GLM-5.1-NVFP4/q_b_proj", + "nvidia/GLM-5.2-NVFP4/q_b_proj" + ] }, { "type": "gemm", "args": { - "m": 524288, - "n": 4096, - "k": 512, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 32, + "n": 9216, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "moonshotai/Kimi-K2.5/gate_up_proj", + "nvidia/Kimi-K2.6-NVFP4/gate_up_proj" + ] }, { "type": "gemm", "args": { - "m": 524288, - "n": 7168, - "k": 2048, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 32, + "n": 9216, + "k": 7168, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/gate_up_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/gate_up_proj", + "nvidia/Kimi-K2.5-NVFP4/gate_up_proj" + ] }, { "type": "gemm", "args": { - "m": 229376, - "n": 1536, + "m": 32, + "n": 9216, "k": 7168, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 32 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 1, + 32 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "amd/DeepSeek-R1-0528-MXFP4-Preview/gate_up_proj", + "amd/DeepSeek-R1-0528-MXFP4-v2/gate_up_proj", + "amd/DeepSeek-R1-0528-MXFP4/gate_up_proj", + "amd/Kimi-K2.5-MXFP4/gate_up_proj" + ] }, { "type": "gemm", "args": { - "m": 229376, - "n": 24576, - "k": 1536, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 32, + "n": 9216, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/gate_up_proj" + ] }, { "type": "gemm", "args": { - "m": 229376, - "n": 576, - "k": 7168, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 32, + "n": 12288, + "k": 1536, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/Kimi-K2.5-MXFP4/q_b_proj", + "moonshotai/Kimi-K2.5/q_b_proj", + "nvidia/DeepSeek-R1-0528-FP4-V2/q_b_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/q_b_proj", + "nvidia/Kimi-K2.5-NVFP4/q_b_proj", + "nvidia/Kimi-K2.6-NVFP4/q_b_proj" + ] }, { "type": "gemm", "args": { - "m": 229376, - "n": 32768, + "m": 32, + "n": 14336, "k": 512, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/GLM-5.1-MXFP4/kv_b_proj", + "amd/GLM-5.2-MXFP4/kv_b_proj", + "nvidia/GLM-5-NVFP4/kv_b_proj", + "nvidia/GLM-5.1-NVFP4/kv_b_proj", + "nvidia/GLM-5.2-NVFP4/kv_b_proj" + ] }, { "type": "gemm", "args": { - "m": 229376, - "n": 7168, - "k": 16384, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 32, + "n": 15360, + "k": 1024, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] }, { "type": "gemm", "args": { - "m": 229376, - "n": 2304, - "k": 7168, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 32, + "n": 16384, + "k": 512, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/Kimi-K2.5-MXFP4/kv_b_proj", + "moonshotai/Kimi-K2.5/kv_b_proj", + "nvidia/DeepSeek-R1-0528-FP4-V2/kv_b_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/kv_b_proj", + "nvidia/Kimi-K2.5-NVFP4/kv_b_proj", + "nvidia/Kimi-K2.6-NVFP4/kv_b_proj" + ] }, { "type": "gemm", "args": { - "m": 229376, - "n": 7168, - "k": 2304, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 32, + "n": 16384, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_b" + ] }, { "type": "gemm", "args": { - "m": 56, - "n": 1536, - "k": 7168, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 32, + "n": 24576, + "k": 1536, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/q_b_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/q_b_proj" + ] }, { "type": "gemm", "args": { - "m": 56, + "m": 32, "n": 24576, "k": 1536, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/q_b_proj" + ] }, { "type": "gemm", "args": { - "m": 56, - "n": 576, - "k": 7168, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 32, + "n": 30720, + "k": 1024, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] }, { "type": "gemm", "args": { - "m": 56, + "m": 32, "n": 32768, "k": 512, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/kv_b_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/kv_b_proj" + ] }, { "type": "gemm", "args": { - "m": 56, - "n": 7168, - "k": 16384, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 32, + "n": 32768, + "k": 512, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/kv_b_proj" + ] }, { "type": "gemm", "args": { - "m": 56, - "n": 2304, - "k": 7168, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 32, + "n": 65536, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_b" + ] }, { "type": "gemm", "args": { - "m": 56, - "n": 7168, - "k": 2304, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 32, + "n": 122880, + "k": 1024, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] }, { "type": "gemm", "args": { - "m": 294912, - "n": 1536, + "m": 33, + "n": 576, "k": 7168, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/Kimi-K2.5-MXFP4/kv_a_proj_with_mqa", + "moonshotai/Kimi-K2.5/kv_a_proj_with_mqa", + "nvidia/DeepSeek-R1-0528-FP4-V2/kv_a_proj_with_mqa", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/kv_a_proj_with_mqa", + "nvidia/Kimi-K2.5-NVFP4/kv_a_proj_with_mqa", + "nvidia/Kimi-K2.6-NVFP4/kv_a_proj_with_mqa" + ] }, { "type": "gemm", "args": { - "m": 294912, - "n": 24576, - "k": 1536, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 33, + "n": 1536, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/Kimi-K2.5-MXFP4/q_a_proj", + "moonshotai/Kimi-K2.5/q_a_proj", + "nvidia/DeepSeek-R1-0528-FP4-V2/q_a_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/q_a_proj", + "nvidia/Kimi-K2.5-NVFP4/q_a_proj", + "nvidia/Kimi-K2.6-NVFP4/q_a_proj" + ] }, { "type": "gemm", "args": { - "m": 294912, - "n": 576, + "m": 33, + "n": 4608, "k": 7168, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/gate_up_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/gate_up_proj", + "nvidia/Kimi-K2.5-NVFP4/gate_up_proj" + ] }, { "type": "gemm", "args": { - "m": 294912, - "n": 32768, - "k": 512, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 33, + "n": 7168, + "k": 4608, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/down_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/down_proj", + "nvidia/Kimi-K2.5-NVFP4/down_proj" + ] }, { "type": "gemm", "args": { - "m": 294912, + "m": 33, "n": 7168, - "k": 16384, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "k": 4608, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 32 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 1, + 32 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "amd/DeepSeek-R1-0528-MXFP4-Preview/down_proj", + "amd/DeepSeek-R1-0528-MXFP4-v2/down_proj", + "amd/DeepSeek-R1-0528-MXFP4/down_proj", + "amd/Kimi-K2.5-MXFP4/down_proj" + ] }, { "type": "gemm", "args": { - "m": 294912, - "n": 2304, - "k": 7168, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 33, + "n": 7168, + "k": 9216, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/down_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/down_proj", + "nvidia/Kimi-K2.5-NVFP4/down_proj" + ] }, { "type": "gemm", "args": { - "m": 294912, + "m": 33, "n": 7168, - "k": 2304, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "k": 16384, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/o_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/o_proj" + ] }, { "type": "gemm", "args": { - "m": 72, - "n": 1536, + "m": 33, + "n": 9216, "k": 7168, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/gate_up_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/gate_up_proj", + "nvidia/Kimi-K2.5-NVFP4/gate_up_proj" + ] }, { "type": "gemm", "args": { - "m": 72, - "n": 24576, - "k": 1536, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 33, + "n": 9216, + "k": 7168, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 32 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 1, + 32 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "amd/DeepSeek-R1-0528-MXFP4-Preview/gate_up_proj", + "amd/DeepSeek-R1-0528-MXFP4-v2/gate_up_proj", + "amd/DeepSeek-R1-0528-MXFP4/gate_up_proj", + "amd/Kimi-K2.5-MXFP4/gate_up_proj" + ] }, { "type": "gemm", "args": { - "m": 72, - "n": 576, - "k": 7168, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 33, + "n": 24576, + "k": 1536, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/q_b_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/q_b_proj" + ] }, { "type": "gemm", "args": { - "m": 72, + "m": 33, "n": 32768, "k": 512, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/kv_b_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/kv_b_proj" + ] }, { "type": "gemm", "args": { - "m": 72, - "n": 7168, - "k": 16384, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 35, + "n": 576, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/Kimi-K2.5-MXFP4/kv_a_proj_with_mqa", + "moonshotai/Kimi-K2.5/kv_a_proj_with_mqa", + "nvidia/DeepSeek-R1-0528-FP4-V2/kv_a_proj_with_mqa", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/kv_a_proj_with_mqa", + "nvidia/Kimi-K2.5-NVFP4/kv_a_proj_with_mqa", + "nvidia/Kimi-K2.6-NVFP4/kv_a_proj_with_mqa" + ] }, { "type": "gemm", "args": { - "m": 72, - "n": 2304, + "m": 35, + "n": 576, "k": 7168, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/kv_a_proj_with_mqa" + ] }, { "type": "gemm", "args": { - "m": 72, - "n": 7168, - "k": 2304, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 35, + "n": 1536, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/Kimi-K2.5-MXFP4/q_a_proj", + "moonshotai/Kimi-K2.5/q_a_proj", + "nvidia/DeepSeek-R1-0528-FP4-V2/q_a_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/q_a_proj", + "nvidia/Kimi-K2.5-NVFP4/q_a_proj", + "nvidia/Kimi-K2.6-NVFP4/q_a_proj" + ] }, { "type": "gemm", "args": { - "m": 1114112, + "m": 35, "n": 1536, "k": 7168, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/gate_up_proj", + "deepseek-ai/DeepSeek-R1-0528/q_a_proj", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w3" + ] }, { "type": "gemm", "args": { - "m": 1114112, - "n": 24576, - "k": 1536, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 35, + "n": 7168, + "k": 576, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/down_proj" + ] }, { "type": "gemm", "args": { - "m": 1114112, - "n": 576, - "k": 7168, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 35, + "n": 7168, + "k": 16384, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/o_proj", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 35, + "n": 24576, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/q_b_proj" + ] }, { "type": "gemm", "args": { - "m": 1114112, + "m": 35, "n": 32768, "k": 512, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/kv_b_proj" + ] }, { "type": "gemm", "args": { - "m": 1114112, - "n": 7168, - "k": 16384, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 36, + "n": 576, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/Kimi-K2.5-MXFP4/kv_a_proj_with_mqa", + "moonshotai/Kimi-K2.5/kv_a_proj_with_mqa", + "nvidia/DeepSeek-R1-0528-FP4-V2/kv_a_proj_with_mqa", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/kv_a_proj_with_mqa", + "nvidia/Kimi-K2.5-NVFP4/kv_a_proj_with_mqa", + "nvidia/Kimi-K2.6-NVFP4/kv_a_proj_with_mqa" + ] }, { "type": "gemm", "args": { - "m": 1114112, - "n": 2304, + "m": 36, + "n": 576, "k": 7168, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/kv_a_proj_with_mqa" + ] }, { "type": "gemm", "args": { - "m": 1114112, - "n": 7168, - "k": 2304, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 36, + "n": 1536, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/Kimi-K2.5-MXFP4/q_a_proj", + "moonshotai/Kimi-K2.5/q_a_proj", + "nvidia/DeepSeek-R1-0528-FP4-V2/q_a_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/q_a_proj", + "nvidia/Kimi-K2.5-NVFP4/q_a_proj", + "nvidia/Kimi-K2.6-NVFP4/q_a_proj" + ] }, { "type": "gemm", "args": { - "m": 272, + "m": 36, "n": 1536, "k": 7168, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/gate_up_proj", + "deepseek-ai/DeepSeek-R1-0528/q_a_proj", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w3" + ] }, { "type": "gemm", "args": { - "m": 272, - "n": 24576, - "k": 1536, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 36, + "n": 2304, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/gate_up_proj" + ] }, { "type": "gemm", "args": { - "m": 272, - "n": 576, - "k": 7168, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 36, + "n": 3072, + "k": 1536, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/Kimi-K2.5-MXFP4/q_b_proj", + "amd/MiniMax-M2.5-MXFP4/o_proj", + "moonshotai/Kimi-K2.5/q_b_proj", + "nvidia/DeepSeek-R1-0528-FP4-V2/q_b_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/q_b_proj", + "nvidia/Kimi-K2.5-NVFP4/q_b_proj", + "nvidia/Kimi-K2.6-NVFP4/q_b_proj", + "nvidia/MiniMax-M2.5-NVFP4/o_proj" + ] }, { "type": "gemm", "args": { - "m": 272, - "n": 32768, + "m": 36, + "n": 4096, "k": 512, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "Qwen/Qwen3.5-397B-A17B/o_proj", + "Qwen/Qwen3.5-397B-A17B/shared_expert.down_proj", + "amd/Kimi-K2.5-MXFP4/kv_b_proj", + "amd/Qwen3.5-397B-A17B-MXFP4/o_proj", + "amd/Qwen3.5-397B-A17B-MXFP4/shared_expert.down_proj", + "moonshotai/Kimi-K2.5/kv_b_proj", + "nvidia/DeepSeek-R1-0528-FP4-V2/kv_b_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/kv_b_proj", + "nvidia/Kimi-K2.5-NVFP4/kv_b_proj", + "nvidia/Kimi-K2.6-NVFP4/kv_b_proj", + "nvidia/Qwen3.5-397B-A17B-NVFP4/o_proj", + "nvidia/Qwen3.5-397B-A17B-NVFP4/shared_expert.down_proj" + ] }, { "type": "gemm", "args": { - "m": 272, - "n": 7168, - "k": 16384, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 36, + "n": 4608, + "k": 7168, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/gate_up_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/gate_up_proj", + "nvidia/Kimi-K2.5-NVFP4/gate_up_proj" + ] }, { "type": "gemm", "args": { - "m": 272, - "n": 2304, + "m": 36, + "n": 4608, "k": 7168, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/gate_up_proj" + ] }, { "type": "gemm", "args": { - "m": 272, + "m": 36, "n": 7168, - "k": 2304, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "k": 2048, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/o_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/o_proj" + ] }, { "type": "gemm", "args": { - "m": 1, - "n": 1536, - "k": 7168, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 36, + "n": 7168, + "k": 2304, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/down_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/down_proj", + "nvidia/Kimi-K2.5-NVFP4/down_proj" + ] }, { "type": "gemm", "args": { - "m": 1, - "n": 3072, - "k": 1536, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 36, + "n": 7168, + "k": 2304, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/down_proj" + ] }, { "type": "gemm", "args": { - "m": 1, - "n": 576, - "k": 7168, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 36, + "n": 7168, + "k": 4608, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/down_proj" + ] }, { "type": "gemm", "args": { - "m": 1, - "n": 4096, - "k": 512, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 36, + "n": 7168, + "k": 9216, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/down_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/down_proj", + "nvidia/Kimi-K2.5-NVFP4/down_proj" + ] }, { "type": "gemm", "args": { - "m": 1, + "m": 36, "n": 7168, - "k": 2048, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "k": 16384, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/o_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/o_proj" + ] }, { "type": "gemm", "args": { - "m": 1, - "n": 2304, - "k": 7168, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 36, + "n": 7168, + "k": 16384, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/o_proj", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] }, { "type": "gemm", "args": { - "m": 1, - "n": 7168, - "k": 2304, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 36, + "n": 9216, + "k": 7168, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/gate_up_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/gate_up_proj", + "nvidia/Kimi-K2.5-NVFP4/gate_up_proj" + ] }, { "type": "gemm", "args": { - "m": 1048576, - "n": 1536, + "m": 36, + "n": 9216, "k": 7168, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 32 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 1, + 32 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "amd/DeepSeek-R1-0528-MXFP4-Preview/gate_up_proj", + "amd/DeepSeek-R1-0528-MXFP4-v2/gate_up_proj", + "amd/DeepSeek-R1-0528-MXFP4/gate_up_proj", + "amd/Kimi-K2.5-MXFP4/gate_up_proj" + ] }, { "type": "gemm", "args": { - "m": 1048576, - "n": 3072, + "m": 36, + "n": 24576, "k": 1536, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/q_b_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/q_b_proj" + ] }, { "type": "gemm", "args": { - "m": 1048576, - "n": 576, - "k": 7168, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 36, + "n": 24576, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/q_b_proj" + ] }, { "type": "gemm", "args": { - "m": 1048576, - "n": 4096, + "m": 36, + "n": 32768, "k": 512, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/kv_b_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/kv_b_proj" + ] }, { "type": "gemm", "args": { - "m": 1048576, - "n": 7168, - "k": 2048, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 36, + "n": 32768, + "k": 512, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/kv_b_proj" + ] }, { "type": "gemm", "args": { - "m": 1048576, - "n": 2304, + "m": 38, + "n": 576, "k": 7168, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/Kimi-K2.5-MXFP4/kv_a_proj_with_mqa", + "moonshotai/Kimi-K2.5/kv_a_proj_with_mqa", + "nvidia/DeepSeek-R1-0528-FP4-V2/kv_a_proj_with_mqa", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/kv_a_proj_with_mqa", + "nvidia/Kimi-K2.5-NVFP4/kv_a_proj_with_mqa", + "nvidia/Kimi-K2.6-NVFP4/kv_a_proj_with_mqa" + ] }, { "type": "gemm", "args": { - "m": 1048576, - "n": 7168, - "k": 2304, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 38, + "n": 576, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/kv_a_proj_with_mqa" + ] }, { "type": "gemm", "args": { - "m": 98304, - "n": 1536, + "m": 38, + "n": 1152, "k": 7168, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-NVFP4-v2/gate_up_proj" + ] }, { "type": "gemm", "args": { - "m": 98304, - "n": 24576, - "k": 1536, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 38, + "n": 1152, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/gate_up_proj" + ] }, { "type": "gemm", "args": { - "m": 98304, - "n": 576, + "m": 38, + "n": 1536, "k": 7168, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/Kimi-K2.5-MXFP4/q_a_proj", + "moonshotai/Kimi-K2.5/q_a_proj", + "nvidia/DeepSeek-R1-0528-FP4-V2/q_a_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/q_a_proj", + "nvidia/Kimi-K2.5-NVFP4/q_a_proj", + "nvidia/Kimi-K2.6-NVFP4/q_a_proj" + ] }, { "type": "gemm", "args": { - "m": 98304, - "n": 32768, - "k": 512, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 38, + "n": 1536, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/gate_up_proj", + "deepseek-ai/DeepSeek-R1-0528/q_a_proj", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w3" + ] }, { "type": "gemm", "args": { - "m": 98304, + "m": 38, "n": 7168, - "k": 16384, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "k": 576, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-NVFP4-v2/down_proj" + ] }, { "type": "gemm", "args": { - "m": 98304, - "n": 2304, - "k": 7168, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 38, + "n": 7168, + "k": 576, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/down_proj" + ] }, { "type": "gemm", "args": { - "m": 98304, + "m": 38, "n": 7168, - "k": 2304, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "k": 1152, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/down_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/down_proj", + "nvidia/Kimi-K2.5-NVFP4/down_proj" + ] }, { "type": "gemm", "args": { - "m": 786432, - "n": 1536, - "k": 7168, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 38, + "n": 7168, + "k": 1152, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/down_proj" + ] }, { "type": "gemm", "args": { - "m": 786432, - "n": 3072, - "k": 1536, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 38, + "n": 7168, + "k": 8192, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/Kimi-K2.5-MXFP4/o_proj", + "moonshotai/Kimi-K2.5/o_proj", + "nvidia/Kimi-K2.5-NVFP4/o_proj", + "nvidia/Kimi-K2.6-NVFP4/o_proj" + ] }, { "type": "gemm", "args": { - "m": 786432, - "n": 576, - "k": 7168, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 38, + "n": 7168, + "k": 16384, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/o_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/o_proj" + ] }, { "type": "gemm", "args": { - "m": 786432, - "n": 4096, - "k": 512, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 38, + "n": 7168, + "k": 16384, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/o_proj", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] }, { "type": "gemm", "args": { - "m": 786432, - "n": 7168, - "k": 2048, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 38, + "n": 12288, + "k": 1536, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/Kimi-K2.5-MXFP4/q_b_proj", + "moonshotai/Kimi-K2.5/q_b_proj", + "nvidia/DeepSeek-R1-0528-FP4-V2/q_b_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/q_b_proj", + "nvidia/Kimi-K2.5-NVFP4/q_b_proj", + "nvidia/Kimi-K2.6-NVFP4/q_b_proj" + ] }, { "type": "gemm", "args": { - "m": 786432, - "n": 2304, - "k": 7168, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 38, + "n": 16384, + "k": 512, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/Kimi-K2.5-MXFP4/kv_b_proj", + "moonshotai/Kimi-K2.5/kv_b_proj", + "nvidia/DeepSeek-R1-0528-FP4-V2/kv_b_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/kv_b_proj", + "nvidia/Kimi-K2.5-NVFP4/kv_b_proj", + "nvidia/Kimi-K2.6-NVFP4/kv_b_proj" + ] }, { "type": "gemm", "args": { - "m": 786432, - "n": 7168, - "k": 2304, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 38, + "n": 24576, + "k": 1536, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/q_b_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/q_b_proj" + ] }, { "type": "gemm", "args": { - "m": 262144, + "m": 38, "n": 24576, "k": 1536, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/q_b_proj" + ] }, { "type": "gemm", "args": { - "m": 262144, + "m": 38, "n": 32768, "k": 512, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 262144, - "n": 7168, - "k": 16384, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 32, - "n": 24576, - "k": 1536, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/kv_b_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/kv_b_proj" + ] }, { "type": "gemm", "args": { - "m": 32, + "m": 38, "n": 32768, "k": 512, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/kv_b_proj" + ] }, { "type": "gemm", "args": { - "m": 32, - "n": 7168, - "k": 16384, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 39, + "n": 576, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/Kimi-K2.5-MXFP4/kv_a_proj_with_mqa", + "moonshotai/Kimi-K2.5/kv_a_proj_with_mqa", + "nvidia/DeepSeek-R1-0528-FP4-V2/kv_a_proj_with_mqa", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/kv_a_proj_with_mqa", + "nvidia/Kimi-K2.5-NVFP4/kv_a_proj_with_mqa", + "nvidia/Kimi-K2.6-NVFP4/kv_a_proj_with_mqa" + ] }, { "type": "gemm", "args": { - "m": 655360, + "m": 39, "n": 1536, "k": 7168, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/Kimi-K2.5-MXFP4/q_a_proj", + "moonshotai/Kimi-K2.5/q_a_proj", + "nvidia/DeepSeek-R1-0528-FP4-V2/q_a_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/q_a_proj", + "nvidia/Kimi-K2.5-NVFP4/q_a_proj", + "nvidia/Kimi-K2.6-NVFP4/q_a_proj" + ] }, { "type": "gemm", "args": { - "m": 655360, - "n": 24576, - "k": 1536, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 39, + "n": 4608, + "k": 7168, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/gate_up_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/gate_up_proj", + "nvidia/Kimi-K2.5-NVFP4/gate_up_proj" + ] }, { "type": "gemm", "args": { - "m": 655360, - "n": 576, - "k": 7168, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 39, + "n": 7168, + "k": 4608, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/down_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/down_proj", + "nvidia/Kimi-K2.5-NVFP4/down_proj" + ] }, { "type": "gemm", "args": { - "m": 655360, - "n": 32768, - "k": 512, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 39, + "n": 7168, + "k": 8192, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/Kimi-K2.5-MXFP4/o_proj", + "moonshotai/Kimi-K2.5/o_proj", + "nvidia/Kimi-K2.5-NVFP4/o_proj", + "nvidia/Kimi-K2.6-NVFP4/o_proj" + ] }, { "type": "gemm", "args": { - "m": 655360, + "m": 39, "n": 7168, - "k": 16384, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "k": 8192, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] }, { "type": "gemm", "args": { - "m": 655360, - "n": 2304, - "k": 7168, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 39, + "n": 12288, + "k": 1536, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/Kimi-K2.5-MXFP4/q_b_proj", + "moonshotai/Kimi-K2.5/q_b_proj", + "nvidia/DeepSeek-R1-0528-FP4-V2/q_b_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/q_b_proj", + "nvidia/Kimi-K2.5-NVFP4/q_b_proj", + "nvidia/Kimi-K2.6-NVFP4/q_b_proj" + ] }, { "type": "gemm", "args": { - "m": 655360, - "n": 7168, - "k": 2304, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 39, + "n": 16384, + "k": 512, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/Kimi-K2.5-MXFP4/kv_b_proj", + "moonshotai/Kimi-K2.5/kv_b_proj", + "nvidia/DeepSeek-R1-0528-FP4-V2/kv_b_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/kv_b_proj", + "nvidia/Kimi-K2.5-NVFP4/kv_b_proj", + "nvidia/Kimi-K2.6-NVFP4/kv_b_proj" + ] }, { "type": "gemm", "args": { - "m": 80, - "n": 1536, + "m": 40, + "n": 576, "k": 7168, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/kv_a_proj_with_mqa" + ] }, { "type": "gemm", "args": { - "m": 80, - "n": 24576, - "k": 1536, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 40, + "n": 1536, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/gate_up_proj", + "deepseek-ai/DeepSeek-R1-0528/q_a_proj", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w3" + ] }, { "type": "gemm", "args": { - "m": 80, - "n": 576, + "m": 40, + "n": 2304, "k": 7168, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/gate_up_proj" + ] }, { "type": "gemm", "args": { - "m": 80, - "n": 32768, - "k": 512, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 40, + "n": 3072, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "MiniMaxAI/MiniMax-M2.5/experts.down_proj", + "MiniMaxAI/MiniMax-M2.5/o_proj", + "deepseek-ai/DeepSeek-R1-0528/q_b_proj" + ] }, { "type": "gemm", "args": { - "m": 80, - "n": 7168, - "k": 16384, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 40, + "n": 4096, + "k": 512, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "Qwen/Qwen3.5-397B-A17B-FP8/o_proj", + "Qwen/Qwen3.5-397B-A17B-FP8/shared_expert.down_proj", + "deepseek-ai/DeepSeek-R1-0528/kv_b_proj" + ] }, { "type": "gemm", "args": { - "m": 80, - "n": 2304, - "k": 7168, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 40, + "n": 7168, + "k": 2048, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/o_proj", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] }, { "type": "gemm", "args": { - "m": 80, + "m": 40, "n": 7168, "k": 2304, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/down_proj" + ] }, { "type": "gemm", "args": { - "m": 334848, - "n": 1536, - "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 40, + "n": 7168, + "k": 16384, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/o_proj", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] }, { "type": "gemm", "args": { - "m": 334848, + "m": 40, "n": 24576, "k": 1536, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 334848, - "n": 576, - "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/q_b_proj" + ] }, { "type": "gemm", "args": { - "m": 334848, + "m": 40, "n": 32768, "k": 512, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/kv_b_proj" + ] }, { "type": "gemm", "args": { - "m": 334848, - "n": 7168, - "k": 16384, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 41, + "n": 576, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/Kimi-K2.5-MXFP4/kv_a_proj_with_mqa", + "moonshotai/Kimi-K2.5/kv_a_proj_with_mqa", + "nvidia/DeepSeek-R1-0528-FP4-V2/kv_a_proj_with_mqa", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/kv_a_proj_with_mqa", + "nvidia/Kimi-K2.5-NVFP4/kv_a_proj_with_mqa", + "nvidia/Kimi-K2.6-NVFP4/kv_a_proj_with_mqa" + ] }, { "type": "gemm", "args": { - "m": 334848, - "n": 9216, + "m": 41, + "n": 576, "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/kv_a_proj_with_mqa" + ] }, { "type": "gemm", "args": { - "m": 334848, - "n": 7168, - "k": 9216, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 41, + "n": 1152, + "k": 7168, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-NVFP4-v2/gate_up_proj" + ] }, { "type": "gemm", "args": { - "m": 654, + "m": 41, "n": 1536, "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/Kimi-K2.5-MXFP4/q_a_proj", + "moonshotai/Kimi-K2.5/q_a_proj", + "nvidia/DeepSeek-R1-0528-FP4-V2/q_a_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/q_a_proj", + "nvidia/Kimi-K2.5-NVFP4/q_a_proj", + "nvidia/Kimi-K2.6-NVFP4/q_a_proj" + ] }, { "type": "gemm", "args": { - "m": 654, - "n": 24576, - "k": 1536, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 41, + "n": 1536, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/gate_up_proj", + "deepseek-ai/DeepSeek-R1-0528/q_a_proj", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w3" + ] }, { "type": "gemm", "args": { - "m": 654, - "n": 576, + "m": 41, + "n": 2304, "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/gate_up_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/gate_up_proj", + "nvidia/Kimi-K2.5-NVFP4/gate_up_proj" + ] }, { "type": "gemm", "args": { - "m": 654, - "n": 32768, - "k": 512, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 654, - "n": 7168, - "k": 16384, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 654, - "n": 9216, + "m": 41, + "n": 2304, "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/gate_up_proj" + ] }, { "type": "gemm", "args": { - "m": 654, + "m": 41, "n": 7168, - "k": 9216, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "k": 576, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-NVFP4-v2/down_proj" + ] }, { "type": "gemm", "args": { - "m": 83712, - "n": 1536, - "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 41, + "n": 7168, + "k": 1152, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/down_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/down_proj", + "nvidia/Kimi-K2.5-NVFP4/down_proj" + ] }, { "type": "gemm", "args": { - "m": 83712, - "n": 24576, - "k": 1536, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 41, + "n": 7168, + "k": 1152, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/down_proj" + ] }, { "type": "gemm", "args": { - "m": 83712, - "n": 576, - "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 41, + "n": 7168, + "k": 2304, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/down_proj" + ] }, { "type": "gemm", "args": { - "m": 83712, - "n": 32768, - "k": 512, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 41, + "n": 7168, + "k": 8192, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/Kimi-K2.5-MXFP4/o_proj", + "moonshotai/Kimi-K2.5/o_proj", + "nvidia/Kimi-K2.5-NVFP4/o_proj", + "nvidia/Kimi-K2.6-NVFP4/o_proj" + ] }, { "type": "gemm", "args": { - "m": 83712, + "m": 41, "n": 7168, "k": 16384, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/o_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/o_proj" + ] }, { "type": "gemm", "args": { - "m": 83712, - "n": 2304, - "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 41, + "n": 7168, + "k": 16384, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/o_proj", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] }, { "type": "gemm", "args": { - "m": 83712, - "n": 7168, - "k": 2304, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 41, + "n": 12288, + "k": 1536, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/Kimi-K2.5-MXFP4/q_b_proj", + "moonshotai/Kimi-K2.5/q_b_proj", + "nvidia/DeepSeek-R1-0528-FP4-V2/q_b_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/q_b_proj", + "nvidia/Kimi-K2.5-NVFP4/q_b_proj", + "nvidia/Kimi-K2.6-NVFP4/q_b_proj" + ] }, { "type": "gemm", "args": { - "m": 163, - "n": 1536, - "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 41, + "n": 16384, + "k": 512, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/Kimi-K2.5-MXFP4/kv_b_proj", + "moonshotai/Kimi-K2.5/kv_b_proj", + "nvidia/DeepSeek-R1-0528-FP4-V2/kv_b_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/kv_b_proj", + "nvidia/Kimi-K2.5-NVFP4/kv_b_proj", + "nvidia/Kimi-K2.6-NVFP4/kv_b_proj" + ] }, { "type": "gemm", "args": { - "m": 163, + "m": 41, "n": 24576, "k": 1536, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/q_b_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/q_b_proj" + ] }, { "type": "gemm", "args": { - "m": 163, - "n": 576, - "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 41, + "n": 24576, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/q_b_proj" + ] }, { "type": "gemm", "args": { - "m": 163, + "m": 41, "n": 32768, "k": 512, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/kv_b_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/kv_b_proj" + ] }, { "type": "gemm", "args": { - "m": 163, - "n": 7168, - "k": 16384, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 41, + "n": 32768, + "k": 512, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/kv_b_proj" + ] }, { "type": "gemm", "args": { - "m": 163, - "n": 2304, + "m": 42, + "n": 384, "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [] }, { "type": "gemm", "args": { - "m": 163, - "n": 7168, - "k": 2304, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 42, + "n": 384, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w3" + ] }, { "type": "gemm", "args": { - "m": 138752, - "n": 1536, + "m": 42, + "n": 576, "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/Kimi-K2.5-MXFP4/kv_a_proj_with_mqa", + "moonshotai/Kimi-K2.5/kv_a_proj_with_mqa", + "nvidia/DeepSeek-R1-0528-FP4-V2/kv_a_proj_with_mqa", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/kv_a_proj_with_mqa", + "nvidia/Kimi-K2.5-NVFP4/kv_a_proj_with_mqa", + "nvidia/Kimi-K2.6-NVFP4/kv_a_proj_with_mqa" + ] }, { "type": "gemm", "args": { - "m": 138752, - "n": 24576, - "k": 1536, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 42, + "n": 576, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/kv_a_proj_with_mqa" + ] }, { "type": "gemm", "args": { - "m": 138752, - "n": 576, + "m": 42, + "n": 1536, "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/Kimi-K2.5-MXFP4/q_a_proj", + "moonshotai/Kimi-K2.5/q_a_proj", + "nvidia/DeepSeek-R1-0528-FP4-V2/q_a_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/q_a_proj", + "nvidia/Kimi-K2.5-NVFP4/q_a_proj", + "nvidia/Kimi-K2.6-NVFP4/q_a_proj" + ] }, { "type": "gemm", "args": { - "m": 138752, - "n": 32768, - "k": 512, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 42, + "n": 1536, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/gate_up_proj", + "deepseek-ai/DeepSeek-R1-0528/q_a_proj", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w3" + ] }, { "type": "gemm", "args": { - "m": 138752, + "m": 42, "n": 7168, - "k": 16384, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "k": 384, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-NVFP4-v2/down_proj" + ] }, { "type": "gemm", "args": { - "m": 138752, - "n": 9216, - "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 42, + "n": 7168, + "k": 384, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w2" + ] }, { "type": "gemm", "args": { - "m": 138752, + "m": 42, "n": 7168, - "k": 9216, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "k": 16384, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/o_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/o_proj" + ] }, { "type": "gemm", "args": { - "m": 271, - "n": 1536, - "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 42, + "n": 7168, + "k": 16384, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/o_proj", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] }, { "type": "gemm", "args": { - "m": 271, + "m": 42, "n": 24576, "k": 1536, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/q_b_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/q_b_proj" + ] }, { "type": "gemm", "args": { - "m": 271, - "n": 576, - "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 42, + "n": 24576, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/q_b_proj" + ] }, { "type": "gemm", "args": { - "m": 271, + "m": 42, "n": 32768, "k": 512, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/kv_b_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/kv_b_proj" + ] }, { "type": "gemm", "args": { - "m": 271, - "n": 7168, - "k": 16384, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 42, + "n": 32768, + "k": 512, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/kv_b_proj" + ] }, { "type": "gemm", "args": { - "m": 271, - "n": 9216, + "m": 44, + "n": 576, "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/Kimi-K2.5-MXFP4/kv_a_proj_with_mqa", + "moonshotai/Kimi-K2.5/kv_a_proj_with_mqa", + "nvidia/DeepSeek-R1-0528-FP4-V2/kv_a_proj_with_mqa", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/kv_a_proj_with_mqa", + "nvidia/Kimi-K2.5-NVFP4/kv_a_proj_with_mqa", + "nvidia/Kimi-K2.6-NVFP4/kv_a_proj_with_mqa" + ] }, { "type": "gemm", "args": { - "m": 271, - "n": 7168, - "k": 9216, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 44, + "n": 1536, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/Kimi-K2.5-MXFP4/q_a_proj", + "moonshotai/Kimi-K2.5/q_a_proj", + "nvidia/DeepSeek-R1-0528-FP4-V2/q_a_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/q_a_proj", + "nvidia/Kimi-K2.5-NVFP4/q_a_proj", + "nvidia/Kimi-K2.6-NVFP4/q_a_proj" + ] }, { "type": "gemm", "args": { - "m": 34688, - "n": 1536, + "m": 44, + "n": 2304, "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/gate_up_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/gate_up_proj", + "nvidia/Kimi-K2.5-NVFP4/gate_up_proj" + ] }, { "type": "gemm", "args": { - "m": 34688, - "n": 24576, + "m": 44, + "n": 3072, "k": 1536, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/Kimi-K2.5-MXFP4/q_b_proj", + "amd/MiniMax-M2.5-MXFP4/o_proj", + "moonshotai/Kimi-K2.5/q_b_proj", + "nvidia/DeepSeek-R1-0528-FP4-V2/q_b_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/q_b_proj", + "nvidia/Kimi-K2.5-NVFP4/q_b_proj", + "nvidia/Kimi-K2.6-NVFP4/q_b_proj", + "nvidia/MiniMax-M2.5-NVFP4/o_proj" + ] }, { "type": "gemm", "args": { - "m": 34688, - "n": 576, - "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 44, + "n": 4096, + "k": 512, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "Qwen/Qwen3.5-397B-A17B/o_proj", + "Qwen/Qwen3.5-397B-A17B/shared_expert.down_proj", + "amd/Kimi-K2.5-MXFP4/kv_b_proj", + "amd/Qwen3.5-397B-A17B-MXFP4/o_proj", + "amd/Qwen3.5-397B-A17B-MXFP4/shared_expert.down_proj", + "moonshotai/Kimi-K2.5/kv_b_proj", + "nvidia/DeepSeek-R1-0528-FP4-V2/kv_b_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/kv_b_proj", + "nvidia/Kimi-K2.5-NVFP4/kv_b_proj", + "nvidia/Kimi-K2.6-NVFP4/kv_b_proj", + "nvidia/Qwen3.5-397B-A17B-NVFP4/o_proj", + "nvidia/Qwen3.5-397B-A17B-NVFP4/shared_expert.down_proj" + ] }, { "type": "gemm", "args": { - "m": 34688, - "n": 32768, - "k": 512, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 44, + "n": 7168, + "k": 2048, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/o_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/o_proj" + ] }, { "type": "gemm", "args": { - "m": 34688, + "m": 44, "n": 7168, - "k": 16384, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "k": 2304, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/down_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/down_proj", + "nvidia/Kimi-K2.5-NVFP4/down_proj" + ] }, { "type": "gemm", "args": { - "m": 34688, - "n": 2304, + "m": 45, + "n": 576, "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/Kimi-K2.5-MXFP4/kv_a_proj_with_mqa", + "moonshotai/Kimi-K2.5/kv_a_proj_with_mqa", + "nvidia/DeepSeek-R1-0528-FP4-V2/kv_a_proj_with_mqa", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/kv_a_proj_with_mqa", + "nvidia/Kimi-K2.5-NVFP4/kv_a_proj_with_mqa", + "nvidia/Kimi-K2.6-NVFP4/kv_a_proj_with_mqa" + ] }, { "type": "gemm", "args": { - "m": 34688, - "n": 7168, - "k": 2304, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 45, + "n": 1536, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/Kimi-K2.5-MXFP4/q_a_proj", + "moonshotai/Kimi-K2.5/q_a_proj", + "nvidia/DeepSeek-R1-0528-FP4-V2/q_a_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/q_a_proj", + "nvidia/Kimi-K2.5-NVFP4/q_a_proj", + "nvidia/Kimi-K2.6-NVFP4/q_a_proj" + ] }, { "type": "gemm", "args": { - "m": 67, - "n": 1536, + "m": 45, + "n": 2304, "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/gate_up_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/gate_up_proj", + "nvidia/Kimi-K2.5-NVFP4/gate_up_proj" + ] }, { "type": "gemm", "args": { - "m": 67, - "n": 24576, + "m": 45, + "n": 3072, "k": 1536, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/Kimi-K2.5-MXFP4/q_b_proj", + "amd/MiniMax-M2.5-MXFP4/o_proj", + "moonshotai/Kimi-K2.5/q_b_proj", + "nvidia/DeepSeek-R1-0528-FP4-V2/q_b_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/q_b_proj", + "nvidia/Kimi-K2.5-NVFP4/q_b_proj", + "nvidia/Kimi-K2.6-NVFP4/q_b_proj", + "nvidia/MiniMax-M2.5-NVFP4/o_proj" + ] }, { "type": "gemm", "args": { - "m": 67, - "n": 576, + "m": 45, + "n": 4096, + "k": 512, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "Qwen/Qwen3.5-397B-A17B/o_proj", + "Qwen/Qwen3.5-397B-A17B/shared_expert.down_proj", + "amd/Kimi-K2.5-MXFP4/kv_b_proj", + "amd/Qwen3.5-397B-A17B-MXFP4/o_proj", + "amd/Qwen3.5-397B-A17B-MXFP4/shared_expert.down_proj", + "moonshotai/Kimi-K2.5/kv_b_proj", + "nvidia/DeepSeek-R1-0528-FP4-V2/kv_b_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/kv_b_proj", + "nvidia/Kimi-K2.5-NVFP4/kv_b_proj", + "nvidia/Kimi-K2.6-NVFP4/kv_b_proj", + "nvidia/Qwen3.5-397B-A17B-NVFP4/o_proj", + "nvidia/Qwen3.5-397B-A17B-NVFP4/shared_expert.down_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 45, + "n": 4608, "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/gate_up_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/gate_up_proj", + "nvidia/Kimi-K2.5-NVFP4/gate_up_proj" + ] }, { "type": "gemm", "args": { - "m": 67, - "n": 32768, - "k": 512, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 45, + "n": 7168, + "k": 2048, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/o_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/o_proj" + ] }, { "type": "gemm", "args": { - "m": 67, + "m": 45, "n": 7168, - "k": 16384, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "k": 2304, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/down_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/down_proj", + "nvidia/Kimi-K2.5-NVFP4/down_proj" + ] }, { "type": "gemm", "args": { - "m": 67, - "n": 2304, - "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 45, + "n": 7168, + "k": 4608, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/down_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/down_proj", + "nvidia/Kimi-K2.5-NVFP4/down_proj" + ] }, { "type": "gemm", "args": { - "m": 67, + "m": 45, "n": 7168, - "k": 2304, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "k": 4608, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 32 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 1, + 32 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "amd/DeepSeek-R1-0528-MXFP4-Preview/down_proj", + "amd/DeepSeek-R1-0528-MXFP4-v2/down_proj", + "amd/DeepSeek-R1-0528-MXFP4/down_proj", + "amd/Kimi-K2.5-MXFP4/down_proj" + ] }, { "type": "gemm", "args": { - "m": 5632, - "n": 1536, - "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 45, + "n": 7168, + "k": 16384, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/o_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/o_proj" + ] }, { "type": "gemm", "args": { - "m": 5632, + "m": 45, "n": 24576, "k": 1536, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/q_b_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/q_b_proj" + ] }, { "type": "gemm", "args": { - "m": 5632, - "n": 576, - "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 45, + "n": 32768, + "k": 512, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/kv_b_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/kv_b_proj" + ] }, { "type": "gemm", "args": { - "m": 5632, - "n": 32768, - "k": 512, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 48, + "n": 576, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/Kimi-K2.5-MXFP4/kv_a_proj_with_mqa", + "moonshotai/Kimi-K2.5/kv_a_proj_with_mqa", + "nvidia/DeepSeek-R1-0528-FP4-V2/kv_a_proj_with_mqa", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/kv_a_proj_with_mqa", + "nvidia/Kimi-K2.5-NVFP4/kv_a_proj_with_mqa", + "nvidia/Kimi-K2.6-NVFP4/kv_a_proj_with_mqa" + ] }, { "type": "gemm", "args": { - "m": 5632, - "n": 7168, - "k": 16384, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 48, + "n": 576, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/kv_a_proj_with_mqa" + ] }, { "type": "gemm", "args": { - "m": 5632, - "n": 9216, + "m": 48, + "n": 1088, "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] }, { "type": "gemm", "args": { - "m": 5632, - "n": 7168, - "k": 9216, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 48, + "n": 1536, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/Kimi-K2.5-MXFP4/q_a_proj", + "moonshotai/Kimi-K2.5/q_a_proj", + "nvidia/DeepSeek-R1-0528-FP4-V2/q_a_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/q_a_proj", + "nvidia/Kimi-K2.5-NVFP4/q_a_proj", + "nvidia/Kimi-K2.6-NVFP4/q_a_proj" + ] }, { "type": "gemm", "args": { - "m": 11, - "n": 9216, + "m": 48, + "n": 1536, "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/gate_up_proj", + "deepseek-ai/DeepSeek-R1-0528/q_a_proj", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w3" + ] }, { "type": "gemm", "args": { - "m": 11, - "n": 7168, - "k": 9216, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 48, + "n": 2304, + "k": 7168, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/gate_up_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/gate_up_proj", + "nvidia/Kimi-K2.5-NVFP4/gate_up_proj" + ] }, { "type": "gemm", "args": { - "m": 11264, - "n": 1536, + "m": 48, + "n": 2304, "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/gate_up_proj" + ] }, { "type": "gemm", "args": { - "m": 11264, + "m": 48, "n": 3072, "k": 1536, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/Kimi-K2.5-MXFP4/q_b_proj", + "amd/MiniMax-M2.5-MXFP4/o_proj", + "moonshotai/Kimi-K2.5/q_b_proj", + "nvidia/DeepSeek-R1-0528-FP4-V2/q_b_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/q_b_proj", + "nvidia/Kimi-K2.5-NVFP4/q_b_proj", + "nvidia/Kimi-K2.6-NVFP4/q_b_proj", + "nvidia/MiniMax-M2.5-NVFP4/o_proj" + ] }, { "type": "gemm", "args": { - "m": 11264, - "n": 576, - "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 48, + "n": 3072, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "MiniMaxAI/MiniMax-M2.5/experts.down_proj", + "MiniMaxAI/MiniMax-M2.5/o_proj", + "deepseek-ai/DeepSeek-R1-0528/q_b_proj" + ] }, { "type": "gemm", "args": { - "m": 11264, + "m": 48, "n": 4096, "k": 512, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "Qwen/Qwen3.5-397B-A17B/o_proj", + "Qwen/Qwen3.5-397B-A17B/shared_expert.down_proj", + "amd/Kimi-K2.5-MXFP4/kv_b_proj", + "amd/Qwen3.5-397B-A17B-MXFP4/o_proj", + "amd/Qwen3.5-397B-A17B-MXFP4/shared_expert.down_proj", + "moonshotai/Kimi-K2.5/kv_b_proj", + "nvidia/DeepSeek-R1-0528-FP4-V2/kv_b_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/kv_b_proj", + "nvidia/Kimi-K2.5-NVFP4/kv_b_proj", + "nvidia/Kimi-K2.6-NVFP4/kv_b_proj", + "nvidia/Qwen3.5-397B-A17B-NVFP4/o_proj", + "nvidia/Qwen3.5-397B-A17B-NVFP4/shared_expert.down_proj" + ] }, { "type": "gemm", "args": { - "m": 11264, - "n": 7168, - "k": 2048, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 48, + "n": 4096, + "k": 512, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "Qwen/Qwen3.5-397B-A17B-FP8/o_proj", + "Qwen/Qwen3.5-397B-A17B-FP8/shared_expert.down_proj", + "deepseek-ai/DeepSeek-R1-0528/kv_b_proj" + ] }, { "type": "gemm", "args": { - "m": 11264, - "n": 2304, + "m": 48, + "n": 4608, "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/gate_up_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/gate_up_proj", + "nvidia/Kimi-K2.5-NVFP4/gate_up_proj" + ] }, { "type": "gemm", "args": { - "m": 11264, - "n": 7168, - "k": 2304, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 48, + "n": 4608, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/gate_up_proj" + ] }, { "type": "gemm", "args": { - "m": 22, - "n": 3072, + "m": 48, + "n": 6144, "k": 1536, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/GLM-5.1-MXFP4/down_proj", + "amd/GLM-5.2-MXFP4/down_proj", + "amd/Kimi-K2.5-MXFP4/q_b_proj", + "amd/MiniMax-M3-MXFP4/down_proj", + "moonshotai/Kimi-K2.5/q_b_proj", + "nvidia/DeepSeek-R1-0528-FP4-V2/q_b_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/q_b_proj", + "nvidia/GLM-5.1-NVFP4/down_proj", + "nvidia/GLM-5.2-NVFP4/down_proj", + "nvidia/Kimi-K2.5-NVFP4/q_b_proj", + "nvidia/Kimi-K2.6-NVFP4/q_b_proj" + ] }, { "type": "gemm", "args": { - "m": 22, - "n": 4096, - "k": 512, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 48, + "n": 7168, + "k": 2048, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/o_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/o_proj" + ] }, { "type": "gemm", "args": { - "m": 22, + "m": 48, "n": 7168, "k": 2048, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/o_proj", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] }, { "type": "gemm", "args": { - "m": 5120, - "n": 1536, - "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 48, + "n": 7168, + "k": 2304, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/down_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/down_proj", + "nvidia/Kimi-K2.5-NVFP4/down_proj" + ] }, { "type": "gemm", "args": { - "m": 5120, - "n": 24576, - "k": 1536, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 48, + "n": 7168, + "k": 2304, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/down_proj" + ] }, { "type": "gemm", "args": { - "m": 5120, - "n": 576, - "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 48, + "n": 7168, + "k": 4096, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/o_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/o_proj" + ] }, { "type": "gemm", "args": { - "m": 5120, - "n": 32768, - "k": 512, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 48, + "n": 7168, + "k": 4608, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/down_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/down_proj", + "nvidia/Kimi-K2.5-NVFP4/down_proj" + ] }, { "type": "gemm", "args": { - "m": 5120, + "m": 48, "n": 7168, - "k": 16384, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "k": 8192, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/Kimi-K2.5-MXFP4/o_proj", + "moonshotai/Kimi-K2.5/o_proj", + "nvidia/Kimi-K2.5-NVFP4/o_proj", + "nvidia/Kimi-K2.6-NVFP4/o_proj" + ] }, { "type": "gemm", "args": { - "m": 5120, - "n": 9216, - "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 48, + "n": 7168, + "k": 8192, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] }, { "type": "gemm", "args": { - "m": 5120, + "m": 48, "n": 7168, "k": 9216, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/down_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/down_proj", + "nvidia/Kimi-K2.5-NVFP4/down_proj" + ] }, { "type": "gemm", "args": { - "m": 10, - "n": 24576, - "k": 1536, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 48, + "n": 7168, + "k": 16384, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/o_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/o_proj" + ] }, { "type": "gemm", "args": { - "m": 10, - "n": 32768, - "k": 512, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 48, + "n": 7168, + "k": 16384, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 32 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 1, + 32 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "amd/DeepSeek-R1-0528-MXFP4-Preview/o_proj", + "amd/DeepSeek-R1-0528-MXFP4-v2/o_proj", + "amd/DeepSeek-R1-0528-MXFP4/o_proj" + ] }, { "type": "gemm", "args": { - "m": 10, + "m": 48, "n": 7168, "k": 16384, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] }, { "type": "gemm", "args": { - "m": 10, - "n": 9216, - "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 48, + "n": 7168, + "k": 65536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] }, { "type": "gemm", "args": { - "m": 10, - "n": 7168, - "k": 9216, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 48, + "n": 8192, + "k": 512, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/Kimi-K2.5-MXFP4/kv_b_proj", + "moonshotai/Kimi-K2.5/kv_b_proj", + "nvidia/DeepSeek-R1-0528-FP4-V2/kv_b_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/kv_b_proj", + "nvidia/Kimi-K2.5-NVFP4/kv_b_proj", + "nvidia/Kimi-K2.6-NVFP4/kv_b_proj" + ] }, { "type": "gemm", "args": { - "m": 10240, - "n": 24576, - "k": 1536, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 48, + "n": 9216, + "k": 7168, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/gate_up_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/gate_up_proj", + "nvidia/Kimi-K2.5-NVFP4/gate_up_proj" + ] }, { "type": "gemm", "args": { - "m": 10240, - "n": 32768, - "k": 512, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 48, + "n": 12288, + "k": 1536, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/Kimi-K2.5-MXFP4/q_b_proj", + "moonshotai/Kimi-K2.5/q_b_proj", + "nvidia/DeepSeek-R1-0528-FP4-V2/q_b_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/q_b_proj", + "nvidia/Kimi-K2.5-NVFP4/q_b_proj", + "nvidia/Kimi-K2.6-NVFP4/q_b_proj" + ] }, { "type": "gemm", "args": { - "m": 10240, - "n": 7168, - "k": 16384, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 48, + "n": 16384, + "k": 512, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/Kimi-K2.5-MXFP4/kv_b_proj", + "moonshotai/Kimi-K2.5/kv_b_proj", + "nvidia/DeepSeek-R1-0528-FP4-V2/kv_b_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/kv_b_proj", + "nvidia/Kimi-K2.5-NVFP4/kv_b_proj", + "nvidia/Kimi-K2.6-NVFP4/kv_b_proj" + ] }, { "type": "gemm", "args": { - "m": 10240, - "n": 9216, - "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 48, + "n": 24576, + "k": 1536, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/q_b_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/q_b_proj" + ] }, { "type": "gemm", "args": { - "m": 10240, - "n": 7168, - "k": 9216, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 48, + "n": 24576, + "k": 1536, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 32 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 1, + 32 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "amd/DeepSeek-R1-0528-MXFP4-Preview/q_b_proj", + "amd/DeepSeek-R1-0528-MXFP4-v2/q_b_proj", + "amd/DeepSeek-R1-0528-MXFP4/q_b_proj" + ] }, { "type": "gemm", "args": { - "m": 20, - "n": 24576, - "k": 1536, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 48, + "n": 32768, + "k": 512, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/kv_b_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/kv_b_proj" + ] }, { "type": "gemm", "args": { - "m": 20, + "m": 48, "n": 32768, "k": 512, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 32 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 1, + 32 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "amd/DeepSeek-R1-0528-MXFP4-Preview/kv_b_proj", + "amd/DeepSeek-R1-0528-MXFP4-v2/kv_b_proj", + "amd/DeepSeek-R1-0528-MXFP4/kv_b_proj" + ] }, { "type": "gemm", "args": { - "m": 20, - "n": 7168, - "k": 16384, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 48, + "n": 65536, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_b" + ] }, { "type": "gemm", "args": { - "m": 20, - "n": 9216, - "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 48, + "n": 122880, + "k": 1024, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] }, { "type": "gemm", "args": { - "m": 20, - "n": 7168, - "k": 9216, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 50, + "n": 576, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/Kimi-K2.5-MXFP4/kv_a_proj_with_mqa", + "moonshotai/Kimi-K2.5/kv_a_proj_with_mqa", + "nvidia/DeepSeek-R1-0528-FP4-V2/kv_a_proj_with_mqa", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/kv_a_proj_with_mqa", + "nvidia/Kimi-K2.5-NVFP4/kv_a_proj_with_mqa", + "nvidia/Kimi-K2.6-NVFP4/kv_a_proj_with_mqa" + ] }, { "type": "gemm", "args": { - "m": 12800, + "m": 50, "n": 1536, "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/Kimi-K2.5-MXFP4/q_a_proj", + "moonshotai/Kimi-K2.5/q_a_proj", + "nvidia/DeepSeek-R1-0528-FP4-V2/q_a_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/q_a_proj", + "nvidia/Kimi-K2.5-NVFP4/q_a_proj", + "nvidia/Kimi-K2.6-NVFP4/q_a_proj" + ] }, { "type": "gemm", "args": { - "m": 12800, - "n": 24576, - "k": 1536, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 50, + "n": 2304, + "k": 7168, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/gate_up_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/gate_up_proj", + "nvidia/Kimi-K2.5-NVFP4/gate_up_proj" + ] }, { "type": "gemm", "args": { - "m": 12800, - "n": 576, - "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 50, + "n": 3072, + "k": 1536, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/Kimi-K2.5-MXFP4/q_b_proj", + "amd/MiniMax-M2.5-MXFP4/o_proj", + "moonshotai/Kimi-K2.5/q_b_proj", + "nvidia/DeepSeek-R1-0528-FP4-V2/q_b_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/q_b_proj", + "nvidia/Kimi-K2.5-NVFP4/q_b_proj", + "nvidia/Kimi-K2.6-NVFP4/q_b_proj", + "nvidia/MiniMax-M2.5-NVFP4/o_proj" + ] }, { "type": "gemm", "args": { - "m": 12800, - "n": 32768, + "m": 50, + "n": 4096, "k": 512, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "Qwen/Qwen3.5-397B-A17B/o_proj", + "Qwen/Qwen3.5-397B-A17B/shared_expert.down_proj", + "amd/Kimi-K2.5-MXFP4/kv_b_proj", + "amd/Qwen3.5-397B-A17B-MXFP4/o_proj", + "amd/Qwen3.5-397B-A17B-MXFP4/shared_expert.down_proj", + "moonshotai/Kimi-K2.5/kv_b_proj", + "nvidia/DeepSeek-R1-0528-FP4-V2/kv_b_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/kv_b_proj", + "nvidia/Kimi-K2.5-NVFP4/kv_b_proj", + "nvidia/Kimi-K2.6-NVFP4/kv_b_proj", + "nvidia/Qwen3.5-397B-A17B-NVFP4/o_proj", + "nvidia/Qwen3.5-397B-A17B-NVFP4/shared_expert.down_proj" + ] }, { "type": "gemm", "args": { - "m": 12800, + "m": 50, "n": 7168, - "k": 16384, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "k": 2048, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/o_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/o_proj" + ] }, { "type": "gemm", "args": { - "m": 12800, - "n": 9216, + "m": 50, + "n": 7168, + "k": 2304, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/down_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/down_proj", + "nvidia/Kimi-K2.5-NVFP4/down_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 52, + "n": 576, "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/Kimi-K2.5-MXFP4/kv_a_proj_with_mqa", + "moonshotai/Kimi-K2.5/kv_a_proj_with_mqa", + "nvidia/DeepSeek-R1-0528-FP4-V2/kv_a_proj_with_mqa", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/kv_a_proj_with_mqa", + "nvidia/Kimi-K2.5-NVFP4/kv_a_proj_with_mqa", + "nvidia/Kimi-K2.6-NVFP4/kv_a_proj_with_mqa" + ] }, { "type": "gemm", "args": { - "m": 12800, - "n": 7168, - "k": 9216, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 52, + "n": 576, + "k": 7168, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 32 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 1, + 32 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "amd/DeepSeek-R1-0528-MXFP4-Preview/kv_a_proj_with_mqa", + "amd/DeepSeek-R1-0528-MXFP4-v2/kv_a_proj_with_mqa", + "amd/DeepSeek-R1-0528-MXFP4/kv_a_proj_with_mqa" + ] }, { "type": "gemm", "args": { - "m": 25, - "n": 9216, + "m": 52, + "n": 1536, "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/Kimi-K2.5-MXFP4/q_a_proj", + "moonshotai/Kimi-K2.5/q_a_proj", + "nvidia/DeepSeek-R1-0528-FP4-V2/q_a_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/q_a_proj", + "nvidia/Kimi-K2.5-NVFP4/q_a_proj", + "nvidia/Kimi-K2.6-NVFP4/q_a_proj" + ] }, { "type": "gemm", "args": { - "m": 25, - "n": 7168, - "k": 9216, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 52, + "n": 1536, + "k": 7168, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 32 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 1, + 32 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "amd/DeepSeek-R1-0528-MXFP4-Preview/q_a_proj", + "amd/DeepSeek-R1-0528-MXFP4-v2/q_a_proj", + "amd/DeepSeek-R1-0528-MXFP4/q_a_proj" + ] }, { "type": "gemm", "args": { - "m": 30720, - "n": 9216, + "m": 52, + "n": 1536, "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w3" + ] }, { "type": "gemm", "args": { - "m": 30720, + "m": 52, "n": 7168, "k": 9216, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/down_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/down_proj", + "nvidia/Kimi-K2.5-NVFP4/down_proj" + ] }, { "type": "gemm", "args": { - "m": 60, - "n": 24576, - "k": 1536, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 52, + "n": 7168, + "k": 16384, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/o_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/o_proj" + ] }, { "type": "gemm", "args": { - "m": 60, - "n": 32768, - "k": 512, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 52, + "n": 7168, + "k": 16384, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 32 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 1, + 32 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "amd/DeepSeek-R1-0528-MXFP4-Preview/o_proj", + "amd/DeepSeek-R1-0528-MXFP4-v2/o_proj", + "amd/DeepSeek-R1-0528-MXFP4/o_proj" + ] }, { "type": "gemm", "args": { - "m": 60, + "m": 52, "n": 7168, "k": 16384, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] }, { "type": "gemm", "args": { - "m": 60, + "m": 52, "n": 9216, "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 60, - "n": 7168, - "k": 9216, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/gate_up_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/gate_up_proj", + "nvidia/Kimi-K2.5-NVFP4/gate_up_proj" + ] }, { "type": "gemm", "args": { - "m": 61440, + "m": 52, "n": 9216, "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 32 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 1, + 32 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "amd/DeepSeek-R1-0528-MXFP4-Preview/gate_up_proj", + "amd/DeepSeek-R1-0528-MXFP4-v2/gate_up_proj", + "amd/DeepSeek-R1-0528-MXFP4/gate_up_proj", + "amd/Kimi-K2.5-MXFP4/gate_up_proj" + ] }, { "type": "gemm", "args": { - "m": 61440, - "n": 7168, - "k": 9216, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 52, + "n": 24576, + "k": 1536, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/q_b_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/q_b_proj" + ] }, { "type": "gemm", "args": { - "m": 120, + "m": 52, "n": 24576, "k": 1536, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 32 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 1, + 32 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "amd/DeepSeek-R1-0528-MXFP4-Preview/q_b_proj", + "amd/DeepSeek-R1-0528-MXFP4-v2/q_b_proj", + "amd/DeepSeek-R1-0528-MXFP4/q_b_proj" + ] }, { "type": "gemm", "args": { - "m": 120, + "m": 52, "n": 32768, "k": 512, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/kv_b_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/kv_b_proj" + ] }, { "type": "gemm", "args": { - "m": 120, - "n": 7168, - "k": 16384, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 120, - "n": 9216, - "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 120, - "n": 7168, - "k": 9216, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 52, + "n": 32768, + "k": 512, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 32 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 1, + 32 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "amd/DeepSeek-R1-0528-MXFP4-Preview/kv_b_proj", + "amd/DeepSeek-R1-0528-MXFP4-v2/kv_b_proj", + "amd/DeepSeek-R1-0528-MXFP4/kv_b_proj" + ] }, { "type": "gemm", "args": { - "m": 102400, - "n": 9216, + "m": 55, + "n": 576, "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 102400, - "n": 7168, - "k": 9216, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/Kimi-K2.5-MXFP4/kv_a_proj_with_mqa", + "moonshotai/Kimi-K2.5/kv_a_proj_with_mqa", + "nvidia/DeepSeek-R1-0528-FP4-V2/kv_a_proj_with_mqa", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/kv_a_proj_with_mqa", + "nvidia/Kimi-K2.5-NVFP4/kv_a_proj_with_mqa", + "nvidia/Kimi-K2.6-NVFP4/kv_a_proj_with_mqa" + ] }, { "type": "gemm", "args": { - "m": 200, - "n": 9216, + "m": 55, + "n": 1536, "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 200, - "n": 7168, - "k": 9216, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/Kimi-K2.5-MXFP4/q_a_proj", + "moonshotai/Kimi-K2.5/q_a_proj", + "nvidia/DeepSeek-R1-0528-FP4-V2/q_a_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/q_a_proj", + "nvidia/Kimi-K2.5-NVFP4/q_a_proj", + "nvidia/Kimi-K2.6-NVFP4/q_a_proj" + ] }, { "type": "gemm", "args": { - "m": 20480, + "m": 55, "n": 3072, "k": 1536, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/Kimi-K2.5-MXFP4/q_b_proj", + "amd/MiniMax-M2.5-MXFP4/o_proj", + "moonshotai/Kimi-K2.5/q_b_proj", + "nvidia/DeepSeek-R1-0528-FP4-V2/q_b_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/q_b_proj", + "nvidia/Kimi-K2.5-NVFP4/q_b_proj", + "nvidia/Kimi-K2.6-NVFP4/q_b_proj", + "nvidia/MiniMax-M2.5-NVFP4/o_proj" + ] }, { "type": "gemm", "args": { - "m": 20480, + "m": 55, "n": 4096, "k": 512, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "Qwen/Qwen3.5-397B-A17B/o_proj", + "Qwen/Qwen3.5-397B-A17B/shared_expert.down_proj", + "amd/Kimi-K2.5-MXFP4/kv_b_proj", + "amd/Qwen3.5-397B-A17B-MXFP4/o_proj", + "amd/Qwen3.5-397B-A17B-MXFP4/shared_expert.down_proj", + "moonshotai/Kimi-K2.5/kv_b_proj", + "nvidia/DeepSeek-R1-0528-FP4-V2/kv_b_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/kv_b_proj", + "nvidia/Kimi-K2.5-NVFP4/kv_b_proj", + "nvidia/Kimi-K2.6-NVFP4/kv_b_proj", + "nvidia/Qwen3.5-397B-A17B-NVFP4/o_proj", + "nvidia/Qwen3.5-397B-A17B-NVFP4/shared_expert.down_proj" + ] }, { "type": "gemm", "args": { - "m": 20480, - "n": 7168, - "k": 2048, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 55, + "n": 4608, + "k": 7168, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/gate_up_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/gate_up_proj", + "nvidia/Kimi-K2.5-NVFP4/gate_up_proj" + ] }, { "type": "gemm", "args": { - "m": 20480, - "n": 2304, + "m": 55, + "n": 4608, "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 32 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 1, + 32 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "amd/DeepSeek-R1-0528-MXFP4-Preview/gate_up_proj", + "amd/DeepSeek-R1-0528-MXFP4-v2/gate_up_proj", + "amd/DeepSeek-R1-0528-MXFP4/gate_up_proj", + "amd/Kimi-K2.5-MXFP4/gate_up_proj" + ] }, { "type": "gemm", "args": { - "m": 20480, + "m": 55, "n": 7168, - "k": 2304, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "k": 2048, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/Kimi-K2.5-MXFP4/o_proj", + "moonshotai/Kimi-K2.5/o_proj", + "nvidia/Kimi-K2.5-NVFP4/o_proj", + "nvidia/Kimi-K2.6-NVFP4/o_proj" + ] }, { "type": "gemm", "args": { - "m": 40, - "n": 1536, - "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 55, + "n": 7168, + "k": 4608, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/down_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/down_proj", + "nvidia/Kimi-K2.5-NVFP4/down_proj" + ] }, { "type": "gemm", "args": { - "m": 40, - "n": 3072, - "k": 1536, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 55, + "n": 7168, + "k": 9216, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/down_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/down_proj", + "nvidia/Kimi-K2.5-NVFP4/down_proj" + ] }, { "type": "gemm", "args": { - "m": 40, - "n": 576, - "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 55, + "n": 7168, + "k": 16384, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/o_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/o_proj" + ] }, { "type": "gemm", "args": { - "m": 40, - "n": 4096, - "k": 512, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 55, + "n": 7168, + "k": 16384, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 32 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 1, + 32 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "amd/DeepSeek-R1-0528-MXFP4-Preview/o_proj", + "amd/DeepSeek-R1-0528-MXFP4-v2/o_proj", + "amd/DeepSeek-R1-0528-MXFP4/o_proj" + ] }, { "type": "gemm", "args": { - "m": 40, + "m": 55, "n": 7168, - "k": 2048, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "k": 16384, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] }, { "type": "gemm", "args": { - "m": 40, - "n": 2304, + "m": 55, + "n": 9216, "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/gate_up_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/gate_up_proj", + "nvidia/Kimi-K2.5-NVFP4/gate_up_proj" + ] }, { "type": "gemm", "args": { - "m": 40, - "n": 7168, - "k": 2304, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 55, + "n": 24576, + "k": 1536, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/q_b_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/q_b_proj" + ] }, { "type": "gemm", "args": { - "m": 61440, - "n": 3072, + "m": 55, + "n": 24576, "k": 1536, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 32 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 1, + 32 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "amd/DeepSeek-R1-0528-MXFP4-Preview/q_b_proj", + "amd/DeepSeek-R1-0528-MXFP4-v2/q_b_proj", + "amd/DeepSeek-R1-0528-MXFP4/q_b_proj" + ] }, { "type": "gemm", "args": { - "m": 61440, - "n": 4096, + "m": 55, + "n": 32768, "k": 512, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/kv_b_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/kv_b_proj" + ] }, { "type": "gemm", "args": { - "m": 61440, - "n": 7168, - "k": 2048, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 55, + "n": 32768, + "k": 512, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 32 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 1, + 32 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "amd/DeepSeek-R1-0528-MXFP4-Preview/kv_b_proj", + "amd/DeepSeek-R1-0528-MXFP4-v2/kv_b_proj", + "amd/DeepSeek-R1-0528-MXFP4/kv_b_proj" + ] }, { "type": "gemm", "args": { - "m": 61440, - "n": 2304, + "m": 56, + "n": 576, "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/kv_a_proj_with_mqa" + ] }, { "type": "gemm", "args": { - "m": 61440, - "n": 7168, - "k": 2304, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 56, + "n": 1536, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/gate_up_proj", + "deepseek-ai/DeepSeek-R1-0528/q_a_proj", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w3" + ] }, { "type": "gemm", "args": { - "m": 240, - "n": 1536, + "m": 56, + "n": 2304, "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/gate_up_proj" + ] }, { "type": "gemm", "args": { - "m": 240, - "n": 3072, - "k": 1536, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 56, + "n": 7168, + "k": 2304, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/down_proj" + ] }, { "type": "gemm", "args": { - "m": 240, - "n": 576, - "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 56, + "n": 7168, + "k": 16384, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/o_proj", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] }, { "type": "gemm", "args": { - "m": 240, - "n": 4096, - "k": 512, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 56, + "n": 24576, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/q_b_proj" + ] }, { "type": "gemm", "args": { - "m": 240, - "n": 7168, - "k": 2048, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 56, + "n": 32768, + "k": 512, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/kv_b_proj" + ] }, { "type": "gemm", "args": { - "m": 240, - "n": 2304, + "m": 57, + "n": 576, "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/kv_a_proj_with_mqa" + ] }, { "type": "gemm", "args": { - "m": 240, - "n": 7168, - "k": 2304, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 57, + "n": 1152, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/gate_up_proj" + ] }, { "type": "gemm", "args": { - "m": 400, + "m": 57, "n": 1536, "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/gate_up_proj", + "deepseek-ai/DeepSeek-R1-0528/q_a_proj", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w3" + ] }, { "type": "gemm", "args": { - "m": 400, - "n": 3072, - "k": 1536, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 400, - "n": 576, + "m": 57, + "n": 1536, "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w3" + ] }, { "type": "gemm", "args": { - "m": 400, - "n": 4096, - "k": 512, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 57, + "n": 7168, + "k": 1152, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/down_proj" + ] }, { "type": "gemm", "args": { - "m": 400, + "m": 57, "n": 7168, - "k": 2048, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "k": 16384, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/o_proj", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] }, { "type": "gemm", "args": { - "m": 400, - "n": 2304, - "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 57, + "n": 24576, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/q_b_proj" + ] }, { "type": "gemm", "args": { - "m": 400, - "n": 7168, - "k": 2304, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 57, + "n": 32768, + "k": 512, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/kv_b_proj" + ] }, { "type": "gemm", "args": { - "m": 1199104, - "n": 1536, + "m": 60, + "n": 576, "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/Kimi-K2.5-MXFP4/kv_a_proj_with_mqa", + "moonshotai/Kimi-K2.5/kv_a_proj_with_mqa", + "nvidia/DeepSeek-R1-0528-FP4-V2/kv_a_proj_with_mqa", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/kv_a_proj_with_mqa", + "nvidia/Kimi-K2.5-NVFP4/kv_a_proj_with_mqa", + "nvidia/Kimi-K2.6-NVFP4/kv_a_proj_with_mqa" + ] }, { "type": "gemm", "args": { - "m": 1199104, - "n": 24576, - "k": 1536, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 60, + "n": 576, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/kv_a_proj_with_mqa" + ] }, { "type": "gemm", "args": { - "m": 1199104, - "n": 576, + "m": 60, + "n": 1152, "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/gate_up_proj" + ] }, { "type": "gemm", "args": { - "m": 1199104, - "n": 32768, - "k": 512, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 60, + "n": 1536, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/q_b_proj" + ] }, { "type": "gemm", "args": { - "m": 1199104, - "n": 7168, - "k": 16384, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 60, + "n": 1536, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/Kimi-K2.5-MXFP4/q_a_proj", + "moonshotai/Kimi-K2.5/q_a_proj", + "nvidia/DeepSeek-R1-0528-FP4-V2/q_a_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/q_a_proj", + "nvidia/Kimi-K2.5-NVFP4/q_a_proj", + "nvidia/Kimi-K2.6-NVFP4/q_a_proj" + ] }, { "type": "gemm", "args": { - "m": 1199104, - "n": 9216, + "m": 60, + "n": 1536, "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/gate_up_proj", + "deepseek-ai/DeepSeek-R1-0528/q_a_proj", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w3" + ] }, { "type": "gemm", "args": { - "m": 1199104, - "n": 7168, - "k": 9216, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 60, + "n": 2048, + "k": 512, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/kv_b_proj" + ] }, { "type": "gemm", "args": { - "m": 2342, - "n": 1536, + "m": 60, + "n": 2304, "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/gate_up_proj" + ] }, { "type": "gemm", "args": { - "m": 2342, - "n": 24576, + "m": 60, + "n": 3072, "k": 1536, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/Kimi-K2.5-MXFP4/q_b_proj", + "amd/MiniMax-M2.5-MXFP4/o_proj", + "moonshotai/Kimi-K2.5/q_b_proj", + "nvidia/DeepSeek-R1-0528-FP4-V2/q_b_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/q_b_proj", + "nvidia/Kimi-K2.5-NVFP4/q_b_proj", + "nvidia/Kimi-K2.6-NVFP4/q_b_proj", + "nvidia/MiniMax-M2.5-NVFP4/o_proj" + ] }, { "type": "gemm", "args": { - "m": 2342, - "n": 576, - "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 60, + "n": 4096, + "k": 512, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "Qwen/Qwen3.5-397B-A17B/o_proj", + "Qwen/Qwen3.5-397B-A17B/shared_expert.down_proj", + "amd/Kimi-K2.5-MXFP4/kv_b_proj", + "amd/Qwen3.5-397B-A17B-MXFP4/o_proj", + "amd/Qwen3.5-397B-A17B-MXFP4/shared_expert.down_proj", + "moonshotai/Kimi-K2.5/kv_b_proj", + "nvidia/DeepSeek-R1-0528-FP4-V2/kv_b_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/kv_b_proj", + "nvidia/Kimi-K2.5-NVFP4/kv_b_proj", + "nvidia/Kimi-K2.6-NVFP4/kv_b_proj", + "nvidia/Qwen3.5-397B-A17B-NVFP4/o_proj", + "nvidia/Qwen3.5-397B-A17B-NVFP4/shared_expert.down_proj" + ] }, { "type": "gemm", "args": { - "m": 2342, - "n": 32768, - "k": 512, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 60, + "n": 4608, + "k": 7168, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/gate_up_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/gate_up_proj", + "nvidia/Kimi-K2.5-NVFP4/gate_up_proj" + ] }, { "type": "gemm", "args": { - "m": 2342, + "m": 60, "n": 7168, - "k": 16384, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "k": 1024, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/o_proj", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] }, { "type": "gemm", "args": { - "m": 2342, - "n": 9216, - "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 60, + "n": 7168, + "k": 1152, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/down_proj" + ] }, { "type": "gemm", "args": { - "m": 2342, + "m": 60, "n": 7168, - "k": 9216, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "k": 2048, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/Kimi-K2.5-MXFP4/o_proj", + "moonshotai/Kimi-K2.5/o_proj", + "nvidia/Kimi-K2.5-NVFP4/o_proj", + "nvidia/Kimi-K2.6-NVFP4/o_proj" + ] }, { "type": "gemm", "args": { - "m": 299776, - "n": 1536, - "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 60, + "n": 7168, + "k": 2048, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/o_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/o_proj" + ] }, { "type": "gemm", "args": { - "m": 299776, - "n": 24576, - "k": 1536, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 60, + "n": 7168, + "k": 2304, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/down_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/down_proj", + "nvidia/Kimi-K2.5-NVFP4/down_proj" + ] }, { "type": "gemm", "args": { - "m": 299776, - "n": 576, - "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 60, + "n": 7168, + "k": 4608, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/down_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/down_proj", + "nvidia/Kimi-K2.5-NVFP4/down_proj" + ] }, { "type": "gemm", "args": { - "m": 299776, - "n": 32768, - "k": 512, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 60, + "n": 7168, + "k": 9216, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/down_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/down_proj", + "nvidia/Kimi-K2.5-NVFP4/down_proj" + ] }, { "type": "gemm", "args": { - "m": 299776, + "m": 60, "n": 7168, "k": 16384, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/o_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/o_proj" + ] }, { "type": "gemm", "args": { - "m": 299776, - "n": 2304, - "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 60, + "n": 7168, + "k": 16384, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 32 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 1, + 32 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "amd/DeepSeek-R1-0528-MXFP4-Preview/o_proj", + "amd/DeepSeek-R1-0528-MXFP4-v2/o_proj", + "amd/DeepSeek-R1-0528-MXFP4/o_proj" + ] }, { "type": "gemm", "args": { - "m": 299776, + "m": 60, "n": 7168, - "k": 2304, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "k": 16384, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] }, { "type": "gemm", "args": { - "m": 585, - "n": 1536, + "m": 60, + "n": 9216, "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/gate_up_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/gate_up_proj", + "nvidia/Kimi-K2.5-NVFP4/gate_up_proj" + ] }, { "type": "gemm", "args": { - "m": 585, + "m": 60, "n": 24576, "k": 1536, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/q_b_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/q_b_proj" + ] }, { "type": "gemm", "args": { - "m": 585, - "n": 576, - "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 60, + "n": 24576, + "k": 1536, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 32 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 1, + 32 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "amd/DeepSeek-R1-0528-MXFP4-Preview/q_b_proj", + "amd/DeepSeek-R1-0528-MXFP4-v2/q_b_proj", + "amd/DeepSeek-R1-0528-MXFP4/q_b_proj" + ] }, { "type": "gemm", "args": { - "m": 585, + "m": 60, "n": 32768, "k": 512, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/kv_b_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/kv_b_proj" + ] }, { "type": "gemm", "args": { - "m": 585, - "n": 7168, - "k": 16384, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 60, + "n": 32768, + "k": 512, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 32 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 1, + 32 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "amd/DeepSeek-R1-0528-MXFP4-Preview/kv_b_proj", + "amd/DeepSeek-R1-0528-MXFP4-v2/kv_b_proj", + "amd/DeepSeek-R1-0528-MXFP4/kv_b_proj" + ] }, { "type": "gemm", "args": { - "m": 585, - "n": 2304, + "m": 63, + "n": 576, "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/Kimi-K2.5-MXFP4/kv_a_proj_with_mqa", + "moonshotai/Kimi-K2.5/kv_a_proj_with_mqa", + "nvidia/DeepSeek-R1-0528-FP4-V2/kv_a_proj_with_mqa", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/kv_a_proj_with_mqa", + "nvidia/Kimi-K2.5-NVFP4/kv_a_proj_with_mqa", + "nvidia/Kimi-K2.6-NVFP4/kv_a_proj_with_mqa" + ] }, { "type": "gemm", "args": { - "m": 585, - "n": 7168, - "k": 2304, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 63, + "n": 576, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/kv_a_proj_with_mqa" + ] }, { "type": "gemm", "args": { - "m": 4407808, + "m": 63, "n": 1536, "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/Kimi-K2.5-MXFP4/q_a_proj", + "moonshotai/Kimi-K2.5/q_a_proj", + "nvidia/DeepSeek-R1-0528-FP4-V2/q_a_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/q_a_proj", + "nvidia/Kimi-K2.5-NVFP4/q_a_proj", + "nvidia/Kimi-K2.6-NVFP4/q_a_proj" + ] }, { "type": "gemm", "args": { - "m": 4407808, - "n": 24576, - "k": 1536, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 63, + "n": 1536, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/gate_up_proj", + "deepseek-ai/DeepSeek-R1-0528/q_a_proj", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w3" + ] }, { "type": "gemm", "args": { - "m": 4407808, - "n": 576, + "m": 63, + "n": 2304, "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/gate_up_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/gate_up_proj", + "nvidia/Kimi-K2.5-NVFP4/gate_up_proj" + ] }, { "type": "gemm", "args": { - "m": 4407808, - "n": 32768, - "k": 512, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 63, + "n": 2304, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/gate_up_proj" + ] }, { "type": "gemm", "args": { - "m": 4407808, - "n": 7168, - "k": 16384, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 63, + "n": 3072, + "k": 1536, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/Kimi-K2.5-MXFP4/q_b_proj", + "amd/MiniMax-M2.5-MXFP4/o_proj", + "moonshotai/Kimi-K2.5/q_b_proj", + "nvidia/DeepSeek-R1-0528-FP4-V2/q_b_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/q_b_proj", + "nvidia/Kimi-K2.5-NVFP4/q_b_proj", + "nvidia/Kimi-K2.6-NVFP4/q_b_proj", + "nvidia/MiniMax-M2.5-NVFP4/o_proj" + ] }, { "type": "gemm", "args": { - "m": 4407808, - "n": 9216, - "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 63, + "n": 3072, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "MiniMaxAI/MiniMax-M2.5/experts.down_proj", + "MiniMaxAI/MiniMax-M2.5/o_proj", + "deepseek-ai/DeepSeek-R1-0528/q_b_proj" + ] }, { "type": "gemm", "args": { - "m": 4407808, - "n": 7168, - "k": 9216, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 63, + "n": 4096, + "k": 512, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "Qwen/Qwen3.5-397B-A17B/o_proj", + "Qwen/Qwen3.5-397B-A17B/shared_expert.down_proj", + "amd/Kimi-K2.5-MXFP4/kv_b_proj", + "amd/Qwen3.5-397B-A17B-MXFP4/o_proj", + "amd/Qwen3.5-397B-A17B-MXFP4/shared_expert.down_proj", + "moonshotai/Kimi-K2.5/kv_b_proj", + "nvidia/DeepSeek-R1-0528-FP4-V2/kv_b_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/kv_b_proj", + "nvidia/Kimi-K2.5-NVFP4/kv_b_proj", + "nvidia/Kimi-K2.6-NVFP4/kv_b_proj", + "nvidia/Qwen3.5-397B-A17B-NVFP4/o_proj", + "nvidia/Qwen3.5-397B-A17B-NVFP4/shared_expert.down_proj" + ] }, { "type": "gemm", "args": { - "m": 8609, - "n": 1536, - "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 63, + "n": 4096, + "k": 512, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "Qwen/Qwen3.5-397B-A17B-FP8/o_proj", + "Qwen/Qwen3.5-397B-A17B-FP8/shared_expert.down_proj", + "deepseek-ai/DeepSeek-R1-0528/kv_b_proj" + ] }, { "type": "gemm", "args": { - "m": 8609, - "n": 24576, - "k": 1536, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 63, + "n": 7168, + "k": 2048, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/o_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/o_proj" + ] }, { "type": "gemm", "args": { - "m": 8609, - "n": 576, - "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 63, + "n": 7168, + "k": 2048, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/o_proj", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] }, { "type": "gemm", "args": { - "m": 8609, - "n": 32768, - "k": 512, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 63, + "n": 7168, + "k": 2304, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/down_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/down_proj", + "nvidia/Kimi-K2.5-NVFP4/down_proj" + ] }, { "type": "gemm", "args": { - "m": 8609, + "m": 63, "n": 7168, - "k": 16384, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "k": 2304, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/down_proj" + ] }, { "type": "gemm", "args": { - "m": 8609, - "n": 9216, - "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 64, + "n": 64, + "k": 2880, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/gpt-oss-120b-w-mxfp4-a-fp8/k_proj", + "amd/gpt-oss-120b-w-mxfp4-a-fp8/v_proj", + "openai/gpt-oss-120b/k_proj", + "openai/gpt-oss-120b/v_proj" + ] }, { "type": "gemm", "args": { - "m": 8609, - "n": 7168, - "k": 9216, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 64, + "n": 128, + "k": 2880, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/gpt-oss-120b-w-mxfp4-a-fp8/k_proj", + "amd/gpt-oss-120b-w-mxfp4-a-fp8/v_proj", + "openai/gpt-oss-120b/k_proj", + "openai/gpt-oss-120b/v_proj" + ] }, { "type": "gemm", "args": { - "m": 1101952, - "n": 1536, - "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 64, + "n": 128, + "k": 3072, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "MiniMaxAI/MiniMax-M2.5/k_proj", + "MiniMaxAI/MiniMax-M2.5/v_proj" + ] }, { "type": "gemm", "args": { - "m": 1101952, - "n": 24576, - "k": 1536, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 64, + "n": 128, + "k": 4096, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "Qwen/Qwen3.5-397B-A17B/shared_expert.gate_up_proj", + "amd/Qwen3.5-397B-A17B-MXFP4/shared_expert.gate_up_proj", + "nvidia/Qwen3.5-397B-A17B-NVFP4/shared_expert.gate_up_proj" + ] }, { "type": "gemm", "args": { - "m": 1101952, - "n": 576, - "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 64, + "n": 128, + "k": 4096, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "Qwen/Qwen3.5-397B-A17B-FP8/shared_expert.gate_up_proj" + ] }, { "type": "gemm", "args": { - "m": 1101952, - "n": 32768, - "k": 512, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 64, + "n": 192, + "k": 3072, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "MiniMaxAI/MiniMax-M2.5/experts.gate_up_proj" + ] }, { "type": "gemm", "args": { - "m": 1101952, - "n": 7168, - "k": 16384, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 64, + "n": 256, + "k": 2880, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/gpt-oss-120b-w-mxfp4-a-fp8/k_proj", + "amd/gpt-oss-120b-w-mxfp4-a-fp8/q_proj", + "amd/gpt-oss-120b-w-mxfp4-a-fp8/v_proj", + "openai/gpt-oss-120b/k_proj", + "openai/gpt-oss-120b/q_proj", + "openai/gpt-oss-120b/v_proj" + ] }, { "type": "gemm", "args": { - "m": 1101952, - "n": 2304, - "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 64, + "n": 256, + "k": 3072, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/MiniMax-M2.5-MXFP4/k_proj", + "amd/MiniMax-M2.5-MXFP4/v_proj", + "nvidia/MiniMax-M2.5-NVFP4/k_proj", + "nvidia/MiniMax-M2.5-NVFP4/v_proj" + ] }, { "type": "gemm", "args": { - "m": 1101952, - "n": 7168, - "k": 2304, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 64, + "n": 256, + "k": 3072, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "MiniMaxAI/MiniMax-M2.5/k_proj", + "MiniMaxAI/MiniMax-M2.5/v_proj" + ] }, { "type": "gemm", "args": { - "m": 2152, - "n": 1536, - "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 64, + "n": 256, + "k": 4096, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "Qwen/Qwen3.5-397B-A17B/k_proj", + "Qwen/Qwen3.5-397B-A17B/shared_expert.gate_up_proj", + "Qwen/Qwen3.5-397B-A17B/v_proj", + "amd/Qwen3.5-397B-A17B-MXFP4/k_proj", + "amd/Qwen3.5-397B-A17B-MXFP4/shared_expert.gate_up_proj", + "amd/Qwen3.5-397B-A17B-MXFP4/v_proj", + "nvidia/Qwen3.5-397B-A17B-NVFP4/k_proj", + "nvidia/Qwen3.5-397B-A17B-NVFP4/shared_expert.gate_up_proj", + "nvidia/Qwen3.5-397B-A17B-NVFP4/v_proj" + ] }, { "type": "gemm", "args": { - "m": 2152, - "n": 24576, - "k": 1536, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 64, + "n": 256, + "k": 4096, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "Qwen/Qwen3.5-397B-A17B-FP8/k_proj", + "Qwen/Qwen3.5-397B-A17B-FP8/shared_expert.gate_up_proj", + "Qwen/Qwen3.5-397B-A17B-FP8/v_proj" + ] }, { "type": "gemm", "args": { - "m": 2152, - "n": 576, - "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 64, + "n": 384, + "k": 3072, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "MiniMaxAI/MiniMax-M2.5/experts.gate_up_proj", + "MiniMaxAI/MiniMax-M2.5/q_proj" + ] }, { "type": "gemm", "args": { - "m": 2152, - "n": 32768, - "k": 512, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 64, + "n": 512, + "k": 2880, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/gpt-oss-120b-w-mxfp4-a-fp8/k_proj", + "amd/gpt-oss-120b-w-mxfp4-a-fp8/q_proj", + "amd/gpt-oss-120b-w-mxfp4-a-fp8/v_proj", + "openai/gpt-oss-120b/k_proj", + "openai/gpt-oss-120b/q_proj", + "openai/gpt-oss-120b/v_proj" + ] }, { "type": "gemm", "args": { - "m": 2152, - "n": 7168, - "k": 16384, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 64, + "n": 512, + "k": 3072, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/MiniMax-M2.5-MXFP4/k_proj", + "amd/MiniMax-M2.5-MXFP4/v_proj", + "nvidia/MiniMax-M2.5-NVFP4/k_proj", + "nvidia/MiniMax-M2.5-NVFP4/v_proj" + ] }, { "type": "gemm", "args": { - "m": 2152, - "n": 2304, - "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 64, + "n": 512, + "k": 3072, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "MiniMaxAI/MiniMax-M2.5/k_proj", + "MiniMaxAI/MiniMax-M2.5/v_proj" + ] }, { "type": "gemm", "args": { - "m": 2152, - "n": 7168, - "k": 2304, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 64, + "n": 512, + "k": 4096, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "Qwen/Qwen3.5-397B-A17B/k_proj", + "Qwen/Qwen3.5-397B-A17B/q_proj", + "Qwen/Qwen3.5-397B-A17B/shared_expert.gate_up_proj", + "Qwen/Qwen3.5-397B-A17B/v_proj", + "amd/Qwen3.5-397B-A17B-MXFP4/k_proj", + "amd/Qwen3.5-397B-A17B-MXFP4/q_proj", + "amd/Qwen3.5-397B-A17B-MXFP4/shared_expert.gate_up_proj", + "amd/Qwen3.5-397B-A17B-MXFP4/v_proj", + "nvidia/Qwen3.5-397B-A17B-NVFP4/k_proj", + "nvidia/Qwen3.5-397B-A17B-NVFP4/q_proj", + "nvidia/Qwen3.5-397B-A17B-NVFP4/shared_expert.gate_up_proj", + "nvidia/Qwen3.5-397B-A17B-NVFP4/v_proj" + ] }, { "type": "gemm", "args": { - "m": 6618112, - "n": 1536, - "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 64, + "n": 512, + "k": 4096, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "Qwen/Qwen3.5-397B-A17B-FP8/k_proj", + "Qwen/Qwen3.5-397B-A17B-FP8/q_proj", + "Qwen/Qwen3.5-397B-A17B-FP8/shared_expert.gate_up_proj", + "Qwen/Qwen3.5-397B-A17B-FP8/v_proj", + "deepseek-ai/DeepSeek-V4.1-Flash/attn.wo_a" + ] }, { "type": "gemm", "args": { - "m": 6618112, - "n": 24576, - "k": 1536, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 64, + "n": 576, + "k": 6144, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/GLM-5.1-MXFP4/kv_a_proj_with_mqa", + "amd/GLM-5.2-MXFP4/kv_a_proj_with_mqa", + "nvidia/GLM-5-NVFP4/kv_a_proj_with_mqa", + "nvidia/GLM-5.1-NVFP4/kv_a_proj_with_mqa", + "nvidia/GLM-5.2-NVFP4/kv_a_proj_with_mqa" + ] }, { "type": "gemm", "args": { - "m": 6618112, + "m": 64, "n": 576, - "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "k": 6144, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "zai-org/GLM-5-FP8/kv_a_proj_with_mqa", + "zai-org/GLM-5.1-FP8/kv_a_proj_with_mqa", + "zai-org/GLM-5.2-FP8/kv_a_proj_with_mqa" + ] }, { "type": "gemm", "args": { - "m": 6618112, - "n": 32768, - "k": 512, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 64, + "n": 576, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/Kimi-K2.5-MXFP4/kv_a_proj_with_mqa", + "moonshotai/Kimi-K2.5/kv_a_proj_with_mqa", + "nvidia/DeepSeek-R1-0528-FP4-V2/kv_a_proj_with_mqa", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/kv_a_proj_with_mqa", + "nvidia/Kimi-K2.5-NVFP4/kv_a_proj_with_mqa", + "nvidia/Kimi-K2.6-NVFP4/kv_a_proj_with_mqa" + ] }, { "type": "gemm", "args": { - "m": 6618112, - "n": 7168, - "k": 16384, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 64, + "n": 576, + "k": 7168, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 32 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 1, + 32 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "amd/DeepSeek-R1-0528-MXFP4-Preview/kv_a_proj_with_mqa", + "amd/DeepSeek-R1-0528-MXFP4-v2/kv_a_proj_with_mqa", + "amd/DeepSeek-R1-0528-MXFP4/kv_a_proj_with_mqa" + ] }, { "type": "gemm", "args": { - "m": 6618112, - "n": 9216, + "m": 64, + "n": 576, "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/kv_a_proj_with_mqa" + ] }, { "type": "gemm", "args": { - "m": 6618112, - "n": 7168, - "k": 9216, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 64, + "n": 768, + "k": 3072, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/MiniMax-M2.5-NVFP4/experts.gate_up_proj" + ] }, { "type": "gemm", "args": { - "m": 12926, - "n": 1536, - "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 64, + "n": 768, + "k": 3072, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 32 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 1, + 32 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "amd/MiniMax-M2.5-MXFP4/experts.gate_up_proj" + ] }, { "type": "gemm", "args": { - "m": 12926, - "n": 24576, - "k": 1536, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 64, + "n": 768, + "k": 3072, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "MiniMaxAI/MiniMax-M2.5/experts.gate_up_proj", + "MiniMaxAI/MiniMax-M2.5/q_proj" + ] }, { "type": "gemm", "args": { - "m": 12926, - "n": 576, - "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 64, + "n": 1024, + "k": 2880, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/gpt-oss-120b-w-mxfp4-a-fp8/q_proj", + "openai/gpt-oss-120b/q_proj" + ] }, { "type": "gemm", "args": { - "m": 12926, - "n": 32768, - "k": 512, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 64, + "n": 1024, + "k": 3072, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "MiniMaxAI/MiniMax-M2.5/k_proj", + "MiniMaxAI/MiniMax-M2.5/qkv_proj", + "MiniMaxAI/MiniMax-M2.5/v_proj" + ] }, { "type": "gemm", "args": { - "m": 12926, - "n": 7168, - "k": 16384, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 64, + "n": 1024, + "k": 4096, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "Qwen/Qwen3.5-397B-A17B/q_proj", + "Qwen/Qwen3.5-397B-A17B/shared_expert.gate_up_proj", + "amd/Qwen3.5-397B-A17B-MXFP4/q_proj", + "amd/Qwen3.5-397B-A17B-MXFP4/shared_expert.gate_up_proj", + "nvidia/Qwen3.5-397B-A17B-NVFP4/q_proj", + "nvidia/Qwen3.5-397B-A17B-NVFP4/shared_expert.gate_up_proj" + ] }, { "type": "gemm", "args": { - "m": 12926, - "n": 9216, + "m": 64, + "n": 1024, + "k": 4096, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "Qwen/Qwen3.5-397B-A17B-FP8/q_proj", + "Qwen/Qwen3.5-397B-A17B-FP8/shared_expert.gate_up_proj", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_a", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_a", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_a" + ] + }, + { + "type": "gemm", + "args": { + "m": 64, + "n": 1088, "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] }, { "type": "gemm", "args": { - "m": 12926, - "n": 7168, - "k": 9216, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 64, + "n": 1152, + "k": 7168, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-NVFP4-v2/gate_up_proj" + ] }, { "type": "gemm", "args": { - "m": 1654528, - "n": 1536, + "m": 64, + "n": 1152, "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/gate_up_proj" + ] }, { "type": "gemm", "args": { - "m": 1654528, - "n": 24576, + "m": 64, + "n": 1536, "k": 1536, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/Kimi-K2.5-MXFP4/q_b_proj", + "moonshotai/Kimi-K2.5/q_b_proj", + "nvidia/DeepSeek-R1-0528-FP4-V2/q_b_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/q_b_proj", + "nvidia/Kimi-K2.5-NVFP4/q_b_proj", + "nvidia/Kimi-K2.6-NVFP4/q_b_proj" + ] }, { "type": "gemm", "args": { - "m": 1654528, - "n": 576, - "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 64, + "n": 1536, + "k": 3072, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 32 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 1, + 32 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "amd/MiniMax-M2.5-MXFP4/experts.gate_up_proj" + ] }, { "type": "gemm", "args": { - "m": 1654528, - "n": 32768, - "k": 512, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 64, + "n": 1536, + "k": 3072, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "MiniMaxAI/MiniMax-M2.5/experts.gate_up_proj", + "MiniMaxAI/MiniMax-M2.5/q_proj" + ] }, { "type": "gemm", "args": { - "m": 1654528, - "n": 7168, - "k": 16384, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 64, + "n": 1536, + "k": 6144, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "zai-org/GLM-5-FP8/gate_up_proj", + "zai-org/GLM-5.1-FP8/gate_up_proj", + "zai-org/GLM-5.2-FP8/gate_up_proj" + ] }, { "type": "gemm", "args": { - "m": 1654528, - "n": 2304, + "m": 64, + "n": 1536, "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/Kimi-K2.5-MXFP4/q_a_proj", + "moonshotai/Kimi-K2.5/q_a_proj", + "nvidia/DeepSeek-R1-0528-FP4-V2/q_a_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/q_a_proj", + "nvidia/Kimi-K2.5-NVFP4/q_a_proj", + "nvidia/Kimi-K2.6-NVFP4/q_a_proj" + ] }, { "type": "gemm", "args": { - "m": 1654528, - "n": 7168, - "k": 2304, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 64, + "n": 1536, + "k": 7168, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 32 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 1, + 32 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "amd/DeepSeek-R1-0528-MXFP4-Preview/q_a_proj", + "amd/DeepSeek-R1-0528-MXFP4-v2/q_a_proj", + "amd/DeepSeek-R1-0528-MXFP4/q_a_proj" + ] }, { "type": "gemm", "args": { - "m": 3231, + "m": 64, "n": 1536, "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/gate_up_proj", + "deepseek-ai/DeepSeek-R1-0528/q_a_proj", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w3" + ] }, { "type": "gemm", "args": { - "m": 3231, - "n": 24576, - "k": 1536, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 64, + "n": 1536, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w3" + ] }, { "type": "gemm", "args": { - "m": 3231, - "n": 576, - "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 64, + "n": 2048, + "k": 512, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/Kimi-K2.5-MXFP4/kv_b_proj", + "moonshotai/Kimi-K2.5/kv_b_proj", + "nvidia/DeepSeek-R1-0528-FP4-V2/kv_b_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/kv_b_proj", + "nvidia/Kimi-K2.5-NVFP4/kv_b_proj", + "nvidia/Kimi-K2.6-NVFP4/kv_b_proj" + ] }, { "type": "gemm", "args": { - "m": 3231, - "n": 32768, - "k": 512, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 64, + "n": 2048, + "k": 2048, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "zai-org/GLM-5-FP8/q_b_proj", + "zai-org/GLM-5.1-FP8/q_b_proj", + "zai-org/GLM-5.2-FP8/q_b_proj" + ] }, { "type": "gemm", "args": { - "m": 3231, - "n": 7168, - "k": 16384, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 64, + "n": 2048, + "k": 2880, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/gpt-oss-120b-w-mxfp4-a-fp8/q_proj", + "openai/gpt-oss-120b/q_proj" + ] }, { "type": "gemm", "args": { - "m": 3231, - "n": 2304, - "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 64, + "n": 2048, + "k": 4096, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "Qwen/Qwen3.5-397B-A17B/q_proj", + "Qwen/Qwen3.5-397B-A17B/shared_expert.gate_up_proj", + "amd/Qwen3.5-397B-A17B-MXFP4/q_proj", + "amd/Qwen3.5-397B-A17B-MXFP4/shared_expert.gate_up_proj", + "nvidia/Qwen3.5-397B-A17B-NVFP4/q_proj", + "nvidia/Qwen3.5-397B-A17B-NVFP4/shared_expert.gate_up_proj" + ] }, { "type": "gemm", "args": { - "m": 3231, - "n": 7168, - "k": 2304, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 64, + "n": 2048, + "k": 4096, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "Qwen/Qwen3.5-397B-A17B-FP8/q_proj", + "Qwen/Qwen3.5-397B-A17B-FP8/shared_expert.gate_up_proj" + ] }, { "type": "gemm", "args": { - "m": 602112, - "n": 1536, - "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 64, + "n": 2048, + "k": 6144, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/GLM-5.1-MXFP4/q_a_proj", + "amd/GLM-5.2-MXFP4/q_a_proj", + "amd/MiniMax-M3-MXFP4/q_proj", + "nvidia/GLM-5-NVFP4/q_a_proj", + "nvidia/GLM-5.1-NVFP4/q_a_proj", + "nvidia/GLM-5.2-NVFP4/q_a_proj" + ] }, { "type": "gemm", "args": { - "m": 602112, - "n": 24576, - "k": 1536, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 64, + "n": 2048, + "k": 6144, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "zai-org/GLM-5-FP8/q_a_proj", + "zai-org/GLM-5.1-FP8/q_a_proj", + "zai-org/GLM-5.2-FP8/q_a_proj" + ] }, { "type": "gemm", "args": { - "m": 602112, - "n": 576, + "m": 64, + "n": 2304, "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/gate_up_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/gate_up_proj", + "nvidia/Kimi-K2.5-NVFP4/gate_up_proj" + ] }, { "type": "gemm", "args": { - "m": 602112, - "n": 32768, + "m": 64, + "n": 2880, "k": 512, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/gpt-oss-120b-w-mxfp4-a-fp8/o_proj", + "openai/gpt-oss-120b/o_proj" + ] }, { "type": "gemm", "args": { - "m": 602112, - "n": 7168, - "k": 16384, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 64, + "n": 2880, + "k": 1024, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/gpt-oss-120b-w-mxfp4-a-fp8/o_proj", + "openai/gpt-oss-120b/o_proj" + ] }, { "type": "gemm", "args": { - "m": 602112, - "n": 9216, - "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 64, + "n": 2880, + "k": 2048, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/gpt-oss-120b-w-mxfp4-a-fp8/o_proj", + "openai/gpt-oss-120b/o_proj" + ] }, { "type": "gemm", "args": { - "m": 602112, - "n": 7168, - "k": 9216, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 64, + "n": 2880, + "k": 4096, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/gpt-oss-120b-w-mxfp4-a-fp8/o_proj", + "openai/gpt-oss-120b/o_proj" + ] }, { "type": "gemm", "args": { - "m": 588, - "n": 1536, - "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 64, + "n": 3072, + "k": 192, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "MiniMaxAI/MiniMax-M2.5/experts.down_proj" + ] }, { "type": "gemm", "args": { - "m": 588, - "n": 24576, - "k": 1536, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 64, + "n": 3072, + "k": 384, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/MiniMax-M2.5-NVFP4/experts.down_proj" + ] }, { "type": "gemm", "args": { - "m": 588, - "n": 576, - "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 64, + "n": 3072, + "k": 384, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 32 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 1, + 32 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "amd/MiniMax-M2.5-MXFP4/experts.down_proj" + ] }, { "type": "gemm", "args": { - "m": 588, - "n": 32768, - "k": 512, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 64, + "n": 3072, + "k": 384, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "MiniMaxAI/MiniMax-M2.5/experts.down_proj", + "MiniMaxAI/MiniMax-M2.5/o_proj" + ] }, { "type": "gemm", "args": { - "m": 588, - "n": 7168, - "k": 16384, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 64, + "n": 3072, + "k": 768, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 32 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 1, + 32 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "amd/MiniMax-M2.5-MXFP4/experts.down_proj" + ] }, { "type": "gemm", "args": { - "m": 588, - "n": 9216, - "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 64, + "n": 3072, + "k": 768, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "MiniMaxAI/MiniMax-M2.5/experts.down_proj", + "MiniMaxAI/MiniMax-M2.5/o_proj" + ] }, { "type": "gemm", "args": { - "m": 588, - "n": 7168, - "k": 9216, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 64, + "n": 3072, + "k": 1536, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/Kimi-K2.5-MXFP4/q_b_proj", + "amd/MiniMax-M2.5-MXFP4/o_proj", + "moonshotai/Kimi-K2.5/q_b_proj", + "nvidia/DeepSeek-R1-0528-FP4-V2/q_b_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/q_b_proj", + "nvidia/Kimi-K2.5-NVFP4/q_b_proj", + "nvidia/Kimi-K2.6-NVFP4/q_b_proj", + "nvidia/MiniMax-M2.5-NVFP4/o_proj" + ] }, { "type": "gemm", "args": { - "m": 150528, - "n": 1536, - "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 64, + "n": 3072, + "k": 1536, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/MiniMax-M2.5-NVFP4/experts.down_proj" + ] }, { "type": "gemm", "args": { - "m": 150528, - "n": 24576, + "m": 64, + "n": 3072, "k": 1536, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 32 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 1, + 32 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "amd/DeepSeek-R1-0528-MXFP4-Preview/q_b_proj", + "amd/DeepSeek-R1-0528-MXFP4-v2/q_b_proj", + "amd/DeepSeek-R1-0528-MXFP4/q_b_proj", + "amd/MiniMax-M2.5-MXFP4/experts.down_proj" + ] }, { "type": "gemm", "args": { - "m": 150528, - "n": 576, - "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 64, + "n": 3072, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "MiniMaxAI/MiniMax-M2.5/experts.down_proj", + "MiniMaxAI/MiniMax-M2.5/o_proj", + "deepseek-ai/DeepSeek-R1-0528/q_b_proj" + ] }, { "type": "gemm", "args": { - "m": 150528, - "n": 32768, - "k": 512, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 64, + "n": 3072, + "k": 3072, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/MiniMax-M2.5-MXFP4/o_proj", + "amd/MiniMax-M2.5-MXFP4/q_proj", + "nvidia/MiniMax-M2.5-NVFP4/o_proj", + "nvidia/MiniMax-M2.5-NVFP4/q_proj" + ] }, { "type": "gemm", "args": { - "m": 150528, - "n": 7168, - "k": 16384, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 64, + "n": 3072, + "k": 3072, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/MiniMax-M2.5-NVFP4/experts.gate_up_proj" + ] }, { "type": "gemm", "args": { - "m": 150528, - "n": 2304, - "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 64, + "n": 3072, + "k": 3072, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 32 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 1, + 32 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "amd/MiniMax-M2.5-MXFP4/experts.gate_up_proj" + ] }, { "type": "gemm", "args": { - "m": 150528, - "n": 7168, - "k": 2304, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 64, + "n": 3072, + "k": 3072, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "MiniMaxAI/MiniMax-M2.5/experts.gate_up_proj", + "MiniMaxAI/MiniMax-M2.5/o_proj", + "MiniMaxAI/MiniMax-M2.5/q_proj" + ] }, { "type": "gemm", "args": { - "m": 147, - "n": 1536, - "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 64, + "n": 3072, + "k": 6144, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/GLM-5.1-MXFP4/gate_up_proj", + "amd/GLM-5.2-MXFP4/gate_up_proj", + "amd/MiniMax-M2.5-MXFP4/o_proj", + "amd/MiniMax-M3-MXFP4/gate_up_proj", + "nvidia/GLM-5.1-NVFP4/gate_up_proj", + "nvidia/GLM-5.2-NVFP4/gate_up_proj", + "nvidia/MiniMax-M2.5-NVFP4/o_proj" + ] }, { "type": "gemm", "args": { - "m": 147, - "n": 24576, - "k": 1536, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 64, + "n": 3072, + "k": 6144, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "MiniMaxAI/MiniMax-M2.5/o_proj", + "zai-org/GLM-5-FP8/gate_up_proj", + "zai-org/GLM-5.1-FP8/gate_up_proj", + "zai-org/GLM-5.2-FP8/gate_up_proj" + ] }, { "type": "gemm", "args": { - "m": 147, - "n": 576, - "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 64, + "n": 3584, + "k": 512, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "zai-org/GLM-5-FP8/kv_b_proj", + "zai-org/GLM-5.1-FP8/kv_b_proj", + "zai-org/GLM-5.2-FP8/kv_b_proj" + ] }, { "type": "gemm", "args": { - "m": 147, - "n": 32768, - "k": 512, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 64, + "n": 4096, + "k": 128, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "Qwen/Qwen3.5-397B-A17B/shared_expert.down_proj", + "amd/Qwen3.5-397B-A17B-MXFP4/shared_expert.down_proj", + "nvidia/Qwen3.5-397B-A17B-NVFP4/shared_expert.down_proj" + ] }, { "type": "gemm", "args": { - "m": 147, - "n": 7168, - "k": 16384, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 64, + "n": 4096, + "k": 128, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "Qwen/Qwen3.5-397B-A17B-FP8/shared_expert.down_proj" + ] }, { "type": "gemm", "args": { - "m": 147, - "n": 2304, - "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 64, + "n": 4096, + "k": 256, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "Qwen/Qwen3.5-397B-A17B/shared_expert.down_proj", + "amd/Qwen3.5-397B-A17B-MXFP4/shared_expert.down_proj", + "nvidia/Qwen3.5-397B-A17B-NVFP4/shared_expert.down_proj" + ] }, { "type": "gemm", "args": { - "m": 147, - "n": 7168, - "k": 2304, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 64, + "n": 4096, + "k": 256, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "Qwen/Qwen3.5-397B-A17B-FP8/shared_expert.down_proj" + ] }, { "type": "gemm", "args": { - "m": 3072, - "n": 1536, - "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 64, + "n": 4096, + "k": 512, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "Qwen/Qwen3.5-397B-A17B/o_proj", + "Qwen/Qwen3.5-397B-A17B/shared_expert.down_proj", + "amd/Kimi-K2.5-MXFP4/kv_b_proj", + "amd/Qwen3.5-397B-A17B-MXFP4/o_proj", + "amd/Qwen3.5-397B-A17B-MXFP4/shared_expert.down_proj", + "moonshotai/Kimi-K2.5/kv_b_proj", + "nvidia/DeepSeek-R1-0528-FP4-V2/kv_b_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/kv_b_proj", + "nvidia/Kimi-K2.5-NVFP4/kv_b_proj", + "nvidia/Kimi-K2.6-NVFP4/kv_b_proj", + "nvidia/Qwen3.5-397B-A17B-NVFP4/o_proj", + "nvidia/Qwen3.5-397B-A17B-NVFP4/shared_expert.down_proj" + ] }, { "type": "gemm", "args": { - "m": 3072, - "n": 24576, - "k": 1536, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 64, + "n": 4096, + "k": 512, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 32 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 1, + 32 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "amd/DeepSeek-R1-0528-MXFP4-Preview/kv_b_proj", + "amd/DeepSeek-R1-0528-MXFP4-v2/kv_b_proj", + "amd/DeepSeek-R1-0528-MXFP4/kv_b_proj", + "amd/Qwen3.5-397B-A17B-MXFP4-AttnFP8-V2/shared_expert.down_proj" + ] }, { "type": "gemm", "args": { - "m": 3072, - "n": 576, - "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 64, + "n": 4096, + "k": 512, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "Qwen/Qwen3.5-397B-A17B-FP8/o_proj", + "Qwen/Qwen3.5-397B-A17B-FP8/shared_expert.down_proj", + "deepseek-ai/DeepSeek-R1-0528/kv_b_proj" + ] }, { "type": "gemm", "args": { - "m": 3072, - "n": 32768, - "k": 512, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 64, + "n": 4096, + "k": 1024, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "Qwen/Qwen3.5-397B-A17B/o_proj", + "Qwen/Qwen3.5-397B-A17B/shared_expert.down_proj", + "amd/Qwen3.5-397B-A17B-MXFP4/o_proj", + "amd/Qwen3.5-397B-A17B-MXFP4/shared_expert.down_proj", + "nvidia/Qwen3.5-397B-A17B-NVFP4/o_proj", + "nvidia/Qwen3.5-397B-A17B-NVFP4/shared_expert.down_proj" + ] }, { "type": "gemm", "args": { - "m": 3072, - "n": 7168, - "k": 16384, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 64, + "n": 4096, + "k": 1024, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "Qwen/Qwen3.5-397B-A17B-FP8/o_proj", + "Qwen/Qwen3.5-397B-A17B-FP8/shared_expert.down_proj" + ] }, { "type": "gemm", "args": { - "m": 3072, - "n": 9216, - "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 64, + "n": 4096, + "k": 2048, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "Qwen/Qwen3.5-397B-A17B/o_proj", + "amd/GLM-5.1-MXFP4/q_b_proj", + "amd/GLM-5.2-MXFP4/q_b_proj", + "amd/Qwen3.5-397B-A17B-MXFP4/o_proj", + "nvidia/GLM-5-NVFP4/q_b_proj", + "nvidia/GLM-5.1-NVFP4/q_b_proj", + "nvidia/GLM-5.2-NVFP4/q_b_proj", + "nvidia/Qwen3.5-397B-A17B-NVFP4/o_proj" + ] }, { "type": "gemm", "args": { - "m": 3072, - "n": 7168, - "k": 9216, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 64, + "n": 4096, + "k": 2048, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "Qwen/Qwen3.5-397B-A17B-FP8/o_proj", + "zai-org/GLM-5-FP8/q_b_proj", + "zai-org/GLM-5.1-FP8/q_b_proj", + "zai-org/GLM-5.2-FP8/q_b_proj" + ] }, { "type": "gemm", "args": { - "m": 3, - "n": 9216, - "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 64, + "n": 4096, + "k": 2880, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/gpt-oss-120b-w-mxfp4-a-fp8/q_proj", + "openai/gpt-oss-120b/q_proj" + ] }, { "type": "gemm", "args": { - "m": 3, - "n": 7168, - "k": 9216, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 64, + "n": 4096, + "k": 4096, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "Qwen/Qwen3.5-397B-A17B/o_proj", + "Qwen/Qwen3.5-397B-A17B/q_proj", + "amd/Qwen3.5-397B-A17B-MXFP4/o_proj", + "amd/Qwen3.5-397B-A17B-MXFP4/q_proj", + "nvidia/Qwen3.5-397B-A17B-NVFP4/o_proj", + "nvidia/Qwen3.5-397B-A17B-NVFP4/q_proj" + ] }, { "type": "gemm", "args": { - "m": 2560, - "n": 9216, - "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 64, + "n": 4096, + "k": 4096, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "Qwen/Qwen3.5-397B-A17B-FP8/o_proj", + "Qwen/Qwen3.5-397B-A17B-FP8/q_proj" + ] }, { "type": "gemm", "args": { - "m": 2560, - "n": 7168, - "k": 9216, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 64, + "n": 4608, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "moonshotai/Kimi-K2.5/gate_up_proj", + "nvidia/Kimi-K2.6-NVFP4/gate_up_proj" + ] }, { "type": "gemm", "args": { - "m": 2, - "n": 9216, + "m": 64, + "n": 4608, "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/gate_up_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/gate_up_proj", + "nvidia/Kimi-K2.5-NVFP4/gate_up_proj" + ] }, { "type": "gemm", "args": { - "m": 2, - "n": 7168, - "k": 9216, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 64, + "n": 4608, + "k": 7168, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 32 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 1, + 32 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "amd/DeepSeek-R1-0528-MXFP4-Preview/gate_up_proj", + "amd/DeepSeek-R1-0528-MXFP4-v2/gate_up_proj", + "amd/DeepSeek-R1-0528-MXFP4/gate_up_proj", + "amd/Kimi-K2.5-MXFP4/gate_up_proj" + ] }, { "type": "gemm", "args": { - "m": 5, - "n": 9216, + "m": 64, + "n": 4608, "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/gate_up_proj" + ] }, { "type": "gemm", "args": { - "m": 5, - "n": 7168, - "k": 9216, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 64, + "n": 6144, + "k": 1536, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/GLM-5.1-MXFP4/down_proj", + "amd/GLM-5.2-MXFP4/down_proj", + "amd/Kimi-K2.5-MXFP4/q_b_proj", + "amd/MiniMax-M3-MXFP4/down_proj", + "moonshotai/Kimi-K2.5/q_b_proj", + "nvidia/DeepSeek-R1-0528-FP4-V2/q_b_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/q_b_proj", + "nvidia/GLM-5.1-NVFP4/down_proj", + "nvidia/GLM-5.2-NVFP4/down_proj", + "nvidia/Kimi-K2.5-NVFP4/q_b_proj", + "nvidia/Kimi-K2.6-NVFP4/q_b_proj" + ] }, { "type": "gemm", "args": { - "m": 7680, - "n": 1536, - "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 64, + "n": 6144, + "k": 1536, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 32 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 1, + 32 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "amd/DeepSeek-R1-0528-MXFP4-Preview/q_b_proj", + "amd/DeepSeek-R1-0528-MXFP4-v2/q_b_proj", + "amd/DeepSeek-R1-0528-MXFP4/q_b_proj" + ] }, { "type": "gemm", "args": { - "m": 7680, - "n": 24576, + "m": 64, + "n": 6144, "k": 1536, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/q_b_proj", + "zai-org/GLM-5-FP8/down_proj", + "zai-org/GLM-5.1-FP8/down_proj", + "zai-org/GLM-5.2-FP8/down_proj" + ] }, { "type": "gemm", "args": { - "m": 7680, - "n": 576, - "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 64, + "n": 6144, + "k": 2048, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "zai-org/GLM-5-FP8/o_proj", + "zai-org/GLM-5.1-FP8/o_proj", + "zai-org/GLM-5.2-FP8/o_proj" + ] }, { "type": "gemm", "args": { - "m": 7680, - "n": 32768, - "k": 512, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 64, + "n": 6144, + "k": 3072, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/GLM-5.1-MXFP4/down_proj", + "amd/GLM-5.2-MXFP4/down_proj", + "amd/MiniMax-M2.5-MXFP4/q_proj", + "amd/MiniMax-M3-MXFP4/down_proj", + "nvidia/GLM-5.1-NVFP4/down_proj", + "nvidia/GLM-5.2-NVFP4/down_proj", + "nvidia/MiniMax-M2.5-NVFP4/q_proj" + ] }, { "type": "gemm", "args": { - "m": 7680, - "n": 7168, - "k": 16384, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 64, + "n": 6144, + "k": 3072, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/GLM-5-NVFP4/down_proj" + ] }, { "type": "gemm", "args": { - "m": 7680, - "n": 9216, - "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 64, + "n": 6144, + "k": 3072, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "MiniMaxAI/MiniMax-M2.5/q_proj", + "zai-org/GLM-5-FP8/down_proj", + "zai-org/GLM-5.1-FP8/down_proj", + "zai-org/GLM-5.2-FP8/down_proj" + ] }, { "type": "gemm", "args": { - "m": 7680, - "n": 7168, - "k": 9216, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 64, + "n": 6144, + "k": 4096, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/GLM-5.1-MXFP4/o_proj", + "amd/GLM-5.2-MXFP4/o_proj", + "amd/MiniMax-M3-MXFP4/o_proj", + "nvidia/GLM-5-NVFP4/o_proj", + "nvidia/GLM-5.1-NVFP4/o_proj", + "nvidia/GLM-5.2-NVFP4/o_proj" + ] }, { "type": "gemm", "args": { - "m": 7, - "n": 9216, - "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 64, + "n": 6144, + "k": 4096, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "zai-org/GLM-5-FP8/o_proj", + "zai-org/GLM-5.1-FP8/o_proj", + "zai-org/GLM-5.2-FP8/o_proj" + ] }, { "type": "gemm", "args": { - "m": 7, - "n": 7168, - "k": 9216, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 64, + "n": 6144, + "k": 6144, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/GLM-5.1-MXFP4/down_proj", + "amd/GLM-5.1-MXFP4/gate_up_proj", + "amd/GLM-5.2-MXFP4/down_proj", + "amd/GLM-5.2-MXFP4/gate_up_proj", + "amd/MiniMax-M3-MXFP4/gate_up_proj", + "nvidia/GLM-5.1-NVFP4/down_proj", + "nvidia/GLM-5.1-NVFP4/gate_up_proj", + "nvidia/GLM-5.2-NVFP4/down_proj", + "nvidia/GLM-5.2-NVFP4/gate_up_proj" + ] }, { "type": "gemm", "args": { - "m": 12, - "n": 9216, - "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 64, + "n": 6144, + "k": 6144, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/GLM-5-NVFP4/down_proj", + "nvidia/GLM-5-NVFP4/gate_up_proj" + ] }, { "type": "gemm", "args": { - "m": 12, - "n": 7168, - "k": 9216, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 64, + "n": 6144, + "k": 6144, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "zai-org/GLM-5-FP8/down_proj", + "zai-org/GLM-5-FP8/gate_up_proj", + "zai-org/GLM-5.1-FP8/down_proj", + "zai-org/GLM-5.1-FP8/gate_up_proj", + "zai-org/GLM-5.2-FP8/down_proj", + "zai-org/GLM-5.2-FP8/gate_up_proj" + ] }, { "type": "gemm", "args": { - "m": 5120, + "m": 64, "n": 6144, - "k": 1536, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "k": 8192, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/GLM-5.1-MXFP4/o_proj", + "amd/GLM-5.2-MXFP4/o_proj", + "amd/MiniMax-M3-MXFP4/o_proj", + "nvidia/GLM-5-NVFP4/o_proj", + "nvidia/GLM-5.1-NVFP4/o_proj", + "nvidia/GLM-5.2-NVFP4/o_proj" + ] }, { "type": "gemm", "args": { - "m": 5120, - "n": 8192, + "m": 64, + "n": 7168, "k": 512, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/GLM-5.1-MXFP4/kv_b_proj", + "amd/GLM-5.2-MXFP4/kv_b_proj", + "amd/Kimi-K2.5-MXFP4/o_proj", + "moonshotai/Kimi-K2.5/o_proj", + "nvidia/GLM-5-NVFP4/kv_b_proj", + "nvidia/GLM-5.1-NVFP4/kv_b_proj", + "nvidia/GLM-5.2-NVFP4/kv_b_proj", + "nvidia/Kimi-K2.5-NVFP4/o_proj", + "nvidia/Kimi-K2.6-NVFP4/o_proj" + ] }, { "type": "gemm", "args": { - "m": 5120, + "m": 64, "n": 7168, - "k": 4096, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "k": 512, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "zai-org/GLM-5-FP8/kv_b_proj", + "zai-org/GLM-5.1-FP8/kv_b_proj", + "zai-org/GLM-5.2-FP8/kv_b_proj" + ] }, { "type": "gemm", "args": { - "m": 5120, - "n": 4608, - "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 64, + "n": 7168, + "k": 576, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-NVFP4-v2/down_proj" + ] }, { "type": "gemm", "args": { - "m": 5120, + "m": 64, "n": 7168, - "k": 4608, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "k": 576, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/down_proj" + ] }, { "type": "gemm", "args": { - "m": 5, - "n": 6144, - "k": 1536, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 64, + "n": 7168, + "k": 1024, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/Kimi-K2.5-MXFP4/o_proj", + "moonshotai/Kimi-K2.5/o_proj", + "nvidia/Kimi-K2.5-NVFP4/o_proj", + "nvidia/Kimi-K2.6-NVFP4/o_proj" + ] }, { "type": "gemm", "args": { - "m": 5, - "n": 8192, - "k": 512, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 64, + "n": 7168, + "k": 1152, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/down_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/down_proj", + "nvidia/Kimi-K2.5-NVFP4/down_proj" + ] }, { "type": "gemm", "args": { - "m": 5, + "m": 64, "n": 7168, - "k": 4096, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "k": 2048, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/Kimi-K2.5-MXFP4/o_proj", + "moonshotai/Kimi-K2.5/o_proj", + "nvidia/Kimi-K2.5-NVFP4/o_proj", + "nvidia/Kimi-K2.6-NVFP4/o_proj" + ] }, { "type": "gemm", "args": { - "m": 10240, - "n": 6144, - "k": 1536, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 64, + "n": 7168, + "k": 2048, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/o_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/o_proj" + ] }, { "type": "gemm", "args": { - "m": 10240, - "n": 8192, - "k": 512, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 64, + "n": 7168, + "k": 2048, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 32 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 1, + 32 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "amd/DeepSeek-R1-0528-MXFP4-Preview/o_proj", + "amd/DeepSeek-R1-0528-MXFP4-v2/o_proj", + "amd/DeepSeek-R1-0528-MXFP4/o_proj" + ] }, { "type": "gemm", "args": { - "m": 10240, + "m": 64, "n": 7168, - "k": 4096, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "k": 2048, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/o_proj", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] }, { "type": "gemm", "args": { - "m": 10240, - "n": 4608, - "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 64, + "n": 7168, + "k": 2304, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "moonshotai/Kimi-K2.5/down_proj", + "nvidia/Kimi-K2.6-NVFP4/down_proj" + ] }, { "type": "gemm", "args": { - "m": 10240, + "m": 64, "n": 7168, - "k": 4608, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "k": 2304, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/down_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/down_proj", + "nvidia/Kimi-K2.5-NVFP4/down_proj" + ] }, { "type": "gemm", "args": { - "m": 10, - "n": 6144, - "k": 1536, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 64, + "n": 7168, + "k": 2304, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 32 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 1, + 32 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "amd/DeepSeek-R1-0528-MXFP4-Preview/down_proj", + "amd/DeepSeek-R1-0528-MXFP4-v2/down_proj", + "amd/DeepSeek-R1-0528-MXFP4/down_proj", + "amd/Kimi-K2.5-MXFP4/down_proj" + ] }, { "type": "gemm", "args": { - "m": 10, - "n": 8192, - "k": 512, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 64, + "n": 7168, + "k": 2304, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/down_proj" + ] }, { "type": "gemm", "args": { - "m": 10, + "m": 64, "n": 7168, "k": 4096, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/o_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/o_proj" + ] }, { "type": "gemm", "args": { - "m": 10, - "n": 4608, - "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 64, + "n": 7168, + "k": 4096, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 32 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 1, + 32 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "amd/DeepSeek-R1-0528-MXFP4-Preview/o_proj", + "amd/DeepSeek-R1-0528-MXFP4-v2/o_proj", + "amd/DeepSeek-R1-0528-MXFP4/o_proj" + ] }, { "type": "gemm", "args": { - "m": 10, + "m": 64, "n": 7168, - "k": 4608, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "k": 4096, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/o_proj", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] }, { "type": "gemm", "args": { - "m": 15360, - "n": 6144, - "k": 1536, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 64, + "n": 7168, + "k": 4608, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "moonshotai/Kimi-K2.5/down_proj", + "nvidia/Kimi-K2.6-NVFP4/down_proj" + ] }, { "type": "gemm", "args": { - "m": 15360, - "n": 8192, - "k": 512, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 64, + "n": 7168, + "k": 4608, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/down_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/down_proj", + "nvidia/Kimi-K2.5-NVFP4/down_proj" + ] }, { "type": "gemm", "args": { - "m": 15360, + "m": 64, "n": 7168, - "k": 4096, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "k": 4608, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 32 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 1, + 32 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "amd/DeepSeek-R1-0528-MXFP4-Preview/down_proj", + "amd/DeepSeek-R1-0528-MXFP4-v2/down_proj", + "amd/DeepSeek-R1-0528-MXFP4/down_proj", + "amd/Kimi-K2.5-MXFP4/down_proj" + ] }, { "type": "gemm", "args": { - "m": 15360, - "n": 4608, - "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 64, + "n": 7168, + "k": 4608, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/down_proj" + ] }, { "type": "gemm", "args": { - "m": 15360, + "m": 64, "n": 7168, - "k": 4608, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "k": 8192, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/Kimi-K2.5-MXFP4/o_proj", + "moonshotai/Kimi-K2.5/o_proj", + "nvidia/Kimi-K2.5-NVFP4/o_proj", + "nvidia/Kimi-K2.6-NVFP4/o_proj" + ] }, { "type": "gemm", "args": { - "m": 15, - "n": 6144, - "k": 1536, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 64, + "n": 7168, + "k": 8192, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] }, { "type": "gemm", "args": { - "m": 15, - "n": 8192, - "k": 512, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 64, + "n": 7168, + "k": 16384, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/o_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/o_proj" + ] }, { "type": "gemm", "args": { - "m": 15, + "m": 64, "n": 7168, - "k": 4096, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "k": 16384, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/o_proj", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] }, { "type": "gemm", "args": { - "m": 25600, - "n": 6144, - "k": 1536, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 64, + "n": 7168, + "k": 16384, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] }, { "type": "gemm", "args": { - "m": 25600, + "m": 64, + "n": 7168, + "k": 65536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 64, "n": 8192, "k": 512, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/Kimi-K2.5-MXFP4/kv_b_proj", + "moonshotai/Kimi-K2.5/kv_b_proj", + "nvidia/DeepSeek-R1-0528-FP4-V2/kv_b_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/kv_b_proj", + "nvidia/Kimi-K2.5-NVFP4/kv_b_proj", + "nvidia/Kimi-K2.6-NVFP4/kv_b_proj" + ] }, { "type": "gemm", "args": { - "m": 25600, - "n": 7168, - "k": 4096, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 64, + "n": 8192, + "k": 512, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 32 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 1, + 32 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "amd/DeepSeek-R1-0528-MXFP4-Preview/kv_b_proj", + "amd/DeepSeek-R1-0528-MXFP4-v2/kv_b_proj", + "amd/DeepSeek-R1-0528-MXFP4/kv_b_proj" + ] }, { "type": "gemm", "args": { - "m": 25600, - "n": 4608, - "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 64, + "n": 8192, + "k": 512, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/kv_b_proj" + ] }, { "type": "gemm", "args": { - "m": 25600, - "n": 7168, - "k": 4608, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 64, + "n": 8192, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_b" + ] }, { "type": "gemm", "args": { - "m": 25, - "n": 6144, + "m": 64, + "n": 8192, + "k": 2048, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/GLM-5.1-MXFP4/q_b_proj", + "amd/GLM-5.2-MXFP4/q_b_proj", + "nvidia/GLM-5-NVFP4/q_b_proj", + "nvidia/GLM-5.1-NVFP4/q_b_proj", + "nvidia/GLM-5.2-NVFP4/q_b_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 64, + "n": 12288, "k": 1536, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/Kimi-K2.5-MXFP4/q_b_proj", + "moonshotai/Kimi-K2.5/q_b_proj", + "nvidia/DeepSeek-R1-0528-FP4-V2/q_b_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/q_b_proj", + "nvidia/Kimi-K2.5-NVFP4/q_b_proj", + "nvidia/Kimi-K2.6-NVFP4/q_b_proj" + ] }, { "type": "gemm", "args": { - "m": 25, - "n": 8192, + "m": 64, + "n": 14336, "k": 512, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/GLM-5.1-MXFP4/kv_b_proj", + "amd/GLM-5.2-MXFP4/kv_b_proj", + "nvidia/GLM-5-NVFP4/kv_b_proj", + "nvidia/GLM-5.1-NVFP4/kv_b_proj", + "nvidia/GLM-5.2-NVFP4/kv_b_proj" + ] }, { "type": "gemm", "args": { - "m": 25, - "n": 7168, - "k": 4096, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 64, + "n": 15360, + "k": 1024, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] }, { "type": "gemm", "args": { - "m": 30, - "n": 9216, - "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 64, + "n": 16384, + "k": 512, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/Kimi-K2.5-MXFP4/kv_b_proj", + "moonshotai/Kimi-K2.5/kv_b_proj", + "nvidia/DeepSeek-R1-0528-FP4-V2/kv_b_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/kv_b_proj", + "nvidia/Kimi-K2.5-NVFP4/kv_b_proj", + "nvidia/Kimi-K2.6-NVFP4/kv_b_proj" + ] }, { "type": "gemm", "args": { - "m": 30, - "n": 7168, - "k": 9216, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 64, + "n": 16384, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_b" + ] }, { "type": "gemm", "args": { - "m": 56320, - "n": 1536, - "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 64, + "n": 24576, + "k": 1536, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/q_b_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/q_b_proj" + ] }, { "type": "gemm", "args": { - "m": 56320, + "m": 64, "n": 24576, "k": 1536, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/q_b_proj" + ] }, { "type": "gemm", "args": { - "m": 56320, - "n": 576, - "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 64, + "n": 30720, + "k": 1024, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] }, { "type": "gemm", "args": { - "m": 56320, + "m": 64, "n": 32768, "k": 512, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/kv_b_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/kv_b_proj" + ] }, { "type": "gemm", "args": { - "m": 56320, - "n": 7168, - "k": 16384, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 64, + "n": 32768, + "k": 512, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/kv_b_proj" + ] }, { "type": "gemm", "args": { - "m": 56320, - "n": 9216, - "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 64, + "n": 65536, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_b" + ] }, { "type": "gemm", "args": { - "m": 56320, - "n": 7168, - "k": 9216, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 64, + "n": 122880, + "k": 1024, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] }, { "type": "gemm", "args": { - "m": 55, - "n": 1536, + "m": 66, + "n": 576, "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/Kimi-K2.5-MXFP4/kv_a_proj_with_mqa", + "moonshotai/Kimi-K2.5/kv_a_proj_with_mqa", + "nvidia/DeepSeek-R1-0528-FP4-V2/kv_a_proj_with_mqa", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/kv_a_proj_with_mqa", + "nvidia/Kimi-K2.5-NVFP4/kv_a_proj_with_mqa", + "nvidia/Kimi-K2.6-NVFP4/kv_a_proj_with_mqa" + ] }, { "type": "gemm", "args": { - "m": 55, - "n": 24576, - "k": 1536, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 66, + "n": 1536, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/Kimi-K2.5-MXFP4/q_a_proj", + "moonshotai/Kimi-K2.5/q_a_proj", + "nvidia/DeepSeek-R1-0528-FP4-V2/q_a_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/q_a_proj", + "nvidia/Kimi-K2.5-NVFP4/q_a_proj", + "nvidia/Kimi-K2.6-NVFP4/q_a_proj" + ] }, { "type": "gemm", "args": { - "m": 55, - "n": 576, + "m": 66, + "n": 2304, "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/gate_up_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/gate_up_proj", + "nvidia/Kimi-K2.5-NVFP4/gate_up_proj" + ] }, { "type": "gemm", "args": { - "m": 55, - "n": 32768, - "k": 512, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 66, + "n": 3072, + "k": 1536, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/Kimi-K2.5-MXFP4/q_b_proj", + "amd/MiniMax-M2.5-MXFP4/o_proj", + "moonshotai/Kimi-K2.5/q_b_proj", + "nvidia/DeepSeek-R1-0528-FP4-V2/q_b_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/q_b_proj", + "nvidia/Kimi-K2.5-NVFP4/q_b_proj", + "nvidia/Kimi-K2.6-NVFP4/q_b_proj", + "nvidia/MiniMax-M2.5-NVFP4/o_proj" + ] }, { "type": "gemm", "args": { - "m": 55, - "n": 7168, - "k": 16384, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 66, + "n": 4096, + "k": 512, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "Qwen/Qwen3.5-397B-A17B/o_proj", + "Qwen/Qwen3.5-397B-A17B/shared_expert.down_proj", + "amd/Kimi-K2.5-MXFP4/kv_b_proj", + "amd/Qwen3.5-397B-A17B-MXFP4/o_proj", + "amd/Qwen3.5-397B-A17B-MXFP4/shared_expert.down_proj", + "moonshotai/Kimi-K2.5/kv_b_proj", + "nvidia/DeepSeek-R1-0528-FP4-V2/kv_b_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/kv_b_proj", + "nvidia/Kimi-K2.5-NVFP4/kv_b_proj", + "nvidia/Kimi-K2.6-NVFP4/kv_b_proj", + "nvidia/Qwen3.5-397B-A17B-NVFP4/o_proj", + "nvidia/Qwen3.5-397B-A17B-NVFP4/shared_expert.down_proj" + ] }, { "type": "gemm", "args": { - "m": 55, - "n": 9216, - "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 66, + "n": 7168, + "k": 2048, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/o_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/o_proj" + ] }, { "type": "gemm", "args": { - "m": 55, + "m": 66, "n": 7168, - "k": 9216, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "k": 2304, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/down_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/down_proj", + "nvidia/Kimi-K2.5-NVFP4/down_proj" + ] }, { "type": "gemm", "args": { - "m": 99840, - "n": 1536, + "m": 67, + "n": 576, "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/Kimi-K2.5-MXFP4/kv_a_proj_with_mqa", + "moonshotai/Kimi-K2.5/kv_a_proj_with_mqa", + "nvidia/DeepSeek-R1-0528-FP4-V2/kv_a_proj_with_mqa", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/kv_a_proj_with_mqa", + "nvidia/Kimi-K2.5-NVFP4/kv_a_proj_with_mqa", + "nvidia/Kimi-K2.6-NVFP4/kv_a_proj_with_mqa" + ] }, { "type": "gemm", "args": { - "m": 99840, - "n": 24576, - "k": 1536, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 67, + "n": 1536, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/Kimi-K2.5-MXFP4/q_a_proj", + "moonshotai/Kimi-K2.5/q_a_proj", + "nvidia/DeepSeek-R1-0528-FP4-V2/q_a_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/q_a_proj", + "nvidia/Kimi-K2.5-NVFP4/q_a_proj", + "nvidia/Kimi-K2.6-NVFP4/q_a_proj" + ] }, { "type": "gemm", "args": { - "m": 99840, - "n": 576, + "m": 67, + "n": 2304, "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/gate_up_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/gate_up_proj", + "nvidia/Kimi-K2.5-NVFP4/gate_up_proj" + ] }, { "type": "gemm", "args": { - "m": 99840, - "n": 32768, - "k": 512, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 67, + "n": 7168, + "k": 2304, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/down_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/down_proj", + "nvidia/Kimi-K2.5-NVFP4/down_proj" + ] }, { "type": "gemm", "args": { - "m": 99840, + "m": 67, "n": 7168, "k": 16384, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/o_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/o_proj" + ] }, { "type": "gemm", "args": { - "m": 99840, - "n": 9216, - "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 67, + "n": 24576, + "k": 1536, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/q_b_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/q_b_proj" + ] }, { "type": "gemm", "args": { - "m": 99840, - "n": 7168, - "k": 9216, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 67, + "n": 32768, + "k": 512, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/kv_b_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/kv_b_proj" + ] }, { "type": "gemm", "args": { - "m": 97, - "n": 1536, + "m": 70, + "n": 576, "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/Kimi-K2.5-MXFP4/kv_a_proj_with_mqa", + "moonshotai/Kimi-K2.5/kv_a_proj_with_mqa", + "nvidia/DeepSeek-R1-0528-FP4-V2/kv_a_proj_with_mqa", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/kv_a_proj_with_mqa", + "nvidia/Kimi-K2.5-NVFP4/kv_a_proj_with_mqa", + "nvidia/Kimi-K2.6-NVFP4/kv_a_proj_with_mqa" + ] }, { "type": "gemm", "args": { - "m": 97, - "n": 24576, - "k": 1536, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 70, + "n": 1536, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/Kimi-K2.5-MXFP4/q_a_proj", + "moonshotai/Kimi-K2.5/q_a_proj", + "nvidia/DeepSeek-R1-0528-FP4-V2/q_a_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/q_a_proj", + "nvidia/Kimi-K2.5-NVFP4/q_a_proj", + "nvidia/Kimi-K2.6-NVFP4/q_a_proj" + ] }, { "type": "gemm", "args": { - "m": 97, - "n": 576, - "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 70, + "n": 7168, + "k": 576, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-NVFP4-v2/down_proj" + ] }, { "type": "gemm", "args": { - "m": 97, - "n": 32768, - "k": 512, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 70, + "n": 7168, + "k": 8192, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/Kimi-K2.5-MXFP4/o_proj", + "moonshotai/Kimi-K2.5/o_proj", + "nvidia/Kimi-K2.5-NVFP4/o_proj", + "nvidia/Kimi-K2.6-NVFP4/o_proj" + ] }, { "type": "gemm", "args": { - "m": 97, + "m": 70, "n": 7168, "k": 16384, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/o_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/o_proj" + ] }, { "type": "gemm", "args": { - "m": 97, - "n": 9216, - "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 70, + "n": 12288, + "k": 1536, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/Kimi-K2.5-MXFP4/q_b_proj", + "moonshotai/Kimi-K2.5/q_b_proj", + "nvidia/DeepSeek-R1-0528-FP4-V2/q_b_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/q_b_proj", + "nvidia/Kimi-K2.5-NVFP4/q_b_proj", + "nvidia/Kimi-K2.6-NVFP4/q_b_proj" + ] }, { "type": "gemm", "args": { - "m": 97, - "n": 7168, - "k": 9216, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 70, + "n": 16384, + "k": 512, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/Kimi-K2.5-MXFP4/kv_b_proj", + "moonshotai/Kimi-K2.5/kv_b_proj", + "nvidia/DeepSeek-R1-0528-FP4-V2/kv_b_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/kv_b_proj", + "nvidia/Kimi-K2.5-NVFP4/kv_b_proj", + "nvidia/Kimi-K2.6-NVFP4/kv_b_proj" + ] }, { "type": "gemm", "args": { - "m": 202240, - "n": 1536, - "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 70, + "n": 24576, + "k": 1536, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/q_b_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/q_b_proj" + ] }, { "type": "gemm", "args": { - "m": 202240, - "n": 24576, - "k": 1536, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 70, + "n": 32768, + "k": 512, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/kv_b_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/kv_b_proj" + ] }, { "type": "gemm", "args": { - "m": 202240, + "m": 72, "n": 576, "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/Kimi-K2.5-MXFP4/kv_a_proj_with_mqa", + "moonshotai/Kimi-K2.5/kv_a_proj_with_mqa", + "nvidia/DeepSeek-R1-0528-FP4-V2/kv_a_proj_with_mqa", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/kv_a_proj_with_mqa", + "nvidia/Kimi-K2.5-NVFP4/kv_a_proj_with_mqa", + "nvidia/Kimi-K2.6-NVFP4/kv_a_proj_with_mqa" + ] }, { "type": "gemm", "args": { - "m": 202240, - "n": 32768, - "k": 512, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 72, + "n": 576, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/kv_a_proj_with_mqa" + ] }, { "type": "gemm", "args": { - "m": 202240, - "n": 7168, - "k": 16384, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 72, + "n": 1536, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/Kimi-K2.5-MXFP4/q_a_proj", + "moonshotai/Kimi-K2.5/q_a_proj", + "nvidia/DeepSeek-R1-0528-FP4-V2/q_a_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/q_a_proj", + "nvidia/Kimi-K2.5-NVFP4/q_a_proj", + "nvidia/Kimi-K2.6-NVFP4/q_a_proj" + ] }, { "type": "gemm", "args": { - "m": 202240, - "n": 9216, + "m": 72, + "n": 1536, "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/gate_up_proj", + "deepseek-ai/DeepSeek-R1-0528/q_a_proj", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w3" + ] }, { "type": "gemm", "args": { - "m": 202240, - "n": 7168, - "k": 9216, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 72, + "n": 2304, + "k": 7168, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/gate_up_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/gate_up_proj", + "nvidia/Kimi-K2.5-NVFP4/gate_up_proj" + ] }, { "type": "gemm", "args": { - "m": 197, - "n": 1536, + "m": 72, + "n": 2304, "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/gate_up_proj" + ] }, { "type": "gemm", "args": { - "m": 197, - "n": 24576, + "m": 72, + "n": 3072, "k": 1536, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/Kimi-K2.5-MXFP4/q_b_proj", + "amd/MiniMax-M2.5-MXFP4/o_proj", + "moonshotai/Kimi-K2.5/q_b_proj", + "nvidia/DeepSeek-R1-0528-FP4-V2/q_b_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/q_b_proj", + "nvidia/Kimi-K2.5-NVFP4/q_b_proj", + "nvidia/Kimi-K2.6-NVFP4/q_b_proj", + "nvidia/MiniMax-M2.5-NVFP4/o_proj" + ] }, { "type": "gemm", "args": { - "m": 197, - "n": 576, + "m": 72, + "n": 4096, + "k": 512, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "Qwen/Qwen3.5-397B-A17B/o_proj", + "Qwen/Qwen3.5-397B-A17B/shared_expert.down_proj", + "amd/Kimi-K2.5-MXFP4/kv_b_proj", + "amd/Qwen3.5-397B-A17B-MXFP4/o_proj", + "amd/Qwen3.5-397B-A17B-MXFP4/shared_expert.down_proj", + "moonshotai/Kimi-K2.5/kv_b_proj", + "nvidia/DeepSeek-R1-0528-FP4-V2/kv_b_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/kv_b_proj", + "nvidia/Kimi-K2.5-NVFP4/kv_b_proj", + "nvidia/Kimi-K2.6-NVFP4/kv_b_proj", + "nvidia/Qwen3.5-397B-A17B-NVFP4/o_proj", + "nvidia/Qwen3.5-397B-A17B-NVFP4/shared_expert.down_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 72, + "n": 4608, "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/gate_up_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/gate_up_proj", + "nvidia/Kimi-K2.5-NVFP4/gate_up_proj" + ] }, { "type": "gemm", "args": { - "m": 197, - "n": 32768, - "k": 512, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 72, + "n": 4608, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/gate_up_proj" + ] }, { "type": "gemm", "args": { - "m": 197, + "m": 72, "n": 7168, - "k": 16384, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "k": 2048, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/o_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/o_proj" + ] }, { "type": "gemm", "args": { - "m": 197, - "n": 9216, - "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 72, + "n": 7168, + "k": 2304, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/down_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/down_proj", + "nvidia/Kimi-K2.5-NVFP4/down_proj" + ] }, { "type": "gemm", "args": { - "m": 197, + "m": 72, "n": 7168, - "k": 9216, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "k": 2304, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/down_proj" + ] }, { "type": "gemm", "args": { - "m": 112640, - "n": 1536, - "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 72, + "n": 7168, + "k": 4608, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/down_proj" + ] }, { "type": "gemm", "args": { - "m": 112640, - "n": 3072, - "k": 1536, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 72, + "n": 7168, + "k": 16384, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/o_proj", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] }, { "type": "gemm", "args": { - "m": 112640, - "n": 576, - "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 72, + "n": 24576, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/q_b_proj" + ] }, { "type": "gemm", "args": { - "m": 112640, - "n": 4096, + "m": 72, + "n": 32768, "k": 512, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/kv_b_proj" + ] }, { "type": "gemm", "args": { - "m": 112640, - "n": 7168, - "k": 2048, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 73, + "n": 576, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/Kimi-K2.5-MXFP4/kv_a_proj_with_mqa", + "moonshotai/Kimi-K2.5/kv_a_proj_with_mqa", + "nvidia/DeepSeek-R1-0528-FP4-V2/kv_a_proj_with_mqa", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/kv_a_proj_with_mqa", + "nvidia/Kimi-K2.5-NVFP4/kv_a_proj_with_mqa", + "nvidia/Kimi-K2.6-NVFP4/kv_a_proj_with_mqa" + ] + }, + { + "type": "gemm", + "args": { + "m": 73, + "n": 1536, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/Kimi-K2.5-MXFP4/q_a_proj", + "moonshotai/Kimi-K2.5/q_a_proj", + "nvidia/DeepSeek-R1-0528-FP4-V2/q_a_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/q_a_proj", + "nvidia/Kimi-K2.5-NVFP4/q_a_proj", + "nvidia/Kimi-K2.6-NVFP4/q_a_proj" + ] }, { "type": "gemm", "args": { - "m": 112640, + "m": 73, "n": 2304, "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/gate_up_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/gate_up_proj", + "nvidia/Kimi-K2.5-NVFP4/gate_up_proj" + ] }, { "type": "gemm", "args": { - "m": 112640, + "m": 73, "n": 7168, "k": 2304, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/down_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/down_proj", + "nvidia/Kimi-K2.5-NVFP4/down_proj" + ] }, { "type": "gemm", "args": { - "m": 110, - "n": 1536, - "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 73, + "n": 7168, + "k": 16384, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/o_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/o_proj" + ] }, { "type": "gemm", "args": { - "m": 110, - "n": 3072, + "m": 73, + "n": 24576, "k": 1536, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/q_b_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/q_b_proj" + ] }, { "type": "gemm", "args": { - "m": 110, - "n": 576, - "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 73, + "n": 32768, + "k": 512, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/kv_b_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/kv_b_proj" + ] }, { "type": "gemm", "args": { - "m": 110, - "n": 4096, - "k": 512, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 76, + "n": 576, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/Kimi-K2.5-MXFP4/kv_a_proj_with_mqa", + "moonshotai/Kimi-K2.5/kv_a_proj_with_mqa", + "nvidia/DeepSeek-R1-0528-FP4-V2/kv_a_proj_with_mqa", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/kv_a_proj_with_mqa", + "nvidia/Kimi-K2.5-NVFP4/kv_a_proj_with_mqa", + "nvidia/Kimi-K2.6-NVFP4/kv_a_proj_with_mqa" + ] }, { "type": "gemm", "args": { - "m": 110, - "n": 7168, - "k": 2048, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 76, + "n": 576, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/kv_a_proj_with_mqa" + ] }, { "type": "gemm", "args": { - "m": 110, - "n": 2304, + "m": 76, + "n": 1152, "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-NVFP4-v2/gate_up_proj" + ] }, { "type": "gemm", "args": { - "m": 110, - "n": 7168, - "k": 2304, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 76, + "n": 1536, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/Kimi-K2.5-MXFP4/q_a_proj", + "moonshotai/Kimi-K2.5/q_a_proj", + "nvidia/DeepSeek-R1-0528-FP4-V2/q_a_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/q_a_proj", + "nvidia/Kimi-K2.5-NVFP4/q_a_proj", + "nvidia/Kimi-K2.6-NVFP4/q_a_proj" + ] }, { "type": "gemm", "args": { - "m": 199680, + "m": 76, "n": 1536, "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/gate_up_proj", + "deepseek-ai/DeepSeek-R1-0528/q_a_proj", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w3" + ] }, { "type": "gemm", "args": { - "m": 199680, - "n": 3072, - "k": 1536, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 76, + "n": 2304, + "k": 7168, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/gate_up_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/gate_up_proj", + "nvidia/Kimi-K2.5-NVFP4/gate_up_proj" + ] }, { "type": "gemm", "args": { - "m": 199680, - "n": 576, + "m": 76, + "n": 2304, "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/gate_up_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 76, + "n": 3072, + "k": 1536, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/Kimi-K2.5-MXFP4/q_b_proj", + "amd/MiniMax-M2.5-MXFP4/o_proj", + "moonshotai/Kimi-K2.5/q_b_proj", + "nvidia/DeepSeek-R1-0528-FP4-V2/q_b_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/q_b_proj", + "nvidia/Kimi-K2.5-NVFP4/q_b_proj", + "nvidia/Kimi-K2.6-NVFP4/q_b_proj", + "nvidia/MiniMax-M2.5-NVFP4/o_proj" + ] }, { "type": "gemm", "args": { - "m": 199680, + "m": 76, "n": 4096, "k": 512, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "Qwen/Qwen3.5-397B-A17B/o_proj", + "Qwen/Qwen3.5-397B-A17B/shared_expert.down_proj", + "amd/Kimi-K2.5-MXFP4/kv_b_proj", + "amd/Qwen3.5-397B-A17B-MXFP4/o_proj", + "amd/Qwen3.5-397B-A17B-MXFP4/shared_expert.down_proj", + "moonshotai/Kimi-K2.5/kv_b_proj", + "nvidia/DeepSeek-R1-0528-FP4-V2/kv_b_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/kv_b_proj", + "nvidia/Kimi-K2.5-NVFP4/kv_b_proj", + "nvidia/Kimi-K2.6-NVFP4/kv_b_proj", + "nvidia/Qwen3.5-397B-A17B-NVFP4/o_proj", + "nvidia/Qwen3.5-397B-A17B-NVFP4/shared_expert.down_proj" + ] }, { "type": "gemm", "args": { - "m": 199680, + "m": 76, "n": 7168, - "k": 2048, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "k": 576, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-NVFP4-v2/down_proj" + ] }, { "type": "gemm", "args": { - "m": 199680, - "n": 2304, - "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 76, + "n": 7168, + "k": 1152, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/down_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/down_proj", + "nvidia/Kimi-K2.5-NVFP4/down_proj" + ] }, { "type": "gemm", "args": { - "m": 199680, + "m": 76, "n": 7168, - "k": 2304, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "k": 1152, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/down_proj" + ] }, { "type": "gemm", "args": { - "m": 195, - "n": 1536, - "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 76, + "n": 7168, + "k": 2048, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/o_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/o_proj" + ] }, { "type": "gemm", "args": { - "m": 195, - "n": 3072, - "k": 1536, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 76, + "n": 7168, + "k": 2304, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/down_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/down_proj", + "nvidia/Kimi-K2.5-NVFP4/down_proj" + ] }, { "type": "gemm", "args": { - "m": 195, - "n": 576, - "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 76, + "n": 7168, + "k": 16384, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/o_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/o_proj" + ] }, { "type": "gemm", "args": { - "m": 195, - "n": 4096, - "k": 512, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 76, + "n": 7168, + "k": 16384, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/o_proj", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] }, { "type": "gemm", "args": { - "m": 195, - "n": 7168, - "k": 2048, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 76, + "n": 24576, + "k": 1536, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/q_b_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/q_b_proj" + ] }, { "type": "gemm", "args": { - "m": 195, - "n": 2304, - "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 76, + "n": 24576, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/q_b_proj" + ] }, { "type": "gemm", "args": { - "m": 195, - "n": 7168, - "k": 2304, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 76, + "n": 32768, + "k": 512, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/kv_b_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/kv_b_proj" + ] }, { "type": "gemm", "args": { - "m": 404480, - "n": 1536, - "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 76, + "n": 32768, + "k": 512, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/kv_b_proj" + ] }, { "type": "gemm", "args": { - "m": 404480, - "n": 3072, - "k": 1536, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 80, + "n": 576, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/Kimi-K2.5-MXFP4/kv_a_proj_with_mqa", + "moonshotai/Kimi-K2.5/kv_a_proj_with_mqa", + "nvidia/DeepSeek-R1-0528-FP4-V2/kv_a_proj_with_mqa", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/kv_a_proj_with_mqa", + "nvidia/Kimi-K2.5-NVFP4/kv_a_proj_with_mqa", + "nvidia/Kimi-K2.6-NVFP4/kv_a_proj_with_mqa" + ] }, { "type": "gemm", "args": { - "m": 404480, + "m": 80, "n": 576, "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/kv_a_proj_with_mqa" + ] }, { "type": "gemm", "args": { - "m": 404480, - "n": 4096, - "k": 512, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 80, + "n": 1536, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/Kimi-K2.5-MXFP4/q_a_proj", + "moonshotai/Kimi-K2.5/q_a_proj", + "nvidia/DeepSeek-R1-0528-FP4-V2/q_a_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/q_a_proj", + "nvidia/Kimi-K2.5-NVFP4/q_a_proj", + "nvidia/Kimi-K2.6-NVFP4/q_a_proj" + ] }, { "type": "gemm", "args": { - "m": 404480, - "n": 7168, - "k": 2048, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 80, + "n": 1536, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/gate_up_proj", + "deepseek-ai/DeepSeek-R1-0528/q_a_proj", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w3" + ] }, { "type": "gemm", "args": { - "m": 404480, + "m": 80, "n": 2304, "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/gate_up_proj" + ] }, { "type": "gemm", "args": { - "m": 404480, - "n": 7168, - "k": 2304, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 80, + "n": 3072, + "k": 1536, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/Kimi-K2.5-MXFP4/q_b_proj", + "amd/MiniMax-M2.5-MXFP4/o_proj", + "moonshotai/Kimi-K2.5/q_b_proj", + "nvidia/DeepSeek-R1-0528-FP4-V2/q_b_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/q_b_proj", + "nvidia/Kimi-K2.5-NVFP4/q_b_proj", + "nvidia/Kimi-K2.6-NVFP4/q_b_proj", + "nvidia/MiniMax-M2.5-NVFP4/o_proj" + ] }, { "type": "gemm", "args": { - "m": 395, - "n": 1536, - "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 80, + "n": 4096, + "k": 512, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "Qwen/Qwen3.5-397B-A17B/o_proj", + "Qwen/Qwen3.5-397B-A17B/shared_expert.down_proj", + "amd/Kimi-K2.5-MXFP4/kv_b_proj", + "amd/Qwen3.5-397B-A17B-MXFP4/o_proj", + "amd/Qwen3.5-397B-A17B-MXFP4/shared_expert.down_proj", + "moonshotai/Kimi-K2.5/kv_b_proj", + "nvidia/DeepSeek-R1-0528-FP4-V2/kv_b_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/kv_b_proj", + "nvidia/Kimi-K2.5-NVFP4/kv_b_proj", + "nvidia/Kimi-K2.6-NVFP4/kv_b_proj", + "nvidia/Qwen3.5-397B-A17B-NVFP4/o_proj", + "nvidia/Qwen3.5-397B-A17B-NVFP4/shared_expert.down_proj" + ] }, { "type": "gemm", "args": { - "m": 395, - "n": 3072, - "k": 1536, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 80, + "n": 4608, + "k": 7168, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/gate_up_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/gate_up_proj", + "nvidia/Kimi-K2.5-NVFP4/gate_up_proj" + ] }, { "type": "gemm", "args": { - "m": 395, - "n": 576, + "m": 80, + "n": 4608, "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/gate_up_proj" + ] }, { "type": "gemm", "args": { - "m": 395, - "n": 4096, - "k": 512, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 80, + "n": 7168, + "k": 2048, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/o_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/o_proj" + ] }, { "type": "gemm", "args": { - "m": 395, + "m": 80, "n": 7168, - "k": 2048, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "k": 2304, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/down_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/down_proj", + "nvidia/Kimi-K2.5-NVFP4/down_proj" + ] }, { "type": "gemm", "args": { - "m": 395, - "n": 2304, - "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 80, + "n": 7168, + "k": 2304, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/down_proj" + ] }, { "type": "gemm", "args": { - "m": 395, + "m": 80, "n": 7168, - "k": 2304, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "k": 4608, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/down_proj" + ] }, { "type": "gemm", "args": { - "m": 2255360, - "n": 1536, - "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 80, + "n": 7168, + "k": 16384, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/o_proj", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] }, { "type": "gemm", "args": { - "m": 2255360, + "m": 80, "n": 24576, "k": 1536, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/q_b_proj" + ] }, { "type": "gemm", "args": { - "m": 2255360, - "n": 576, - "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 80, + "n": 32768, + "k": 512, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/kv_b_proj" + ] }, { "type": "gemm", "args": { - "m": 2255360, - "n": 32768, - "k": 512, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 83, + "n": 576, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/Kimi-K2.5-MXFP4/kv_a_proj_with_mqa", + "moonshotai/Kimi-K2.5/kv_a_proj_with_mqa", + "nvidia/DeepSeek-R1-0528-FP4-V2/kv_a_proj_with_mqa", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/kv_a_proj_with_mqa", + "nvidia/Kimi-K2.5-NVFP4/kv_a_proj_with_mqa", + "nvidia/Kimi-K2.6-NVFP4/kv_a_proj_with_mqa" + ] }, { "type": "gemm", "args": { - "m": 2255360, - "n": 7168, - "k": 16384, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 83, + "n": 576, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/kv_a_proj_with_mqa" + ] }, { "type": "gemm", "args": { - "m": 2255360, - "n": 9216, + "m": 83, + "n": 1152, "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-NVFP4-v2/gate_up_proj" + ] }, { "type": "gemm", "args": { - "m": 2255360, - "n": 7168, - "k": 9216, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 83, + "n": 1152, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/gate_up_proj" + ] }, { "type": "gemm", "args": { - "m": 2202, + "m": 83, "n": 1536, "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/Kimi-K2.5-MXFP4/q_a_proj", + "moonshotai/Kimi-K2.5/q_a_proj", + "nvidia/DeepSeek-R1-0528-FP4-V2/q_a_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/q_a_proj", + "nvidia/Kimi-K2.5-NVFP4/q_a_proj", + "nvidia/Kimi-K2.6-NVFP4/q_a_proj" + ] }, { "type": "gemm", "args": { - "m": 2202, - "n": 24576, - "k": 1536, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 83, + "n": 1536, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/gate_up_proj", + "deepseek-ai/DeepSeek-R1-0528/q_a_proj", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w3" + ] }, { "type": "gemm", "args": { - "m": 2202, - "n": 576, + "m": 83, + "n": 2304, "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/gate_up_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/gate_up_proj", + "nvidia/Kimi-K2.5-NVFP4/gate_up_proj" + ] }, { "type": "gemm", "args": { - "m": 2202, - "n": 32768, - "k": 512, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 83, + "n": 2304, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/gate_up_proj" + ] }, { "type": "gemm", "args": { - "m": 2202, - "n": 7168, - "k": 16384, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 83, + "n": 4608, + "k": 7168, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/gate_up_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/gate_up_proj", + "nvidia/Kimi-K2.5-NVFP4/gate_up_proj" + ] }, { "type": "gemm", "args": { - "m": 2202, - "n": 9216, + "m": 83, + "n": 4608, "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 32 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 1, + 32 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "amd/DeepSeek-R1-0528-MXFP4-Preview/gate_up_proj", + "amd/DeepSeek-R1-0528-MXFP4-v2/gate_up_proj", + "amd/DeepSeek-R1-0528-MXFP4/gate_up_proj", + "amd/Kimi-K2.5-MXFP4/gate_up_proj" + ] }, { "type": "gemm", "args": { - "m": 2202, + "m": 83, "n": 7168, - "k": 9216, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "k": 1152, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/down_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/down_proj", + "nvidia/Kimi-K2.5-NVFP4/down_proj" + ] }, { "type": "gemm", "args": { - "m": 563840, - "n": 1536, - "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 83, + "n": 7168, + "k": 1152, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/down_proj" + ] }, { "type": "gemm", "args": { - "m": 563840, - "n": 24576, - "k": 1536, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 83, + "n": 7168, + "k": 2304, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/down_proj" + ] }, { "type": "gemm", "args": { - "m": 563840, - "n": 576, - "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 83, + "n": 7168, + "k": 4608, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/down_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/down_proj", + "nvidia/Kimi-K2.5-NVFP4/down_proj" + ] }, { "type": "gemm", "args": { - "m": 563840, - "n": 32768, - "k": 512, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 83, + "n": 7168, + "k": 4608, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 32 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 1, + 32 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "amd/DeepSeek-R1-0528-MXFP4-Preview/down_proj", + "amd/DeepSeek-R1-0528-MXFP4-v2/down_proj", + "amd/DeepSeek-R1-0528-MXFP4/down_proj", + "amd/Kimi-K2.5-MXFP4/down_proj" + ] }, { "type": "gemm", "args": { - "m": 563840, + "m": 83, "n": 7168, - "k": 16384, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "k": 8192, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/Kimi-K2.5-MXFP4/o_proj", + "moonshotai/Kimi-K2.5/o_proj", + "nvidia/Kimi-K2.5-NVFP4/o_proj", + "nvidia/Kimi-K2.6-NVFP4/o_proj" + ] }, { "type": "gemm", "args": { - "m": 563840, - "n": 2304, - "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 83, + "n": 7168, + "k": 8192, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] }, { "type": "gemm", "args": { - "m": 563840, + "m": 83, "n": 7168, - "k": 2304, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "k": 16384, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/o_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/o_proj" + ] }, { "type": "gemm", "args": { - "m": 550, - "n": 1536, - "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 83, + "n": 7168, + "k": 16384, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/o_proj", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] }, { "type": "gemm", "args": { - "m": 550, - "n": 24576, + "m": 83, + "n": 12288, "k": 1536, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/Kimi-K2.5-MXFP4/q_b_proj", + "moonshotai/Kimi-K2.5/q_b_proj", + "nvidia/DeepSeek-R1-0528-FP4-V2/q_b_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/q_b_proj", + "nvidia/Kimi-K2.5-NVFP4/q_b_proj", + "nvidia/Kimi-K2.6-NVFP4/q_b_proj" + ] }, { "type": "gemm", "args": { - "m": 550, - "n": 576, - "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 83, + "n": 16384, + "k": 512, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/Kimi-K2.5-MXFP4/kv_b_proj", + "moonshotai/Kimi-K2.5/kv_b_proj", + "nvidia/DeepSeek-R1-0528-FP4-V2/kv_b_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/kv_b_proj", + "nvidia/Kimi-K2.5-NVFP4/kv_b_proj", + "nvidia/Kimi-K2.6-NVFP4/kv_b_proj" + ] }, { "type": "gemm", "args": { - "m": 550, - "n": 32768, - "k": 512, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 83, + "n": 24576, + "k": 1536, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/q_b_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/q_b_proj" + ] }, { "type": "gemm", "args": { - "m": 550, - "n": 7168, - "k": 16384, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 83, + "n": 24576, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/q_b_proj" + ] }, { "type": "gemm", "args": { - "m": 550, - "n": 2304, - "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 83, + "n": 32768, + "k": 512, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/kv_b_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/kv_b_proj" + ] }, { "type": "gemm", "args": { - "m": 550, - "n": 7168, - "k": 2304, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 83, + "n": 32768, + "k": 512, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/kv_b_proj" + ] }, { "type": "gemm", "args": { - "m": 4194304, - "n": 1536, + "m": 84, + "n": 576, "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/kv_a_proj_with_mqa" + ] }, { "type": "gemm", "args": { - "m": 4194304, - "n": 24576, - "k": 1536, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 84, + "n": 1536, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/gate_up_proj", + "deepseek-ai/DeepSeek-R1-0528/q_a_proj", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w3" + ] }, { "type": "gemm", "args": { - "m": 4194304, - "n": 576, + "m": 84, + "n": 2304, "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/gate_up_proj" + ] }, { "type": "gemm", "args": { - "m": 4194304, - "n": 32768, - "k": 512, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 84, + "n": 3072, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "MiniMaxAI/MiniMax-M2.5/experts.down_proj", + "MiniMaxAI/MiniMax-M2.5/o_proj", + "deepseek-ai/DeepSeek-R1-0528/q_b_proj" + ] }, { "type": "gemm", "args": { - "m": 4194304, - "n": 7168, - "k": 16384, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 84, + "n": 4096, + "k": 512, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "Qwen/Qwen3.5-397B-A17B-FP8/o_proj", + "Qwen/Qwen3.5-397B-A17B-FP8/shared_expert.down_proj", + "deepseek-ai/DeepSeek-R1-0528/kv_b_proj" + ] }, { "type": "gemm", "args": { - "m": 4194304, - "n": 9216, - "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 84, + "n": 7168, + "k": 2048, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/o_proj", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] }, { "type": "gemm", "args": { - "m": 4194304, + "m": 84, "n": 7168, - "k": 9216, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "k": 2304, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/down_proj" + ] }, { "type": "gemm", "args": { - "m": 4096, - "n": 9216, + "m": 85, + "n": 384, "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [] }, { "type": "gemm", "args": { - "m": 4096, - "n": 7168, - "k": 9216, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 85, + "n": 384, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w3" + ] }, { "type": "gemm", "args": { - "m": 1048576, - "n": 2304, + "m": 85, + "n": 576, "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/Kimi-K2.5-MXFP4/kv_a_proj_with_mqa", + "moonshotai/Kimi-K2.5/kv_a_proj_with_mqa", + "nvidia/DeepSeek-R1-0528-FP4-V2/kv_a_proj_with_mqa", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/kv_a_proj_with_mqa", + "nvidia/Kimi-K2.5-NVFP4/kv_a_proj_with_mqa", + "nvidia/Kimi-K2.6-NVFP4/kv_a_proj_with_mqa" + ] }, { "type": "gemm", "args": { - "m": 1048576, - "n": 7168, - "k": 2304, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 85, + "n": 576, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/kv_a_proj_with_mqa" + ] }, { "type": "gemm", "args": { - "m": 1024, - "n": 2304, + "m": 85, + "n": 768, "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-NVFP4-v2/gate_up_proj" + ] }, { "type": "gemm", "args": { - "m": 1024, - "n": 7168, - "k": 2304, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 85, + "n": 768, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w3" + ] }, { "type": "gemm", "args": { - "m": 2360832, + "m": 85, "n": 1536, "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 2360832, - "n": 24576, - "k": 1536, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/Kimi-K2.5-MXFP4/q_a_proj", + "moonshotai/Kimi-K2.5/q_a_proj", + "nvidia/DeepSeek-R1-0528-FP4-V2/q_a_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/q_a_proj", + "nvidia/Kimi-K2.5-NVFP4/q_a_proj", + "nvidia/Kimi-K2.6-NVFP4/q_a_proj" + ] }, { "type": "gemm", "args": { - "m": 2360832, - "n": 576, + "m": 85, + "n": 1536, "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/gate_up_proj", + "deepseek-ai/DeepSeek-R1-0528/q_a_proj", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w3" + ] }, { "type": "gemm", "args": { - "m": 2360832, - "n": 32768, - "k": 512, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 85, + "n": 7168, + "k": 384, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-NVFP4-v2/down_proj" + ] }, { "type": "gemm", "args": { - "m": 2360832, + "m": 85, "n": 7168, - "k": 16384, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "k": 384, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w2" + ] }, { "type": "gemm", "args": { - "m": 2360832, - "n": 9216, - "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 85, + "n": 7168, + "k": 768, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/down_proj", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w2" + ] }, { "type": "gemm", "args": { - "m": 2360832, + "m": 85, "n": 7168, - "k": 9216, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "k": 16384, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/o_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/o_proj" + ] }, { "type": "gemm", "args": { - "m": 2305, - "n": 1536, - "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 85, + "n": 7168, + "k": 16384, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/o_proj", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] }, { "type": "gemm", "args": { - "m": 2305, + "m": 85, "n": 24576, "k": 1536, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/q_b_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/q_b_proj" + ] }, { "type": "gemm", "args": { - "m": 2305, - "n": 576, - "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 85, + "n": 24576, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/q_b_proj" + ] }, { "type": "gemm", "args": { - "m": 2305, + "m": 85, "n": 32768, "k": 512, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/kv_b_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/kv_b_proj" + ] }, { "type": "gemm", "args": { - "m": 2305, - "n": 7168, - "k": 16384, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 85, + "n": 32768, + "k": 512, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/kv_b_proj" + ] }, { "type": "gemm", "args": { - "m": 2305, - "n": 9216, + "m": 90, + "n": 576, "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/Kimi-K2.5-MXFP4/kv_a_proj_with_mqa", + "moonshotai/Kimi-K2.5/kv_a_proj_with_mqa", + "nvidia/DeepSeek-R1-0528-FP4-V2/kv_a_proj_with_mqa", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/kv_a_proj_with_mqa", + "nvidia/Kimi-K2.5-NVFP4/kv_a_proj_with_mqa", + "nvidia/Kimi-K2.6-NVFP4/kv_a_proj_with_mqa" + ] }, { "type": "gemm", "args": { - "m": 2305, - "n": 7168, - "k": 9216, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 90, + "n": 1536, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/Kimi-K2.5-MXFP4/q_a_proj", + "moonshotai/Kimi-K2.5/q_a_proj", + "nvidia/DeepSeek-R1-0528-FP4-V2/q_a_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/q_a_proj", + "nvidia/Kimi-K2.5-NVFP4/q_a_proj", + "nvidia/Kimi-K2.6-NVFP4/q_a_proj" + ] }, { "type": "gemm", "args": { - "m": 590208, - "n": 1536, + "m": 90, + "n": 2304, "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/gate_up_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/gate_up_proj", + "nvidia/Kimi-K2.5-NVFP4/gate_up_proj" + ] }, { "type": "gemm", "args": { - "m": 590208, - "n": 24576, + "m": 90, + "n": 3072, "k": 1536, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/Kimi-K2.5-MXFP4/q_b_proj", + "amd/MiniMax-M2.5-MXFP4/o_proj", + "moonshotai/Kimi-K2.5/q_b_proj", + "nvidia/DeepSeek-R1-0528-FP4-V2/q_b_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/q_b_proj", + "nvidia/Kimi-K2.5-NVFP4/q_b_proj", + "nvidia/Kimi-K2.6-NVFP4/q_b_proj", + "nvidia/MiniMax-M2.5-NVFP4/o_proj" + ] }, { "type": "gemm", "args": { - "m": 590208, - "n": 576, - "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 90, + "n": 4096, + "k": 512, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "Qwen/Qwen3.5-397B-A17B/o_proj", + "Qwen/Qwen3.5-397B-A17B/shared_expert.down_proj", + "amd/Kimi-K2.5-MXFP4/kv_b_proj", + "amd/Qwen3.5-397B-A17B-MXFP4/o_proj", + "amd/Qwen3.5-397B-A17B-MXFP4/shared_expert.down_proj", + "moonshotai/Kimi-K2.5/kv_b_proj", + "nvidia/DeepSeek-R1-0528-FP4-V2/kv_b_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/kv_b_proj", + "nvidia/Kimi-K2.5-NVFP4/kv_b_proj", + "nvidia/Kimi-K2.6-NVFP4/kv_b_proj", + "nvidia/Qwen3.5-397B-A17B-NVFP4/o_proj", + "nvidia/Qwen3.5-397B-A17B-NVFP4/shared_expert.down_proj" + ] }, { "type": "gemm", "args": { - "m": 590208, - "n": 32768, - "k": 512, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 90, + "n": 4608, + "k": 7168, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/gate_up_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/gate_up_proj", + "nvidia/Kimi-K2.5-NVFP4/gate_up_proj" + ] }, { "type": "gemm", "args": { - "m": 590208, + "m": 90, "n": 7168, - "k": 16384, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "k": 2048, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/o_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/o_proj" + ] }, { "type": "gemm", "args": { - "m": 590208, - "n": 2304, - "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 90, + "n": 7168, + "k": 2304, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/down_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/down_proj", + "nvidia/Kimi-K2.5-NVFP4/down_proj" + ] }, { "type": "gemm", "args": { - "m": 590208, + "m": 90, "n": 7168, - "k": 2304, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "k": 4608, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/down_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/down_proj", + "nvidia/Kimi-K2.5-NVFP4/down_proj" + ] }, { "type": "gemm", "args": { - "m": 576, - "n": 1536, - "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 90, + "n": 7168, + "k": 4608, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 32 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 1, + 32 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "amd/DeepSeek-R1-0528-MXFP4-Preview/down_proj", + "amd/DeepSeek-R1-0528-MXFP4-v2/down_proj", + "amd/DeepSeek-R1-0528-MXFP4/down_proj", + "amd/Kimi-K2.5-MXFP4/down_proj" + ] }, { "type": "gemm", "args": { - "m": 576, - "n": 24576, - "k": 1536, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 90, + "n": 7168, + "k": 4608, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/down_proj" + ] }, { "type": "gemm", "args": { - "m": 576, - "n": 576, - "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 90, + "n": 7168, + "k": 8192, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/Kimi-K2.5-MXFP4/o_proj", + "moonshotai/Kimi-K2.5/o_proj", + "nvidia/Kimi-K2.5-NVFP4/o_proj", + "nvidia/Kimi-K2.6-NVFP4/o_proj" + ] }, { "type": "gemm", "args": { - "m": 576, - "n": 32768, - "k": 512, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 90, + "n": 7168, + "k": 8192, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] }, { "type": "gemm", "args": { - "m": 576, + "m": 90, "n": 7168, "k": 16384, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/o_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/o_proj" + ] }, { "type": "gemm", "args": { - "m": 576, - "n": 2304, - "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 90, + "n": 7168, + "k": 16384, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 32 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 1, + 32 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "amd/DeepSeek-R1-0528-MXFP4-Preview/o_proj", + "amd/DeepSeek-R1-0528-MXFP4-v2/o_proj", + "amd/DeepSeek-R1-0528-MXFP4/o_proj" + ] }, { "type": "gemm", "args": { - "m": 576, + "m": 90, "n": 7168, - "k": 2304, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "k": 16384, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/o_proj", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] }, { "type": "gemm", "args": { - "m": 9003008, - "n": 1536, - "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 90, + "n": 7168, + "k": 16384, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] }, { "type": "gemm", "args": { - "m": 9003008, - "n": 24576, + "m": 90, + "n": 12288, "k": 1536, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/Kimi-K2.5-MXFP4/q_b_proj", + "moonshotai/Kimi-K2.5/q_b_proj", + "nvidia/DeepSeek-R1-0528-FP4-V2/q_b_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/q_b_proj", + "nvidia/Kimi-K2.5-NVFP4/q_b_proj", + "nvidia/Kimi-K2.6-NVFP4/q_b_proj" + ] }, { "type": "gemm", "args": { - "m": 9003008, - "n": 576, - "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 90, + "n": 16384, + "k": 512, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/Kimi-K2.5-MXFP4/kv_b_proj", + "moonshotai/Kimi-K2.5/kv_b_proj", + "nvidia/DeepSeek-R1-0528-FP4-V2/kv_b_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/kv_b_proj", + "nvidia/Kimi-K2.5-NVFP4/kv_b_proj", + "nvidia/Kimi-K2.6-NVFP4/kv_b_proj" + ] }, { "type": "gemm", "args": { - "m": 9003008, - "n": 32768, - "k": 512, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 90, + "n": 24576, + "k": 1536, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/q_b_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/q_b_proj" + ] }, { "type": "gemm", "args": { - "m": 9003008, - "n": 7168, - "k": 16384, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 90, + "n": 24576, + "k": 1536, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 32 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 1, + 32 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "amd/DeepSeek-R1-0528-MXFP4-Preview/q_b_proj", + "amd/DeepSeek-R1-0528-MXFP4-v2/q_b_proj", + "amd/DeepSeek-R1-0528-MXFP4/q_b_proj" + ] }, { "type": "gemm", "args": { - "m": 9003008, - "n": 9216, - "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 90, + "n": 24576, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/q_b_proj" + ] }, { "type": "gemm", "args": { - "m": 9003008, - "n": 7168, - "k": 9216, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 90, + "n": 32768, + "k": 512, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/kv_b_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/kv_b_proj" + ] }, { "type": "gemm", "args": { - "m": 2198, - "n": 1536, - "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 90, + "n": 32768, + "k": 512, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 32 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 1, + 32 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "amd/DeepSeek-R1-0528-MXFP4-Preview/kv_b_proj", + "amd/DeepSeek-R1-0528-MXFP4-v2/kv_b_proj", + "amd/DeepSeek-R1-0528-MXFP4/kv_b_proj" + ] }, { "type": "gemm", "args": { - "m": 2198, - "n": 24576, - "k": 1536, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 90, + "n": 32768, + "k": 512, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/kv_b_proj" + ] }, { "type": "gemm", "args": { - "m": 2198, + "m": 92, "n": 576, "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/Kimi-K2.5-MXFP4/kv_a_proj_with_mqa", + "moonshotai/Kimi-K2.5/kv_a_proj_with_mqa", + "nvidia/DeepSeek-R1-0528-FP4-V2/kv_a_proj_with_mqa", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/kv_a_proj_with_mqa", + "nvidia/Kimi-K2.5-NVFP4/kv_a_proj_with_mqa", + "nvidia/Kimi-K2.6-NVFP4/kv_a_proj_with_mqa" + ] }, { "type": "gemm", "args": { - "m": 2198, - "n": 32768, - "k": 512, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 92, + "n": 576, + "k": 7168, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 32 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 1, + 32 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "amd/DeepSeek-R1-0528-MXFP4-Preview/kv_a_proj_with_mqa", + "amd/DeepSeek-R1-0528-MXFP4-v2/kv_a_proj_with_mqa", + "amd/DeepSeek-R1-0528-MXFP4/kv_a_proj_with_mqa" + ] }, { "type": "gemm", "args": { - "m": 2198, - "n": 7168, - "k": 16384, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 92, + "n": 1536, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/Kimi-K2.5-MXFP4/q_a_proj", + "moonshotai/Kimi-K2.5/q_a_proj", + "nvidia/DeepSeek-R1-0528-FP4-V2/q_a_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/q_a_proj", + "nvidia/Kimi-K2.5-NVFP4/q_a_proj", + "nvidia/Kimi-K2.6-NVFP4/q_a_proj" + ] }, { "type": "gemm", "args": { - "m": 2198, - "n": 9216, + "m": 92, + "n": 1536, "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 32 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 1, + 32 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "amd/DeepSeek-R1-0528-MXFP4-Preview/q_a_proj", + "amd/DeepSeek-R1-0528-MXFP4-v2/q_a_proj", + "amd/DeepSeek-R1-0528-MXFP4/q_a_proj" + ] }, { "type": "gemm", "args": { - "m": 2198, - "n": 7168, - "k": 9216, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 92, + "n": 1536, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w3" + ] }, { "type": "gemm", "args": { - "m": 2250752, - "n": 1536, + "m": 92, + "n": 4608, "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/gate_up_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/gate_up_proj", + "nvidia/Kimi-K2.5-NVFP4/gate_up_proj" + ] }, { "type": "gemm", "args": { - "m": 2250752, - "n": 24576, - "k": 1536, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 92, + "n": 4608, + "k": 7168, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 32 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 1, + 32 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "amd/DeepSeek-R1-0528-MXFP4-Preview/gate_up_proj", + "amd/DeepSeek-R1-0528-MXFP4-v2/gate_up_proj", + "amd/DeepSeek-R1-0528-MXFP4/gate_up_proj", + "amd/Kimi-K2.5-MXFP4/gate_up_proj" + ] }, { "type": "gemm", "args": { - "m": 2250752, - "n": 576, - "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 92, + "n": 7168, + "k": 4608, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/down_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/down_proj", + "nvidia/Kimi-K2.5-NVFP4/down_proj" + ] }, { "type": "gemm", "args": { - "m": 2250752, - "n": 32768, - "k": 512, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 92, + "n": 7168, + "k": 4608, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 32 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 1, + 32 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "amd/DeepSeek-R1-0528-MXFP4-Preview/down_proj", + "amd/DeepSeek-R1-0528-MXFP4-v2/down_proj", + "amd/DeepSeek-R1-0528-MXFP4/down_proj", + "amd/Kimi-K2.5-MXFP4/down_proj" + ] }, { "type": "gemm", "args": { - "m": 2250752, + "m": 92, "n": 7168, "k": 16384, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/o_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/o_proj" + ] }, { "type": "gemm", "args": { - "m": 2250752, - "n": 2304, - "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 92, + "n": 7168, + "k": 16384, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 32 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 1, + 32 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "amd/DeepSeek-R1-0528-MXFP4-Preview/o_proj", + "amd/DeepSeek-R1-0528-MXFP4-v2/o_proj", + "amd/DeepSeek-R1-0528-MXFP4/o_proj" + ] }, { "type": "gemm", "args": { - "m": 2250752, + "m": 92, "n": 7168, - "k": 2304, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "k": 16384, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] }, { "type": "gemm", "args": { - "m": 549, - "n": 1536, - "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 92, + "n": 24576, + "k": 1536, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/q_b_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/q_b_proj" + ] }, { "type": "gemm", "args": { - "m": 549, + "m": 92, "n": 24576, "k": 1536, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 32 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 1, + 32 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "amd/DeepSeek-R1-0528-MXFP4-Preview/q_b_proj", + "amd/DeepSeek-R1-0528-MXFP4-v2/q_b_proj", + "amd/DeepSeek-R1-0528-MXFP4/q_b_proj" + ] }, { "type": "gemm", "args": { - "m": 549, - "n": 576, - "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 92, + "n": 32768, + "k": 512, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/kv_b_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/kv_b_proj" + ] }, { "type": "gemm", "args": { - "m": 549, + "m": 92, "n": 32768, "k": 512, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 32 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 1, + 32 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "amd/DeepSeek-R1-0528-MXFP4-Preview/kv_b_proj", + "amd/DeepSeek-R1-0528-MXFP4-v2/kv_b_proj", + "amd/DeepSeek-R1-0528-MXFP4/kv_b_proj" + ] }, { "type": "gemm", "args": { - "m": 549, - "n": 7168, - "k": 16384, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 96, + "n": 576, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/kv_a_proj_with_mqa" + ] }, { "type": "gemm", "args": { - "m": 549, - "n": 2304, + "m": 96, + "n": 1536, "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/gate_up_proj", + "deepseek-ai/DeepSeek-R1-0528/q_a_proj", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w3" + ] }, { "type": "gemm", "args": { - "m": 549, - "n": 7168, - "k": 2304, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 96, + "n": 1536, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w3" + ] }, { "type": "gemm", "args": { - "m": 212992, - "n": 1536, + "m": 96, + "n": 2304, "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/gate_up_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/gate_up_proj", + "nvidia/Kimi-K2.5-NVFP4/gate_up_proj" + ] }, { "type": "gemm", "args": { - "m": 212992, - "n": 24576, + "m": 96, + "n": 2304, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/gate_up_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 96, + "n": 3072, "k": 1536, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/Kimi-K2.5-MXFP4/q_b_proj", + "amd/MiniMax-M2.5-MXFP4/o_proj", + "moonshotai/Kimi-K2.5/q_b_proj", + "nvidia/DeepSeek-R1-0528-FP4-V2/q_b_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/q_b_proj", + "nvidia/Kimi-K2.5-NVFP4/q_b_proj", + "nvidia/Kimi-K2.6-NVFP4/q_b_proj", + "nvidia/MiniMax-M2.5-NVFP4/o_proj" + ] }, { "type": "gemm", "args": { - "m": 212992, - "n": 576, - "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 96, + "n": 3072, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "MiniMaxAI/MiniMax-M2.5/experts.down_proj", + "MiniMaxAI/MiniMax-M2.5/o_proj", + "deepseek-ai/DeepSeek-R1-0528/q_b_proj" + ] }, { "type": "gemm", "args": { - "m": 212992, - "n": 32768, + "m": 96, + "n": 4096, "k": 512, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "Qwen/Qwen3.5-397B-A17B/o_proj", + "Qwen/Qwen3.5-397B-A17B/shared_expert.down_proj", + "amd/Kimi-K2.5-MXFP4/kv_b_proj", + "amd/Qwen3.5-397B-A17B-MXFP4/o_proj", + "amd/Qwen3.5-397B-A17B-MXFP4/shared_expert.down_proj", + "moonshotai/Kimi-K2.5/kv_b_proj", + "nvidia/DeepSeek-R1-0528-FP4-V2/kv_b_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/kv_b_proj", + "nvidia/Kimi-K2.5-NVFP4/kv_b_proj", + "nvidia/Kimi-K2.6-NVFP4/kv_b_proj", + "nvidia/Qwen3.5-397B-A17B-NVFP4/o_proj", + "nvidia/Qwen3.5-397B-A17B-NVFP4/shared_expert.down_proj" + ] }, { "type": "gemm", "args": { - "m": 212992, - "n": 7168, - "k": 16384, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 96, + "n": 4096, + "k": 512, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "Qwen/Qwen3.5-397B-A17B-FP8/o_proj", + "Qwen/Qwen3.5-397B-A17B-FP8/shared_expert.down_proj", + "deepseek-ai/DeepSeek-R1-0528/kv_b_proj" + ] }, { "type": "gemm", "args": { - "m": 212992, - "n": 9216, + "m": 96, + "n": 4608, "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/gate_up_proj" + ] }, { "type": "gemm", "args": { - "m": 212992, + "m": 96, "n": 7168, - "k": 9216, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "k": 2048, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/o_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/o_proj" + ] }, { "type": "gemm", "args": { - "m": 52, - "n": 1536, - "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 96, + "n": 7168, + "k": 2048, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/o_proj", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] }, { "type": "gemm", "args": { - "m": 52, - "n": 24576, - "k": 1536, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 96, + "n": 7168, + "k": 2304, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/down_proj" + ] }, { "type": "gemm", "args": { - "m": 52, - "n": 576, - "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 96, + "n": 7168, + "k": 4608, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/down_proj" + ] }, { "type": "gemm", "args": { - "m": 52, - "n": 32768, - "k": 512, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 96, + "n": 7168, + "k": 9216, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/down_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/down_proj", + "nvidia/Kimi-K2.5-NVFP4/down_proj" + ] }, { "type": "gemm", "args": { - "m": 52, + "m": 96, "n": 7168, "k": 16384, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/o_proj", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] }, { "type": "gemm", "args": { - "m": 52, + "m": 96, "n": 9216, "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/gate_up_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/gate_up_proj", + "nvidia/Kimi-K2.5-NVFP4/gate_up_proj" + ] }, { "type": "gemm", "args": { - "m": 52, - "n": 7168, - "k": 9216, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 96, + "n": 9216, + "k": 7168, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 32 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 1, + 32 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "amd/DeepSeek-R1-0528-MXFP4-Preview/gate_up_proj", + "amd/DeepSeek-R1-0528-MXFP4-v2/gate_up_proj", + "amd/DeepSeek-R1-0528-MXFP4/gate_up_proj", + "amd/Kimi-K2.5-MXFP4/gate_up_proj" + ] }, { "type": "gemm", "args": { - "m": 425984, - "n": 1536, - "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 96, + "n": 24576, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/q_b_proj" + ] }, { "type": "gemm", "args": { - "m": 425984, - "n": 6144, - "k": 1536, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 96, + "n": 32768, + "k": 512, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/kv_b_proj" + ] }, { "type": "gemm", "args": { - "m": 425984, + "m": 97, "n": 576, "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/Kimi-K2.5-MXFP4/kv_a_proj_with_mqa", + "moonshotai/Kimi-K2.5/kv_a_proj_with_mqa", + "nvidia/DeepSeek-R1-0528-FP4-V2/kv_a_proj_with_mqa", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/kv_a_proj_with_mqa", + "nvidia/Kimi-K2.5-NVFP4/kv_a_proj_with_mqa", + "nvidia/Kimi-K2.6-NVFP4/kv_a_proj_with_mqa" + ] }, { "type": "gemm", "args": { - "m": 425984, - "n": 8192, - "k": 512, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 97, + "n": 576, + "k": 7168, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 32 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 1, + 32 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "amd/DeepSeek-R1-0528-MXFP4-Preview/kv_a_proj_with_mqa", + "amd/DeepSeek-R1-0528-MXFP4-v2/kv_a_proj_with_mqa", + "amd/DeepSeek-R1-0528-MXFP4/kv_a_proj_with_mqa" + ] }, { "type": "gemm", "args": { - "m": 425984, - "n": 7168, - "k": 4096, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 97, + "n": 1536, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/Kimi-K2.5-MXFP4/q_a_proj", + "moonshotai/Kimi-K2.5/q_a_proj", + "nvidia/DeepSeek-R1-0528-FP4-V2/q_a_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/q_a_proj", + "nvidia/Kimi-K2.5-NVFP4/q_a_proj", + "nvidia/Kimi-K2.6-NVFP4/q_a_proj" + ] }, { "type": "gemm", "args": { - "m": 425984, - "n": 4608, + "m": 97, + "n": 1536, "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 32 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 1, + 32 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "amd/DeepSeek-R1-0528-MXFP4-Preview/q_a_proj", + "amd/DeepSeek-R1-0528-MXFP4-v2/q_a_proj", + "amd/DeepSeek-R1-0528-MXFP4/q_a_proj" + ] }, { "type": "gemm", "args": { - "m": 425984, + "m": 97, + "n": 1536, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w3" + ] + }, + { + "type": "gemm", + "args": { + "m": 97, "n": 7168, - "k": 4608, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "k": 9216, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/down_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/down_proj", + "nvidia/Kimi-K2.5-NVFP4/down_proj" + ] }, { "type": "gemm", "args": { - "m": 104, - "n": 6144, - "k": 1536, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 97, + "n": 7168, + "k": 16384, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/o_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/o_proj" + ] }, { "type": "gemm", "args": { - "m": 104, - "n": 8192, - "k": 512, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 104, + "m": 97, "n": 7168, - "k": 4096, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 104, - "n": 4608, - "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "k": 16384, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 32 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 1, + 32 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "amd/DeepSeek-R1-0528-MXFP4-Preview/o_proj", + "amd/DeepSeek-R1-0528-MXFP4-v2/o_proj", + "amd/DeepSeek-R1-0528-MXFP4/o_proj" + ] }, { "type": "gemm", "args": { - "m": 104, + "m": 97, "n": 7168, - "k": 4608, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "k": 16384, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] }, { "type": "gemm", "args": { - "m": 32768, + "m": 97, "n": 9216, "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/gate_up_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/gate_up_proj", + "nvidia/Kimi-K2.5-NVFP4/gate_up_proj" + ] }, { "type": "gemm", "args": { - "m": 32768, - "n": 7168, - "k": 9216, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 8, + "m": 97, "n": 9216, "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 32 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 1, + 32 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "amd/DeepSeek-R1-0528-MXFP4-Preview/gate_up_proj", + "amd/DeepSeek-R1-0528-MXFP4-v2/gate_up_proj", + "amd/DeepSeek-R1-0528-MXFP4/gate_up_proj", + "amd/Kimi-K2.5-MXFP4/gate_up_proj" + ] }, { "type": "gemm", "args": { - "m": 8, - "n": 7168, - "k": 9216, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 97, + "n": 24576, + "k": 1536, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/q_b_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/q_b_proj" + ] }, { "type": "gemm", "args": { - "m": 131072, + "m": 97, "n": 24576, "k": 1536, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 32 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 1, + 32 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "amd/DeepSeek-R1-0528-MXFP4-Preview/q_b_proj", + "amd/DeepSeek-R1-0528-MXFP4-v2/q_b_proj", + "amd/DeepSeek-R1-0528-MXFP4/q_b_proj" + ] }, { "type": "gemm", "args": { - "m": 131072, + "m": 97, "n": 32768, "k": 512, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/kv_b_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/kv_b_proj" + ] }, { "type": "gemm", "args": { - "m": 131072, - "n": 7168, - "k": 16384, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 97, + "n": 32768, + "k": 512, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 32 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 1, + 32 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "amd/DeepSeek-R1-0528-MXFP4-Preview/kv_b_proj", + "amd/DeepSeek-R1-0528-MXFP4-v2/kv_b_proj", + "amd/DeepSeek-R1-0528-MXFP4/kv_b_proj" + ] }, { "type": "gemm", "args": { - "m": 131072, - "n": 9216, + "m": 100, + "n": 576, "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/Kimi-K2.5-MXFP4/kv_a_proj_with_mqa", + "moonshotai/Kimi-K2.5/kv_a_proj_with_mqa", + "nvidia/DeepSeek-R1-0528-FP4-V2/kv_a_proj_with_mqa", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/kv_a_proj_with_mqa", + "nvidia/Kimi-K2.5-NVFP4/kv_a_proj_with_mqa", + "nvidia/Kimi-K2.6-NVFP4/kv_a_proj_with_mqa" + ] }, { "type": "gemm", "args": { - "m": 131072, - "n": 7168, - "k": 9216, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 100, + "n": 1536, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/Kimi-K2.5-MXFP4/q_a_proj", + "moonshotai/Kimi-K2.5/q_a_proj", + "nvidia/DeepSeek-R1-0528-FP4-V2/q_a_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/q_a_proj", + "nvidia/Kimi-K2.5-NVFP4/q_a_proj", + "nvidia/Kimi-K2.6-NVFP4/q_a_proj" + ] }, { "type": "gemm", "args": { - "m": 32, - "n": 24576, + "m": 100, + "n": 2304, + "k": 7168, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/gate_up_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/gate_up_proj", + "nvidia/Kimi-K2.5-NVFP4/gate_up_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 100, + "n": 3072, "k": 1536, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/Kimi-K2.5-MXFP4/q_b_proj", + "amd/MiniMax-M2.5-MXFP4/o_proj", + "moonshotai/Kimi-K2.5/q_b_proj", + "nvidia/DeepSeek-R1-0528-FP4-V2/q_b_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/q_b_proj", + "nvidia/Kimi-K2.5-NVFP4/q_b_proj", + "nvidia/Kimi-K2.6-NVFP4/q_b_proj", + "nvidia/MiniMax-M2.5-NVFP4/o_proj" + ] }, { "type": "gemm", "args": { - "m": 32, - "n": 32768, + "m": 100, + "n": 4096, "k": 512, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "Qwen/Qwen3.5-397B-A17B/o_proj", + "Qwen/Qwen3.5-397B-A17B/shared_expert.down_proj", + "amd/Kimi-K2.5-MXFP4/kv_b_proj", + "amd/Qwen3.5-397B-A17B-MXFP4/o_proj", + "amd/Qwen3.5-397B-A17B-MXFP4/shared_expert.down_proj", + "moonshotai/Kimi-K2.5/kv_b_proj", + "nvidia/DeepSeek-R1-0528-FP4-V2/kv_b_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/kv_b_proj", + "nvidia/Kimi-K2.5-NVFP4/kv_b_proj", + "nvidia/Kimi-K2.6-NVFP4/kv_b_proj", + "nvidia/Qwen3.5-397B-A17B-NVFP4/o_proj", + "nvidia/Qwen3.5-397B-A17B-NVFP4/shared_expert.down_proj" + ] }, { "type": "gemm", "args": { - "m": 32, + "m": 100, "n": 7168, - "k": 16384, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "k": 2048, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/o_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/o_proj" + ] }, { "type": "gemm", "args": { - "m": 32, - "n": 9216, - "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 100, + "n": 7168, + "k": 2304, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/down_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/down_proj", + "nvidia/Kimi-K2.5-NVFP4/down_proj" + ] }, { "type": "gemm", "args": { - "m": 32, - "n": 7168, - "k": 9216, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 104, + "n": 576, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/Kimi-K2.5-MXFP4/kv_a_proj_with_mqa", + "moonshotai/Kimi-K2.5/kv_a_proj_with_mqa", + "nvidia/DeepSeek-R1-0528-FP4-V2/kv_a_proj_with_mqa", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/kv_a_proj_with_mqa", + "nvidia/Kimi-K2.5-NVFP4/kv_a_proj_with_mqa", + "nvidia/Kimi-K2.6-NVFP4/kv_a_proj_with_mqa" + ] }, { "type": "gemm", "args": { - "m": 64, + "m": 104, "n": 1536, "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/Kimi-K2.5-MXFP4/q_a_proj", + "moonshotai/Kimi-K2.5/q_a_proj", + "nvidia/DeepSeek-R1-0528-FP4-V2/q_a_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/q_a_proj", + "nvidia/Kimi-K2.5-NVFP4/q_a_proj", + "nvidia/Kimi-K2.6-NVFP4/q_a_proj" + ] }, { "type": "gemm", "args": { - "m": 64, - "n": 3072, - "k": 1536, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 104, + "n": 2304, + "k": 7168, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/gate_up_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/gate_up_proj", + "nvidia/Kimi-K2.5-NVFP4/gate_up_proj" + ] }, { "type": "gemm", "args": { - "m": 64, - "n": 576, + "m": 104, + "n": 4608, "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/gate_up_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/gate_up_proj", + "nvidia/Kimi-K2.5-NVFP4/gate_up_proj" + ] }, { "type": "gemm", "args": { - "m": 64, - "n": 4096, - "k": 512, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 104, + "n": 6144, + "k": 1536, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/GLM-5.1-MXFP4/down_proj", + "amd/GLM-5.2-MXFP4/down_proj", + "amd/Kimi-K2.5-MXFP4/q_b_proj", + "amd/MiniMax-M3-MXFP4/down_proj", + "moonshotai/Kimi-K2.5/q_b_proj", + "nvidia/DeepSeek-R1-0528-FP4-V2/q_b_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/q_b_proj", + "nvidia/GLM-5.1-NVFP4/down_proj", + "nvidia/GLM-5.2-NVFP4/down_proj", + "nvidia/Kimi-K2.5-NVFP4/q_b_proj", + "nvidia/Kimi-K2.6-NVFP4/q_b_proj" + ] }, { "type": "gemm", "args": { - "m": 64, + "m": 104, "n": 7168, - "k": 2048, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "k": 2304, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/down_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/down_proj", + "nvidia/Kimi-K2.5-NVFP4/down_proj" + ] }, { "type": "gemm", "args": { - "m": 64, - "n": 2304, - "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 104, + "n": 7168, + "k": 4096, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/o_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/o_proj" + ] }, { "type": "gemm", "args": { - "m": 64, + "m": 104, "n": 7168, - "k": 2304, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "k": 4608, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/down_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/down_proj", + "nvidia/Kimi-K2.5-NVFP4/down_proj" + ] }, { "type": "gemm", "args": { - "m": 741376, - "n": 1536, - "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 104, + "n": 7168, + "k": 16384, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/o_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/o_proj" + ] }, { "type": "gemm", "args": { - "m": 741376, - "n": 24576, - "k": 1536, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 104, + "n": 8192, + "k": 512, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/Kimi-K2.5-MXFP4/kv_b_proj", + "moonshotai/Kimi-K2.5/kv_b_proj", + "nvidia/DeepSeek-R1-0528-FP4-V2/kv_b_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/kv_b_proj", + "nvidia/Kimi-K2.5-NVFP4/kv_b_proj", + "nvidia/Kimi-K2.6-NVFP4/kv_b_proj" + ] }, { "type": "gemm", "args": { - "m": 741376, - "n": 576, - "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 104, + "n": 24576, + "k": 1536, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/q_b_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/q_b_proj" + ] }, { "type": "gemm", "args": { - "m": 741376, + "m": 104, "n": 32768, "k": 512, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/kv_b_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/kv_b_proj" + ] }, { "type": "gemm", "args": { - "m": 741376, - "n": 7168, - "k": 16384, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 105, + "n": 576, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/Kimi-K2.5-MXFP4/kv_a_proj_with_mqa", + "moonshotai/Kimi-K2.5/kv_a_proj_with_mqa", + "nvidia/DeepSeek-R1-0528-FP4-V2/kv_a_proj_with_mqa", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/kv_a_proj_with_mqa", + "nvidia/Kimi-K2.5-NVFP4/kv_a_proj_with_mqa", + "nvidia/Kimi-K2.6-NVFP4/kv_a_proj_with_mqa" + ] }, { "type": "gemm", "args": { - "m": 741376, - "n": 9216, + "m": 105, + "n": 1536, "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/Kimi-K2.5-MXFP4/q_a_proj", + "moonshotai/Kimi-K2.5/q_a_proj", + "nvidia/DeepSeek-R1-0528-FP4-V2/q_a_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/q_a_proj", + "nvidia/Kimi-K2.5-NVFP4/q_a_proj", + "nvidia/Kimi-K2.6-NVFP4/q_a_proj" + ] }, { "type": "gemm", "args": { - "m": 741376, - "n": 7168, - "k": 9216, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 105, + "n": 3072, + "k": 1536, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/Kimi-K2.5-MXFP4/q_b_proj", + "amd/MiniMax-M2.5-MXFP4/o_proj", + "moonshotai/Kimi-K2.5/q_b_proj", + "nvidia/DeepSeek-R1-0528-FP4-V2/q_b_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/q_b_proj", + "nvidia/Kimi-K2.5-NVFP4/q_b_proj", + "nvidia/Kimi-K2.6-NVFP4/q_b_proj", + "nvidia/MiniMax-M2.5-NVFP4/o_proj" + ] }, { "type": "gemm", "args": { - "m": 181, - "n": 1536, - "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 105, + "n": 4096, + "k": 512, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "Qwen/Qwen3.5-397B-A17B/o_proj", + "Qwen/Qwen3.5-397B-A17B/shared_expert.down_proj", + "amd/Kimi-K2.5-MXFP4/kv_b_proj", + "amd/Qwen3.5-397B-A17B-MXFP4/o_proj", + "amd/Qwen3.5-397B-A17B-MXFP4/shared_expert.down_proj", + "moonshotai/Kimi-K2.5/kv_b_proj", + "nvidia/DeepSeek-R1-0528-FP4-V2/kv_b_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/kv_b_proj", + "nvidia/Kimi-K2.5-NVFP4/kv_b_proj", + "nvidia/Kimi-K2.6-NVFP4/kv_b_proj", + "nvidia/Qwen3.5-397B-A17B-NVFP4/o_proj", + "nvidia/Qwen3.5-397B-A17B-NVFP4/shared_expert.down_proj" + ] }, { "type": "gemm", "args": { - "m": 181, - "n": 24576, - "k": 1536, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 105, + "n": 4608, + "k": 7168, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/gate_up_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/gate_up_proj", + "nvidia/Kimi-K2.5-NVFP4/gate_up_proj" + ] }, { "type": "gemm", "args": { - "m": 181, - "n": 576, + "m": 105, + "n": 4608, "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 32 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 1, + 32 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "amd/DeepSeek-R1-0528-MXFP4-Preview/gate_up_proj", + "amd/DeepSeek-R1-0528-MXFP4-v2/gate_up_proj", + "amd/DeepSeek-R1-0528-MXFP4/gate_up_proj", + "amd/Kimi-K2.5-MXFP4/gate_up_proj" + ] }, { "type": "gemm", "args": { - "m": 181, - "n": 32768, - "k": 512, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 105, + "n": 6144, + "k": 1536, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/GLM-5.1-MXFP4/down_proj", + "amd/GLM-5.2-MXFP4/down_proj", + "amd/Kimi-K2.5-MXFP4/q_b_proj", + "amd/MiniMax-M3-MXFP4/down_proj", + "moonshotai/Kimi-K2.5/q_b_proj", + "nvidia/DeepSeek-R1-0528-FP4-V2/q_b_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/q_b_proj", + "nvidia/GLM-5.1-NVFP4/down_proj", + "nvidia/GLM-5.2-NVFP4/down_proj", + "nvidia/Kimi-K2.5-NVFP4/q_b_proj", + "nvidia/Kimi-K2.6-NVFP4/q_b_proj" + ] }, { "type": "gemm", "args": { - "m": 181, + "m": 105, "n": 7168, - "k": 16384, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 181, - "n": 9216, - "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "k": 2048, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/Kimi-K2.5-MXFP4/o_proj", + "moonshotai/Kimi-K2.5/o_proj", + "nvidia/Kimi-K2.5-NVFP4/o_proj", + "nvidia/Kimi-K2.6-NVFP4/o_proj" + ] }, { "type": "gemm", "args": { - "m": 181, + "m": 105, "n": 7168, - "k": 9216, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "k": 4096, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/o_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/o_proj" + ] }, { "type": "gemm", "args": { - "m": 185344, - "n": 1536, - "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 105, + "n": 7168, + "k": 4608, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/down_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/down_proj", + "nvidia/Kimi-K2.5-NVFP4/down_proj" + ] }, { "type": "gemm", "args": { - "m": 185344, - "n": 24576, - "k": 1536, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 105, + "n": 8192, + "k": 512, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/Kimi-K2.5-MXFP4/kv_b_proj", + "moonshotai/Kimi-K2.5/kv_b_proj", + "nvidia/DeepSeek-R1-0528-FP4-V2/kv_b_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/kv_b_proj", + "nvidia/Kimi-K2.5-NVFP4/kv_b_proj", + "nvidia/Kimi-K2.6-NVFP4/kv_b_proj" + ] }, { "type": "gemm", "args": { - "m": 185344, + "m": 110, "n": 576, "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 185344, - "n": 32768, - "k": 512, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/Kimi-K2.5-MXFP4/kv_a_proj_with_mqa", + "moonshotai/Kimi-K2.5/kv_a_proj_with_mqa", + "nvidia/DeepSeek-R1-0528-FP4-V2/kv_a_proj_with_mqa", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/kv_a_proj_with_mqa", + "nvidia/Kimi-K2.5-NVFP4/kv_a_proj_with_mqa", + "nvidia/Kimi-K2.6-NVFP4/kv_a_proj_with_mqa" + ] }, { "type": "gemm", "args": { - "m": 185344, - "n": 7168, - "k": 16384, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 110, + "n": 1536, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/Kimi-K2.5-MXFP4/q_a_proj", + "moonshotai/Kimi-K2.5/q_a_proj", + "nvidia/DeepSeek-R1-0528-FP4-V2/q_a_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/q_a_proj", + "nvidia/Kimi-K2.5-NVFP4/q_a_proj", + "nvidia/Kimi-K2.6-NVFP4/q_a_proj" + ] }, { "type": "gemm", "args": { - "m": 185344, + "m": 110, "n": 2304, "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/gate_up_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/gate_up_proj", + "nvidia/Kimi-K2.5-NVFP4/gate_up_proj" + ] }, { "type": "gemm", "args": { - "m": 185344, - "n": 7168, - "k": 2304, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 110, + "n": 3072, + "k": 1536, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/Kimi-K2.5-MXFP4/q_b_proj", + "amd/MiniMax-M2.5-MXFP4/o_proj", + "moonshotai/Kimi-K2.5/q_b_proj", + "nvidia/DeepSeek-R1-0528-FP4-V2/q_b_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/q_b_proj", + "nvidia/Kimi-K2.5-NVFP4/q_b_proj", + "nvidia/Kimi-K2.6-NVFP4/q_b_proj", + "nvidia/MiniMax-M2.5-NVFP4/o_proj" + ] }, { "type": "gemm", "args": { - "m": 4902912, - "n": 1536, - "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 110, + "n": 4096, + "k": 512, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "Qwen/Qwen3.5-397B-A17B/o_proj", + "Qwen/Qwen3.5-397B-A17B/shared_expert.down_proj", + "amd/Kimi-K2.5-MXFP4/kv_b_proj", + "amd/Qwen3.5-397B-A17B-MXFP4/o_proj", + "amd/Qwen3.5-397B-A17B-MXFP4/shared_expert.down_proj", + "moonshotai/Kimi-K2.5/kv_b_proj", + "nvidia/DeepSeek-R1-0528-FP4-V2/kv_b_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/kv_b_proj", + "nvidia/Kimi-K2.5-NVFP4/kv_b_proj", + "nvidia/Kimi-K2.6-NVFP4/kv_b_proj", + "nvidia/Qwen3.5-397B-A17B-NVFP4/o_proj", + "nvidia/Qwen3.5-397B-A17B-NVFP4/shared_expert.down_proj" + ] }, { "type": "gemm", "args": { - "m": 4902912, - "n": 24576, - "k": 1536, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 110, + "n": 7168, + "k": 2048, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/o_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/o_proj" + ] }, { "type": "gemm", "args": { - "m": 4902912, - "n": 576, - "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 110, + "n": 7168, + "k": 2304, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/down_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/down_proj", + "nvidia/Kimi-K2.5-NVFP4/down_proj" + ] }, { "type": "gemm", "args": { - "m": 4902912, - "n": 32768, - "k": 512, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 112, + "n": 576, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/Kimi-K2.5-MXFP4/kv_a_proj_with_mqa", + "moonshotai/Kimi-K2.5/kv_a_proj_with_mqa", + "nvidia/DeepSeek-R1-0528-FP4-V2/kv_a_proj_with_mqa", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/kv_a_proj_with_mqa", + "nvidia/Kimi-K2.5-NVFP4/kv_a_proj_with_mqa", + "nvidia/Kimi-K2.6-NVFP4/kv_a_proj_with_mqa" + ] }, { "type": "gemm", "args": { - "m": 4902912, - "n": 7168, - "k": 16384, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 112, + "n": 576, + "k": 7168, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 32 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 1, + 32 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "amd/DeepSeek-R1-0528-MXFP4-Preview/kv_a_proj_with_mqa", + "amd/DeepSeek-R1-0528-MXFP4-v2/kv_a_proj_with_mqa", + "amd/DeepSeek-R1-0528-MXFP4/kv_a_proj_with_mqa" + ] }, { "type": "gemm", "args": { - "m": 4902912, - "n": 9216, + "m": 112, + "n": 1536, "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/Kimi-K2.5-MXFP4/q_a_proj", + "moonshotai/Kimi-K2.5/q_a_proj", + "nvidia/DeepSeek-R1-0528-FP4-V2/q_a_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/q_a_proj", + "nvidia/Kimi-K2.5-NVFP4/q_a_proj", + "nvidia/Kimi-K2.6-NVFP4/q_a_proj" + ] }, { "type": "gemm", "args": { - "m": 4902912, - "n": 7168, - "k": 9216, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 112, + "n": 1536, + "k": 7168, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 32 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 1, + 32 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "amd/DeepSeek-R1-0528-MXFP4-Preview/q_a_proj", + "amd/DeepSeek-R1-0528-MXFP4-v2/q_a_proj", + "amd/DeepSeek-R1-0528-MXFP4/q_a_proj" + ] }, { "type": "gemm", "args": { - "m": 1197, + "m": 112, "n": 1536, "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w3" + ] }, { "type": "gemm", "args": { - "m": 1197, - "n": 24576, - "k": 1536, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 112, + "n": 4608, + "k": 7168, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/gate_up_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/gate_up_proj", + "nvidia/Kimi-K2.5-NVFP4/gate_up_proj" + ] }, { "type": "gemm", "args": { - "m": 1197, - "n": 576, + "m": 112, + "n": 4608, "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 32 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 1, + 32 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "amd/DeepSeek-R1-0528-MXFP4-Preview/gate_up_proj", + "amd/DeepSeek-R1-0528-MXFP4-v2/gate_up_proj", + "amd/DeepSeek-R1-0528-MXFP4/gate_up_proj", + "amd/Kimi-K2.5-MXFP4/gate_up_proj" + ] }, { "type": "gemm", "args": { - "m": 1197, - "n": 32768, - "k": 512, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 112, + "n": 7168, + "k": 4608, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/down_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/down_proj", + "nvidia/Kimi-K2.5-NVFP4/down_proj" + ] }, { "type": "gemm", "args": { - "m": 1197, + "m": 112, "n": 7168, - "k": 16384, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "k": 4608, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 32 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 1, + 32 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "amd/DeepSeek-R1-0528-MXFP4-Preview/down_proj", + "amd/DeepSeek-R1-0528-MXFP4-v2/down_proj", + "amd/DeepSeek-R1-0528-MXFP4/down_proj", + "amd/Kimi-K2.5-MXFP4/down_proj" + ] }, { "type": "gemm", "args": { - "m": 1197, - "n": 9216, - "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 112, + "n": 7168, + "k": 16384, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/o_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/o_proj" + ] }, { "type": "gemm", "args": { - "m": 1197, + "m": 112, "n": 7168, - "k": 9216, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "k": 16384, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 32 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 1, + 32 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "amd/DeepSeek-R1-0528-MXFP4-Preview/o_proj", + "amd/DeepSeek-R1-0528-MXFP4-v2/o_proj", + "amd/DeepSeek-R1-0528-MXFP4/o_proj" + ] }, { "type": "gemm", "args": { - "m": 1225728, - "n": 1536, - "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 112, + "n": 7168, + "k": 16384, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] }, { "type": "gemm", "args": { - "m": 1225728, + "m": 112, "n": 24576, "k": 1536, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/q_b_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/q_b_proj" + ] }, { "type": "gemm", "args": { - "m": 1225728, - "n": 576, - "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 112, + "n": 24576, + "k": 1536, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 32 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 1, + 32 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "amd/DeepSeek-R1-0528-MXFP4-Preview/q_b_proj", + "amd/DeepSeek-R1-0528-MXFP4-v2/q_b_proj", + "amd/DeepSeek-R1-0528-MXFP4/q_b_proj" + ] }, { "type": "gemm", "args": { - "m": 1225728, + "m": 112, "n": 32768, "k": 512, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/kv_b_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/kv_b_proj" + ] }, { "type": "gemm", "args": { - "m": 1225728, - "n": 7168, - "k": 16384, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 112, + "n": 32768, + "k": 512, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 32 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 1, + 32 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "amd/DeepSeek-R1-0528-MXFP4-Preview/kv_b_proj", + "amd/DeepSeek-R1-0528-MXFP4-v2/kv_b_proj", + "amd/DeepSeek-R1-0528-MXFP4/kv_b_proj" + ] }, { "type": "gemm", "args": { - "m": 1225728, - "n": 2304, + "m": 115, + "n": 576, "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/kv_a_proj_with_mqa" + ] }, { "type": "gemm", "args": { - "m": 1225728, - "n": 7168, - "k": 2304, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 115, + "n": 1152, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/gate_up_proj" + ] }, { "type": "gemm", "args": { - "m": 299, + "m": 115, "n": 1536, "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 299, - "n": 24576, - "k": 1536, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/gate_up_proj", + "deepseek-ai/DeepSeek-R1-0528/q_a_proj", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w3" + ] }, { "type": "gemm", "args": { - "m": 299, - "n": 576, + "m": 115, + "n": 1536, "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w3" + ] }, { "type": "gemm", "args": { - "m": 299, - "n": 32768, - "k": 512, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 115, + "n": 7168, + "k": 1152, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/down_proj" + ] }, { "type": "gemm", "args": { - "m": 299, + "m": 115, "n": 7168, "k": 16384, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/o_proj", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] }, { "type": "gemm", "args": { - "m": 299, - "n": 2304, - "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 115, + "n": 24576, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/q_b_proj" + ] }, { "type": "gemm", "args": { - "m": 299, - "n": 7168, - "k": 2304, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 115, + "n": 32768, + "k": 512, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/kv_b_proj" + ] }, { "type": "gemm", "args": { - "m": 430080, - "n": 1536, + "m": 120, + "n": 576, "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/Kimi-K2.5-MXFP4/kv_a_proj_with_mqa", + "moonshotai/Kimi-K2.5/kv_a_proj_with_mqa", + "nvidia/DeepSeek-R1-0528-FP4-V2/kv_a_proj_with_mqa", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/kv_a_proj_with_mqa", + "nvidia/Kimi-K2.5-NVFP4/kv_a_proj_with_mqa", + "nvidia/Kimi-K2.6-NVFP4/kv_a_proj_with_mqa" + ] }, { "type": "gemm", "args": { - "m": 430080, - "n": 24576, - "k": 1536, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 120, + "n": 576, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/kv_a_proj_with_mqa" + ] }, { "type": "gemm", "args": { - "m": 430080, - "n": 576, + "m": 120, + "n": 1152, "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/gate_up_proj" + ] }, { "type": "gemm", "args": { - "m": 430080, - "n": 32768, - "k": 512, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 120, + "n": 1536, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/q_b_proj" + ] }, { "type": "gemm", "args": { - "m": 430080, - "n": 7168, - "k": 16384, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 120, + "n": 1536, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/Kimi-K2.5-MXFP4/q_a_proj", + "moonshotai/Kimi-K2.5/q_a_proj", + "nvidia/DeepSeek-R1-0528-FP4-V2/q_a_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/q_a_proj", + "nvidia/Kimi-K2.5-NVFP4/q_a_proj", + "nvidia/Kimi-K2.6-NVFP4/q_a_proj" + ] }, { "type": "gemm", "args": { - "m": 430080, - "n": 9216, + "m": 120, + "n": 1536, "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/gate_up_proj", + "deepseek-ai/DeepSeek-R1-0528/q_a_proj", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w3" + ] }, { "type": "gemm", "args": { - "m": 430080, - "n": 7168, - "k": 9216, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 120, + "n": 2048, + "k": 512, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/kv_b_proj" + ] }, { "type": "gemm", "args": { - "m": 860160, - "n": 1536, + "m": 120, + "n": 2304, "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/gate_up_proj" + ] }, { "type": "gemm", "args": { - "m": 860160, - "n": 6144, + "m": 120, + "n": 3072, "k": 1536, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 860160, - "n": 576, - "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/Kimi-K2.5-MXFP4/q_b_proj", + "amd/MiniMax-M2.5-MXFP4/o_proj", + "moonshotai/Kimi-K2.5/q_b_proj", + "nvidia/DeepSeek-R1-0528-FP4-V2/q_b_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/q_b_proj", + "nvidia/Kimi-K2.5-NVFP4/q_b_proj", + "nvidia/Kimi-K2.6-NVFP4/q_b_proj", + "nvidia/MiniMax-M2.5-NVFP4/o_proj" + ] }, { "type": "gemm", "args": { - "m": 860160, - "n": 8192, + "m": 120, + "n": 4096, "k": 512, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "Qwen/Qwen3.5-397B-A17B/o_proj", + "Qwen/Qwen3.5-397B-A17B/shared_expert.down_proj", + "amd/Kimi-K2.5-MXFP4/kv_b_proj", + "amd/Qwen3.5-397B-A17B-MXFP4/o_proj", + "amd/Qwen3.5-397B-A17B-MXFP4/shared_expert.down_proj", + "moonshotai/Kimi-K2.5/kv_b_proj", + "nvidia/DeepSeek-R1-0528-FP4-V2/kv_b_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/kv_b_proj", + "nvidia/Kimi-K2.5-NVFP4/kv_b_proj", + "nvidia/Kimi-K2.6-NVFP4/kv_b_proj", + "nvidia/Qwen3.5-397B-A17B-NVFP4/o_proj", + "nvidia/Qwen3.5-397B-A17B-NVFP4/shared_expert.down_proj" + ] }, { "type": "gemm", "args": { - "m": 860160, + "m": 120, "n": 7168, - "k": 4096, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "k": 1024, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/o_proj", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] }, { "type": "gemm", "args": { - "m": 860160, - "n": 4608, - "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 120, + "n": 7168, + "k": 1152, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/down_proj" + ] }, { "type": "gemm", "args": { - "m": 860160, + "m": 120, "n": 7168, - "k": 4608, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "k": 2048, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/o_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/o_proj" + ] }, { "type": "gemm", "args": { - "m": 105, - "n": 1536, - "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 120, + "n": 7168, + "k": 2304, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/down_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/down_proj", + "nvidia/Kimi-K2.5-NVFP4/down_proj" + ] }, { "type": "gemm", "args": { - "m": 105, - "n": 6144, - "k": 1536, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 120, + "n": 7168, + "k": 9216, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/down_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/down_proj", + "nvidia/Kimi-K2.5-NVFP4/down_proj" + ] }, { "type": "gemm", "args": { - "m": 105, - "n": 576, - "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 120, + "n": 7168, + "k": 16384, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/o_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/o_proj" + ] }, { "type": "gemm", "args": { - "m": 105, - "n": 8192, - "k": 512, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 120, + "n": 7168, + "k": 16384, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 32 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 1, + 32 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "amd/DeepSeek-R1-0528-MXFP4-Preview/o_proj", + "amd/DeepSeek-R1-0528-MXFP4-v2/o_proj", + "amd/DeepSeek-R1-0528-MXFP4/o_proj" + ] }, { "type": "gemm", "args": { - "m": 105, + "m": 120, "n": 7168, - "k": 4096, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "k": 16384, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] }, { "type": "gemm", "args": { - "m": 105, - "n": 4608, + "m": 120, + "n": 9216, "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/gate_up_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/gate_up_proj", + "nvidia/Kimi-K2.5-NVFP4/gate_up_proj" + ] }, { "type": "gemm", "args": { - "m": 105, - "n": 7168, - "k": 4608, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 120, + "n": 9216, + "k": 7168, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 32 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 1, + 32 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "amd/DeepSeek-R1-0528-MXFP4-Preview/gate_up_proj", + "amd/DeepSeek-R1-0528-MXFP4-v2/gate_up_proj", + "amd/DeepSeek-R1-0528-MXFP4/gate_up_proj", + "amd/Kimi-K2.5-MXFP4/gate_up_proj" + ] }, { "type": "gemm", "args": { - "m": 258048, - "n": 1536, - "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 120, + "n": 24576, + "k": 1536, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/q_b_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/q_b_proj" + ] }, { "type": "gemm", "args": { - "m": 258048, + "m": 120, "n": 24576, "k": 1536, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 32 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 1, + 32 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "amd/DeepSeek-R1-0528-MXFP4-Preview/q_b_proj", + "amd/DeepSeek-R1-0528-MXFP4-v2/q_b_proj", + "amd/DeepSeek-R1-0528-MXFP4/q_b_proj" + ] }, { "type": "gemm", "args": { - "m": 258048, - "n": 576, - "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 120, + "n": 32768, + "k": 512, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/kv_b_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/kv_b_proj" + ] }, { "type": "gemm", "args": { - "m": 258048, + "m": 120, "n": 32768, "k": 512, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 32 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 1, + 32 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "amd/DeepSeek-R1-0528-MXFP4-Preview/kv_b_proj", + "amd/DeepSeek-R1-0528-MXFP4-v2/kv_b_proj", + "amd/DeepSeek-R1-0528-MXFP4/kv_b_proj" + ] }, { "type": "gemm", "args": { - "m": 258048, - "n": 7168, - "k": 16384, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 128, + "n": 128, + "k": 2880, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/gpt-oss-120b-w-mxfp4-a-fp8/k_proj", + "amd/gpt-oss-120b-w-mxfp4-a-fp8/v_proj", + "openai/gpt-oss-120b/k_proj", + "openai/gpt-oss-120b/v_proj" + ] }, { "type": "gemm", "args": { - "m": 258048, - "n": 9216, - "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 128, + "n": 128, + "k": 3072, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "MiniMaxAI/MiniMax-M2.5/k_proj", + "MiniMaxAI/MiniMax-M2.5/v_proj" + ] }, { "type": "gemm", "args": { - "m": 258048, - "n": 7168, - "k": 9216, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 128, + "n": 128, + "k": 4096, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "Qwen/Qwen3.5-397B-A17B/shared_expert.gate_up_proj", + "amd/Qwen3.5-397B-A17B-MXFP4/shared_expert.gate_up_proj", + "nvidia/Qwen3.5-397B-A17B-NVFP4/shared_expert.gate_up_proj" + ] }, { "type": "gemm", "args": { - "m": 31, - "n": 1536, - "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 128, + "n": 128, + "k": 4096, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "Qwen/Qwen3.5-397B-A17B-FP8/shared_expert.gate_up_proj" + ] }, { "type": "gemm", "args": { - "m": 31, - "n": 24576, - "k": 1536, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 128, + "n": 192, + "k": 3072, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "MiniMaxAI/MiniMax-M2.5/experts.gate_up_proj" + ] }, { "type": "gemm", "args": { - "m": 31, - "n": 576, - "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 128, + "n": 256, + "k": 2880, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/gpt-oss-120b-w-mxfp4-a-fp8/k_proj", + "amd/gpt-oss-120b-w-mxfp4-a-fp8/q_proj", + "amd/gpt-oss-120b-w-mxfp4-a-fp8/v_proj", + "openai/gpt-oss-120b/k_proj", + "openai/gpt-oss-120b/q_proj", + "openai/gpt-oss-120b/v_proj" + ] }, { "type": "gemm", "args": { - "m": 31, - "n": 32768, - "k": 512, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 128, + "n": 256, + "k": 3072, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/MiniMax-M2.5-MXFP4/k_proj", + "amd/MiniMax-M2.5-MXFP4/v_proj", + "nvidia/MiniMax-M2.5-NVFP4/k_proj", + "nvidia/MiniMax-M2.5-NVFP4/v_proj" + ] }, { "type": "gemm", "args": { - "m": 31, - "n": 7168, - "k": 16384, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 128, + "n": 256, + "k": 3072, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "MiniMaxAI/MiniMax-M2.5/k_proj", + "MiniMaxAI/MiniMax-M2.5/v_proj" + ] }, { "type": "gemm", "args": { - "m": 31, - "n": 9216, - "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 128, + "n": 256, + "k": 4096, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "Qwen/Qwen3.5-397B-A17B/k_proj", + "Qwen/Qwen3.5-397B-A17B/shared_expert.gate_up_proj", + "Qwen/Qwen3.5-397B-A17B/v_proj", + "amd/Qwen3.5-397B-A17B-MXFP4/k_proj", + "amd/Qwen3.5-397B-A17B-MXFP4/shared_expert.gate_up_proj", + "amd/Qwen3.5-397B-A17B-MXFP4/v_proj", + "nvidia/Qwen3.5-397B-A17B-NVFP4/k_proj", + "nvidia/Qwen3.5-397B-A17B-NVFP4/shared_expert.gate_up_proj", + "nvidia/Qwen3.5-397B-A17B-NVFP4/v_proj" + ] }, { "type": "gemm", "args": { - "m": 31, - "n": 7168, - "k": 9216, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 128, + "n": 256, + "k": 4096, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "Qwen/Qwen3.5-397B-A17B-FP8/k_proj", + "Qwen/Qwen3.5-397B-A17B-FP8/shared_expert.gate_up_proj", + "Qwen/Qwen3.5-397B-A17B-FP8/v_proj" + ] }, { "type": "gemm", "args": { - "m": 516096, - "n": 1536, - "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 128, + "n": 384, + "k": 3072, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "MiniMaxAI/MiniMax-M2.5/experts.gate_up_proj", + "MiniMaxAI/MiniMax-M2.5/q_proj" + ] }, { "type": "gemm", "args": { - "m": 516096, - "n": 3072, - "k": 1536, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 128, + "n": 512, + "k": 2880, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/gpt-oss-120b-w-mxfp4-a-fp8/k_proj", + "amd/gpt-oss-120b-w-mxfp4-a-fp8/q_proj", + "amd/gpt-oss-120b-w-mxfp4-a-fp8/v_proj", + "openai/gpt-oss-120b/k_proj", + "openai/gpt-oss-120b/q_proj", + "openai/gpt-oss-120b/v_proj" + ] }, { "type": "gemm", "args": { - "m": 516096, - "n": 576, - "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 128, + "n": 512, + "k": 3072, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/MiniMax-M2.5-MXFP4/k_proj", + "amd/MiniMax-M2.5-MXFP4/v_proj", + "nvidia/MiniMax-M2.5-NVFP4/k_proj", + "nvidia/MiniMax-M2.5-NVFP4/v_proj" + ] }, { "type": "gemm", "args": { - "m": 516096, - "n": 4096, - "k": 512, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 128, + "n": 512, + "k": 3072, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "MiniMaxAI/MiniMax-M2.5/k_proj", + "MiniMaxAI/MiniMax-M2.5/v_proj" + ] }, { "type": "gemm", "args": { - "m": 516096, - "n": 7168, - "k": 2048, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 128, + "n": 512, + "k": 4096, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "Qwen/Qwen3.5-397B-A17B/k_proj", + "Qwen/Qwen3.5-397B-A17B/q_proj", + "Qwen/Qwen3.5-397B-A17B/shared_expert.gate_up_proj", + "Qwen/Qwen3.5-397B-A17B/v_proj", + "amd/Qwen3.5-397B-A17B-MXFP4/k_proj", + "amd/Qwen3.5-397B-A17B-MXFP4/q_proj", + "amd/Qwen3.5-397B-A17B-MXFP4/shared_expert.gate_up_proj", + "amd/Qwen3.5-397B-A17B-MXFP4/v_proj", + "nvidia/Qwen3.5-397B-A17B-NVFP4/k_proj", + "nvidia/Qwen3.5-397B-A17B-NVFP4/q_proj", + "nvidia/Qwen3.5-397B-A17B-NVFP4/shared_expert.gate_up_proj", + "nvidia/Qwen3.5-397B-A17B-NVFP4/v_proj" + ] }, { "type": "gemm", "args": { - "m": 516096, - "n": 2304, - "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 128, + "n": 512, + "k": 4096, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "Qwen/Qwen3.5-397B-A17B-FP8/k_proj", + "Qwen/Qwen3.5-397B-A17B-FP8/q_proj", + "Qwen/Qwen3.5-397B-A17B-FP8/shared_expert.gate_up_proj", + "Qwen/Qwen3.5-397B-A17B-FP8/v_proj", + "deepseek-ai/DeepSeek-V4.1-Flash/attn.wo_a" + ] }, { "type": "gemm", "args": { - "m": 516096, - "n": 7168, - "k": 2304, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 128, + "n": 576, + "k": 6144, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/GLM-5.1-MXFP4/kv_a_proj_with_mqa", + "amd/GLM-5.2-MXFP4/kv_a_proj_with_mqa", + "nvidia/GLM-5-NVFP4/kv_a_proj_with_mqa", + "nvidia/GLM-5.1-NVFP4/kv_a_proj_with_mqa", + "nvidia/GLM-5.2-NVFP4/kv_a_proj_with_mqa" + ] }, { "type": "gemm", "args": { - "m": 63, - "n": 1536, + "m": 128, + "n": 576, + "k": 6144, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "zai-org/GLM-5-FP8/kv_a_proj_with_mqa", + "zai-org/GLM-5.1-FP8/kv_a_proj_with_mqa", + "zai-org/GLM-5.2-FP8/kv_a_proj_with_mqa" + ] + }, + { + "type": "gemm", + "args": { + "m": 128, + "n": 576, "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/Kimi-K2.5-MXFP4/kv_a_proj_with_mqa", + "moonshotai/Kimi-K2.5/kv_a_proj_with_mqa", + "nvidia/DeepSeek-R1-0528-FP4-V2/kv_a_proj_with_mqa", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/kv_a_proj_with_mqa", + "nvidia/Kimi-K2.5-NVFP4/kv_a_proj_with_mqa", + "nvidia/Kimi-K2.6-NVFP4/kv_a_proj_with_mqa" + ] }, { "type": "gemm", "args": { - "m": 63, - "n": 3072, - "k": 1536, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 128, + "n": 576, + "k": 7168, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 32 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 1, + 32 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "amd/DeepSeek-R1-0528-MXFP4-Preview/kv_a_proj_with_mqa", + "amd/DeepSeek-R1-0528-MXFP4-v2/kv_a_proj_with_mqa", + "amd/DeepSeek-R1-0528-MXFP4/kv_a_proj_with_mqa" + ] }, { "type": "gemm", "args": { - "m": 63, + "m": 128, "n": 576, "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/kv_a_proj_with_mqa" + ] }, { "type": "gemm", "args": { - "m": 63, - "n": 4096, - "k": 512, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 128, + "n": 768, + "k": 3072, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/MiniMax-M2.5-NVFP4/experts.gate_up_proj" + ] }, { "type": "gemm", "args": { - "m": 63, - "n": 7168, - "k": 2048, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 128, + "n": 768, + "k": 3072, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 32 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 1, + 32 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "amd/MiniMax-M2.5-MXFP4/experts.gate_up_proj" + ] }, { "type": "gemm", "args": { - "m": 63, - "n": 2304, - "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 128, + "n": 768, + "k": 3072, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "MiniMaxAI/MiniMax-M2.5/experts.gate_up_proj", + "MiniMaxAI/MiniMax-M2.5/q_proj" + ] }, { "type": "gemm", "args": { - "m": 63, - "n": 7168, - "k": 2304, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 128, + "n": 1024, + "k": 2880, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/gpt-oss-120b-w-mxfp4-a-fp8/q_proj", + "openai/gpt-oss-120b/q_proj" + ] }, { "type": "gemm", "args": { - "m": 16384, - "n": 9216, - "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 128, + "n": 1024, + "k": 3072, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "MiniMaxAI/MiniMax-M2.5/k_proj", + "MiniMaxAI/MiniMax-M2.5/qkv_proj", + "MiniMaxAI/MiniMax-M2.5/v_proj" + ] }, { "type": "gemm", "args": { - "m": 16384, - "n": 7168, - "k": 9216, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 128, + "n": 1024, + "k": 4096, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "Qwen/Qwen3.5-397B-A17B/q_proj", + "Qwen/Qwen3.5-397B-A17B/shared_expert.gate_up_proj", + "amd/Qwen3.5-397B-A17B-MXFP4/q_proj", + "amd/Qwen3.5-397B-A17B-MXFP4/shared_expert.gate_up_proj", + "nvidia/Qwen3.5-397B-A17B-NVFP4/q_proj", + "nvidia/Qwen3.5-397B-A17B-NVFP4/shared_expert.gate_up_proj" + ] }, { "type": "gemm", "args": { - "m": 49152, - "n": 24576, - "k": 1536, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 128, + "n": 1024, + "k": 4096, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "Qwen/Qwen3.5-397B-A17B-FP8/q_proj", + "Qwen/Qwen3.5-397B-A17B-FP8/shared_expert.gate_up_proj", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_a", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_a", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_a" + ] }, { "type": "gemm", "args": { - "m": 49152, - "n": 32768, - "k": 512, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 128, + "n": 1088, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] }, { "type": "gemm", "args": { - "m": 49152, - "n": 7168, - "k": 16384, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 128, + "n": 1152, + "k": 7168, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-NVFP4-v2/gate_up_proj" + ] }, { "type": "gemm", "args": { - "m": 49152, - "n": 9216, + "m": 128, + "n": 1152, "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/gate_up_proj" + ] }, { "type": "gemm", "args": { - "m": 49152, - "n": 7168, - "k": 9216, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 128, + "n": 1536, + "k": 1536, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/Kimi-K2.5-MXFP4/q_b_proj", + "moonshotai/Kimi-K2.5/q_b_proj", + "nvidia/DeepSeek-R1-0528-FP4-V2/q_b_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/q_b_proj", + "nvidia/Kimi-K2.5-NVFP4/q_b_proj", + "nvidia/Kimi-K2.6-NVFP4/q_b_proj" + ] }, { "type": "gemm", "args": { - "m": 6, - "n": 9216, - "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 128, + "n": 1536, + "k": 3072, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/MiniMax-M2.5-NVFP4/experts.gate_up_proj" + ] }, { "type": "gemm", "args": { - "m": 6, - "n": 7168, - "k": 9216, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 128, + "n": 1536, + "k": 3072, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 32 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 1, + 32 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "amd/MiniMax-M2.5-MXFP4/experts.gate_up_proj" + ] }, { "type": "gemm", "args": { - "m": 98304, + "m": 128, "n": 1536, - "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "k": 3072, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "MiniMaxAI/MiniMax-M2.5/experts.gate_up_proj", + "MiniMaxAI/MiniMax-M2.5/q_proj" + ] }, { "type": "gemm", "args": { - "m": 98304, - "n": 6144, - "k": 1536, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 128, + "n": 1536, + "k": 6144, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "zai-org/GLM-5-FP8/gate_up_proj", + "zai-org/GLM-5.1-FP8/gate_up_proj", + "zai-org/GLM-5.2-FP8/gate_up_proj" + ] }, { "type": "gemm", "args": { - "m": 98304, - "n": 576, + "m": 128, + "n": 1536, "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/Kimi-K2.5-MXFP4/q_a_proj", + "moonshotai/Kimi-K2.5/q_a_proj", + "nvidia/DeepSeek-R1-0528-FP4-V2/q_a_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/q_a_proj", + "nvidia/Kimi-K2.5-NVFP4/q_a_proj", + "nvidia/Kimi-K2.6-NVFP4/q_a_proj" + ] }, { "type": "gemm", "args": { - "m": 98304, - "n": 8192, - "k": 512, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 128, + "n": 1536, + "k": 7168, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 32 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 1, + 32 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "amd/DeepSeek-R1-0528-MXFP4-Preview/q_a_proj", + "amd/DeepSeek-R1-0528-MXFP4-v2/q_a_proj", + "amd/DeepSeek-R1-0528-MXFP4/q_a_proj" + ] }, { "type": "gemm", "args": { - "m": 98304, - "n": 7168, - "k": 4096, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 128, + "n": 1536, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/gate_up_proj", + "deepseek-ai/DeepSeek-R1-0528/q_a_proj", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w3" + ] }, { "type": "gemm", "args": { - "m": 98304, - "n": 4608, + "m": 128, + "n": 1536, "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w3" + ] }, { "type": "gemm", "args": { - "m": 98304, - "n": 7168, - "k": 4608, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 128, + "n": 2048, + "k": 512, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/Kimi-K2.5-MXFP4/kv_b_proj", + "moonshotai/Kimi-K2.5/kv_b_proj", + "nvidia/DeepSeek-R1-0528-FP4-V2/kv_b_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/kv_b_proj", + "nvidia/Kimi-K2.5-NVFP4/kv_b_proj", + "nvidia/Kimi-K2.6-NVFP4/kv_b_proj" + ] }, { "type": "gemm", "args": { - "m": 12, - "n": 6144, - "k": 1536, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 128, + "n": 2048, + "k": 2048, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "zai-org/GLM-5-FP8/q_b_proj", + "zai-org/GLM-5.1-FP8/q_b_proj", + "zai-org/GLM-5.2-FP8/q_b_proj" + ] }, { "type": "gemm", "args": { - "m": 12, - "n": 8192, - "k": 512, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 128, + "n": 2048, + "k": 2880, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/gpt-oss-120b-w-mxfp4-a-fp8/q_proj", + "openai/gpt-oss-120b/q_proj" + ] }, { "type": "gemm", "args": { - "m": 12, - "n": 7168, + "m": 128, + "n": 2048, "k": 4096, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "Qwen/Qwen3.5-397B-A17B/q_proj", + "Qwen/Qwen3.5-397B-A17B/shared_expert.gate_up_proj", + "amd/Qwen3.5-397B-A17B-MXFP4/q_proj", + "amd/Qwen3.5-397B-A17B-MXFP4/shared_expert.gate_up_proj", + "nvidia/Qwen3.5-397B-A17B-NVFP4/q_proj", + "nvidia/Qwen3.5-397B-A17B-NVFP4/shared_expert.gate_up_proj" + ] }, { "type": "gemm", "args": { - "m": 2412544, - "n": 1536, - "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 128, + "n": 2048, + "k": 4096, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "Qwen/Qwen3.5-397B-A17B-FP8/q_proj", + "Qwen/Qwen3.5-397B-A17B-FP8/shared_expert.gate_up_proj" + ] }, { "type": "gemm", "args": { - "m": 2412544, - "n": 24576, - "k": 1536, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 128, + "n": 2048, + "k": 6144, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/GLM-5.1-MXFP4/q_a_proj", + "amd/GLM-5.2-MXFP4/q_a_proj", + "amd/MiniMax-M3-MXFP4/q_proj", + "nvidia/GLM-5-NVFP4/q_a_proj", + "nvidia/GLM-5.1-NVFP4/q_a_proj", + "nvidia/GLM-5.2-NVFP4/q_a_proj" + ] }, { "type": "gemm", "args": { - "m": 2412544, - "n": 576, - "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 128, + "n": 2048, + "k": 6144, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "zai-org/GLM-5-FP8/q_a_proj", + "zai-org/GLM-5.1-FP8/q_a_proj", + "zai-org/GLM-5.2-FP8/q_a_proj" + ] }, { "type": "gemm", "args": { - "m": 2412544, - "n": 32768, - "k": 512, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 128, + "n": 2304, + "k": 7168, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/gate_up_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/gate_up_proj", + "nvidia/Kimi-K2.5-NVFP4/gate_up_proj" + ] }, { "type": "gemm", "args": { - "m": 2412544, - "n": 7168, - "k": 16384, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 128, + "n": 2880, + "k": 1024, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/gpt-oss-120b-w-mxfp4-a-fp8/o_proj", + "openai/gpt-oss-120b/o_proj" + ] }, { "type": "gemm", "args": { - "m": 2412544, - "n": 9216, - "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 128, + "n": 2880, + "k": 2048, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/gpt-oss-120b-w-mxfp4-a-fp8/o_proj", + "openai/gpt-oss-120b/o_proj" + ] }, { "type": "gemm", "args": { - "m": 2412544, - "n": 7168, - "k": 9216, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 128, + "n": 2880, + "k": 4096, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/gpt-oss-120b-w-mxfp4-a-fp8/o_proj", + "openai/gpt-oss-120b/o_proj" + ] }, { "type": "gemm", "args": { - "m": 294, - "n": 1536, - "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 128, + "n": 3072, + "k": 192, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "MiniMaxAI/MiniMax-M2.5/experts.down_proj" + ] }, { "type": "gemm", "args": { - "m": 294, - "n": 24576, - "k": 1536, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 128, + "n": 3072, + "k": 384, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/MiniMax-M2.5-NVFP4/experts.down_proj" + ] }, { "type": "gemm", "args": { - "m": 294, - "n": 576, - "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 128, + "n": 3072, + "k": 384, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 32 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 1, + 32 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "amd/MiniMax-M2.5-MXFP4/experts.down_proj" + ] }, { "type": "gemm", "args": { - "m": 294, - "n": 32768, - "k": 512, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 128, + "n": 3072, + "k": 384, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "MiniMaxAI/MiniMax-M2.5/experts.down_proj", + "MiniMaxAI/MiniMax-M2.5/o_proj" + ] }, { "type": "gemm", "args": { - "m": 294, - "n": 7168, - "k": 16384, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 128, + "n": 3072, + "k": 768, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/MiniMax-M2.5-NVFP4/experts.down_proj" + ] }, { "type": "gemm", "args": { - "m": 294, - "n": 9216, - "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 128, + "n": 3072, + "k": 768, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 32 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 1, + 32 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "amd/MiniMax-M2.5-MXFP4/experts.down_proj" + ] }, { "type": "gemm", "args": { - "m": 294, - "n": 7168, - "k": 9216, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 128, + "n": 3072, + "k": 768, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "MiniMaxAI/MiniMax-M2.5/experts.down_proj", + "MiniMaxAI/MiniMax-M2.5/o_proj" + ] }, { "type": "gemm", "args": { - "m": 603136, - "n": 1536, - "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 128, + "n": 3072, + "k": 1536, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/Kimi-K2.5-MXFP4/q_b_proj", + "amd/MiniMax-M2.5-MXFP4/o_proj", + "moonshotai/Kimi-K2.5/q_b_proj", + "nvidia/DeepSeek-R1-0528-FP4-V2/q_b_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/q_b_proj", + "nvidia/Kimi-K2.5-NVFP4/q_b_proj", + "nvidia/Kimi-K2.6-NVFP4/q_b_proj", + "nvidia/MiniMax-M2.5-NVFP4/o_proj" + ] }, { "type": "gemm", "args": { - "m": 603136, - "n": 24576, + "m": 128, + "n": 3072, "k": 1536, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/MiniMax-M2.5-NVFP4/experts.down_proj" + ] }, { "type": "gemm", "args": { - "m": 603136, - "n": 576, - "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 128, + "n": 3072, + "k": 1536, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 32 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 1, + 32 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "amd/DeepSeek-R1-0528-MXFP4-Preview/q_b_proj", + "amd/DeepSeek-R1-0528-MXFP4-v2/q_b_proj", + "amd/DeepSeek-R1-0528-MXFP4/q_b_proj", + "amd/MiniMax-M2.5-MXFP4/experts.down_proj" + ] }, { "type": "gemm", "args": { - "m": 603136, - "n": 32768, - "k": 512, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 128, + "n": 3072, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "MiniMaxAI/MiniMax-M2.5/experts.down_proj", + "MiniMaxAI/MiniMax-M2.5/o_proj", + "deepseek-ai/DeepSeek-R1-0528/q_b_proj" + ] }, { "type": "gemm", "args": { - "m": 603136, - "n": 7168, - "k": 16384, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 128, + "n": 3072, + "k": 3072, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/MiniMax-M2.5-MXFP4/o_proj", + "amd/MiniMax-M2.5-MXFP4/q_proj", + "nvidia/MiniMax-M2.5-NVFP4/o_proj", + "nvidia/MiniMax-M2.5-NVFP4/q_proj" + ] }, { "type": "gemm", "args": { - "m": 603136, - "n": 2304, - "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 128, + "n": 3072, + "k": 3072, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/MiniMax-M2.5-NVFP4/experts.gate_up_proj" + ] }, { "type": "gemm", "args": { - "m": 603136, - "n": 7168, - "k": 2304, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 128, + "n": 3072, + "k": 3072, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 32 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 1, + 32 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "amd/MiniMax-M2.5-MXFP4/experts.gate_up_proj" + ] }, { "type": "gemm", "args": { - "m": 73, - "n": 1536, - "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 128, + "n": 3072, + "k": 3072, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "MiniMaxAI/MiniMax-M2.5/experts.gate_up_proj", + "MiniMaxAI/MiniMax-M2.5/o_proj", + "MiniMaxAI/MiniMax-M2.5/q_proj" + ] }, { "type": "gemm", "args": { - "m": 73, - "n": 24576, - "k": 1536, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 128, + "n": 3072, + "k": 6144, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/GLM-5.1-MXFP4/gate_up_proj", + "amd/GLM-5.2-MXFP4/gate_up_proj", + "amd/MiniMax-M2.5-MXFP4/o_proj", + "amd/MiniMax-M3-MXFP4/gate_up_proj", + "nvidia/GLM-5.1-NVFP4/gate_up_proj", + "nvidia/GLM-5.2-NVFP4/gate_up_proj", + "nvidia/MiniMax-M2.5-NVFP4/o_proj" + ] }, { "type": "gemm", "args": { - "m": 73, - "n": 576, - "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 128, + "n": 3072, + "k": 6144, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "MiniMaxAI/MiniMax-M2.5/o_proj", + "zai-org/GLM-5-FP8/gate_up_proj", + "zai-org/GLM-5.1-FP8/gate_up_proj", + "zai-org/GLM-5.2-FP8/gate_up_proj" + ] }, { "type": "gemm", "args": { - "m": 73, - "n": 32768, + "m": 128, + "n": 3584, "k": 512, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "zai-org/GLM-5-FP8/kv_b_proj", + "zai-org/GLM-5.1-FP8/kv_b_proj", + "zai-org/GLM-5.2-FP8/kv_b_proj" + ] }, { "type": "gemm", "args": { - "m": 73, - "n": 7168, - "k": 16384, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 128, + "n": 4096, + "k": 128, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "Qwen/Qwen3.5-397B-A17B/shared_expert.down_proj", + "amd/Qwen3.5-397B-A17B-MXFP4/shared_expert.down_proj", + "nvidia/Qwen3.5-397B-A17B-NVFP4/shared_expert.down_proj" + ] }, { "type": "gemm", "args": { - "m": 73, - "n": 2304, - "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 128, + "n": 4096, + "k": 128, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "Qwen/Qwen3.5-397B-A17B-FP8/shared_expert.down_proj" + ] }, { "type": "gemm", "args": { - "m": 73, - "n": 7168, - "k": 2304, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 128, + "n": 4096, + "k": 256, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "Qwen/Qwen3.5-397B-A17B/shared_expert.down_proj", + "amd/Qwen3.5-397B-A17B-MXFP4/shared_expert.down_proj", + "nvidia/Qwen3.5-397B-A17B-NVFP4/shared_expert.down_proj" + ] }, { "type": "gemm", "args": { - "m": 4476928, - "n": 1536, - "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 128, + "n": 4096, + "k": 256, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "Qwen/Qwen3.5-397B-A17B-FP8/shared_expert.down_proj" + ] }, { "type": "gemm", "args": { - "m": 4476928, - "n": 24576, - "k": 1536, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 128, + "n": 4096, + "k": 512, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "Qwen/Qwen3.5-397B-A17B/o_proj", + "Qwen/Qwen3.5-397B-A17B/shared_expert.down_proj", + "amd/Kimi-K2.5-MXFP4/kv_b_proj", + "amd/Qwen3.5-397B-A17B-MXFP4/o_proj", + "amd/Qwen3.5-397B-A17B-MXFP4/shared_expert.down_proj", + "moonshotai/Kimi-K2.5/kv_b_proj", + "nvidia/DeepSeek-R1-0528-FP4-V2/kv_b_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/kv_b_proj", + "nvidia/Kimi-K2.5-NVFP4/kv_b_proj", + "nvidia/Kimi-K2.6-NVFP4/kv_b_proj", + "nvidia/Qwen3.5-397B-A17B-NVFP4/o_proj", + "nvidia/Qwen3.5-397B-A17B-NVFP4/shared_expert.down_proj" + ] }, { "type": "gemm", "args": { - "m": 4476928, - "n": 576, - "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 128, + "n": 4096, + "k": 512, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 32 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 1, + 32 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "amd/DeepSeek-R1-0528-MXFP4-Preview/kv_b_proj", + "amd/DeepSeek-R1-0528-MXFP4-v2/kv_b_proj", + "amd/DeepSeek-R1-0528-MXFP4/kv_b_proj", + "amd/Qwen3.5-397B-A17B-MXFP4-AttnFP8-V2/shared_expert.down_proj" + ] }, { "type": "gemm", "args": { - "m": 4476928, - "n": 32768, + "m": 128, + "n": 4096, "k": 512, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "Qwen/Qwen3.5-397B-A17B-FP8/o_proj", + "Qwen/Qwen3.5-397B-A17B-FP8/shared_expert.down_proj", + "deepseek-ai/DeepSeek-R1-0528/kv_b_proj" + ] }, { "type": "gemm", "args": { - "m": 4476928, - "n": 7168, - "k": 16384, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 128, + "n": 4096, + "k": 1024, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "Qwen/Qwen3.5-397B-A17B/o_proj", + "Qwen/Qwen3.5-397B-A17B/shared_expert.down_proj", + "amd/Qwen3.5-397B-A17B-MXFP4/o_proj", + "amd/Qwen3.5-397B-A17B-MXFP4/shared_expert.down_proj", + "nvidia/Qwen3.5-397B-A17B-NVFP4/o_proj", + "nvidia/Qwen3.5-397B-A17B-NVFP4/shared_expert.down_proj" + ] }, { "type": "gemm", "args": { - "m": 4476928, - "n": 9216, - "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 128, + "n": 4096, + "k": 1024, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "Qwen/Qwen3.5-397B-A17B-FP8/o_proj", + "Qwen/Qwen3.5-397B-A17B-FP8/shared_expert.down_proj" + ] }, { "type": "gemm", "args": { - "m": 4476928, - "n": 7168, - "k": 9216, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 128, + "n": 4096, + "k": 2048, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "Qwen/Qwen3.5-397B-A17B/o_proj", + "amd/GLM-5.1-MXFP4/q_b_proj", + "amd/GLM-5.2-MXFP4/q_b_proj", + "amd/Qwen3.5-397B-A17B-MXFP4/o_proj", + "nvidia/GLM-5-NVFP4/q_b_proj", + "nvidia/GLM-5.1-NVFP4/q_b_proj", + "nvidia/GLM-5.2-NVFP4/q_b_proj", + "nvidia/Qwen3.5-397B-A17B-NVFP4/o_proj" + ] }, { "type": "gemm", "args": { - "m": 546, - "n": 1536, - "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 128, + "n": 4096, + "k": 2048, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "Qwen/Qwen3.5-397B-A17B-FP8/o_proj", + "zai-org/GLM-5-FP8/q_b_proj", + "zai-org/GLM-5.1-FP8/q_b_proj", + "zai-org/GLM-5.2-FP8/q_b_proj" + ] }, { "type": "gemm", "args": { - "m": 546, - "n": 24576, - "k": 1536, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 128, + "n": 4096, + "k": 2880, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/gpt-oss-120b-w-mxfp4-a-fp8/q_proj", + "openai/gpt-oss-120b/q_proj" + ] }, { "type": "gemm", "args": { - "m": 546, - "n": 576, - "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 128, + "n": 4096, + "k": 4096, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "Qwen/Qwen3.5-397B-A17B/o_proj", + "Qwen/Qwen3.5-397B-A17B/q_proj", + "amd/Qwen3.5-397B-A17B-MXFP4/o_proj", + "amd/Qwen3.5-397B-A17B-MXFP4/q_proj", + "nvidia/Qwen3.5-397B-A17B-NVFP4/o_proj", + "nvidia/Qwen3.5-397B-A17B-NVFP4/q_proj" + ] }, { "type": "gemm", "args": { - "m": 546, - "n": 32768, - "k": 512, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 128, + "n": 4096, + "k": 4096, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "Qwen/Qwen3.5-397B-A17B-FP8/o_proj", + "Qwen/Qwen3.5-397B-A17B-FP8/q_proj" + ] }, { "type": "gemm", "args": { - "m": 546, - "n": 7168, - "k": 16384, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 128, + "n": 4608, + "k": 7168, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/gate_up_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/gate_up_proj", + "nvidia/Kimi-K2.5-NVFP4/gate_up_proj" + ] }, { "type": "gemm", "args": { - "m": 546, - "n": 9216, + "m": 128, + "n": 4608, "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 32 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 1, + 32 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "amd/DeepSeek-R1-0528-MXFP4-Preview/gate_up_proj", + "amd/DeepSeek-R1-0528-MXFP4-v2/gate_up_proj", + "amd/DeepSeek-R1-0528-MXFP4/gate_up_proj", + "amd/Kimi-K2.5-MXFP4/gate_up_proj" + ] }, { "type": "gemm", "args": { - "m": 546, - "n": 7168, - "k": 9216, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 128, + "n": 4608, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/gate_up_proj" + ] }, { "type": "gemm", "args": { - "m": 1119232, - "n": 1536, - "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 128, + "n": 6144, + "k": 1536, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/GLM-5.1-MXFP4/down_proj", + "amd/GLM-5.2-MXFP4/down_proj", + "amd/Kimi-K2.5-MXFP4/q_b_proj", + "amd/MiniMax-M3-MXFP4/down_proj", + "moonshotai/Kimi-K2.5/q_b_proj", + "nvidia/DeepSeek-R1-0528-FP4-V2/q_b_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/q_b_proj", + "nvidia/GLM-5.1-NVFP4/down_proj", + "nvidia/GLM-5.2-NVFP4/down_proj", + "nvidia/Kimi-K2.5-NVFP4/q_b_proj", + "nvidia/Kimi-K2.6-NVFP4/q_b_proj" + ] }, { "type": "gemm", "args": { - "m": 1119232, - "n": 24576, + "m": 128, + "n": 6144, "k": 1536, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 32 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 1, + 32 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "amd/DeepSeek-R1-0528-MXFP4-Preview/q_b_proj", + "amd/DeepSeek-R1-0528-MXFP4-v2/q_b_proj", + "amd/DeepSeek-R1-0528-MXFP4/q_b_proj" + ] }, { "type": "gemm", "args": { - "m": 1119232, - "n": 576, - "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 128, + "n": 6144, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/q_b_proj", + "zai-org/GLM-5-FP8/down_proj", + "zai-org/GLM-5.1-FP8/down_proj", + "zai-org/GLM-5.2-FP8/down_proj" + ] }, { "type": "gemm", "args": { - "m": 1119232, - "n": 32768, - "k": 512, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 128, + "n": 6144, + "k": 2048, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "zai-org/GLM-5-FP8/o_proj", + "zai-org/GLM-5.1-FP8/o_proj", + "zai-org/GLM-5.2-FP8/o_proj" + ] }, { "type": "gemm", "args": { - "m": 1119232, - "n": 7168, - "k": 16384, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 128, + "n": 6144, + "k": 3072, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/GLM-5.1-MXFP4/down_proj", + "amd/GLM-5.2-MXFP4/down_proj", + "amd/MiniMax-M2.5-MXFP4/q_proj", + "amd/MiniMax-M3-MXFP4/down_proj", + "nvidia/GLM-5.1-NVFP4/down_proj", + "nvidia/GLM-5.2-NVFP4/down_proj", + "nvidia/MiniMax-M2.5-NVFP4/q_proj" + ] }, { "type": "gemm", "args": { - "m": 1119232, - "n": 2304, - "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 128, + "n": 6144, + "k": 3072, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/GLM-5-NVFP4/down_proj" + ] }, { "type": "gemm", "args": { - "m": 1119232, - "n": 7168, - "k": 2304, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 128, + "n": 6144, + "k": 3072, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "MiniMaxAI/MiniMax-M2.5/q_proj", + "zai-org/GLM-5-FP8/down_proj", + "zai-org/GLM-5.1-FP8/down_proj", + "zai-org/GLM-5.2-FP8/down_proj" + ] }, { "type": "gemm", "args": { - "m": 136, - "n": 1536, - "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 128, + "n": 6144, + "k": 4096, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/GLM-5.1-MXFP4/o_proj", + "amd/GLM-5.2-MXFP4/o_proj", + "amd/MiniMax-M3-MXFP4/o_proj", + "nvidia/GLM-5-NVFP4/o_proj", + "nvidia/GLM-5.1-NVFP4/o_proj", + "nvidia/GLM-5.2-NVFP4/o_proj" + ] }, { "type": "gemm", "args": { - "m": 136, - "n": 24576, - "k": 1536, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 128, + "n": 6144, + "k": 4096, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "zai-org/GLM-5-FP8/o_proj", + "zai-org/GLM-5.1-FP8/o_proj", + "zai-org/GLM-5.2-FP8/o_proj" + ] }, { "type": "gemm", "args": { - "m": 136, - "n": 576, - "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 128, + "n": 6144, + "k": 6144, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/GLM-5.1-MXFP4/down_proj", + "amd/GLM-5.1-MXFP4/gate_up_proj", + "amd/GLM-5.2-MXFP4/down_proj", + "amd/GLM-5.2-MXFP4/gate_up_proj", + "amd/MiniMax-M3-MXFP4/gate_up_proj", + "nvidia/GLM-5.1-NVFP4/down_proj", + "nvidia/GLM-5.1-NVFP4/gate_up_proj", + "nvidia/GLM-5.2-NVFP4/down_proj", + "nvidia/GLM-5.2-NVFP4/gate_up_proj" + ] }, { "type": "gemm", "args": { - "m": 136, - "n": 32768, - "k": 512, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 128, + "n": 6144, + "k": 6144, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/GLM-5-NVFP4/down_proj", + "nvidia/GLM-5-NVFP4/gate_up_proj" + ] }, { "type": "gemm", "args": { - "m": 136, - "n": 7168, - "k": 16384, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 128, + "n": 6144, + "k": 6144, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "zai-org/GLM-5-FP8/down_proj", + "zai-org/GLM-5-FP8/gate_up_proj", + "zai-org/GLM-5.1-FP8/down_proj", + "zai-org/GLM-5.1-FP8/gate_up_proj", + "zai-org/GLM-5.2-FP8/down_proj", + "zai-org/GLM-5.2-FP8/gate_up_proj" + ] }, { "type": "gemm", "args": { - "m": 136, - "n": 2304, - "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 128, + "n": 6144, + "k": 8192, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/GLM-5.1-MXFP4/o_proj", + "amd/GLM-5.2-MXFP4/o_proj", + "amd/MiniMax-M3-MXFP4/o_proj", + "nvidia/GLM-5-NVFP4/o_proj", + "nvidia/GLM-5.1-NVFP4/o_proj", + "nvidia/GLM-5.2-NVFP4/o_proj" + ] }, { "type": "gemm", "args": { - "m": 136, + "m": 128, "n": 7168, - "k": 2304, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "k": 512, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/GLM-5.1-MXFP4/kv_b_proj", + "amd/GLM-5.2-MXFP4/kv_b_proj", + "amd/Kimi-K2.5-MXFP4/o_proj", + "moonshotai/Kimi-K2.5/o_proj", + "nvidia/GLM-5-NVFP4/kv_b_proj", + "nvidia/GLM-5.1-NVFP4/kv_b_proj", + "nvidia/GLM-5.2-NVFP4/kv_b_proj", + "nvidia/Kimi-K2.5-NVFP4/o_proj", + "nvidia/Kimi-K2.6-NVFP4/o_proj" + ] }, { "type": "gemm", "args": { - "m": 8388608, - "n": 1536, - "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 128, + "n": 7168, + "k": 512, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "zai-org/GLM-5-FP8/kv_b_proj", + "zai-org/GLM-5.1-FP8/kv_b_proj", + "zai-org/GLM-5.2-FP8/kv_b_proj" + ] }, { "type": "gemm", "args": { - "m": 8388608, - "n": 24576, - "k": 1536, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 128, + "n": 7168, + "k": 576, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-NVFP4-v2/down_proj" + ] }, { "type": "gemm", "args": { - "m": 8388608, - "n": 576, - "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 128, + "n": 7168, + "k": 576, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/down_proj" + ] }, { "type": "gemm", "args": { - "m": 8388608, - "n": 32768, - "k": 512, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 128, + "n": 7168, + "k": 1024, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/Kimi-K2.5-MXFP4/o_proj", + "moonshotai/Kimi-K2.5/o_proj", + "nvidia/Kimi-K2.5-NVFP4/o_proj", + "nvidia/Kimi-K2.6-NVFP4/o_proj" + ] }, { "type": "gemm", "args": { - "m": 8388608, + "m": 128, "n": 7168, - "k": 16384, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "k": 1152, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/down_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/down_proj", + "nvidia/Kimi-K2.5-NVFP4/down_proj" + ] }, { "type": "gemm", "args": { - "m": 8388608, - "n": 9216, - "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 128, + "n": 7168, + "k": 2048, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/Kimi-K2.5-MXFP4/o_proj", + "moonshotai/Kimi-K2.5/o_proj", + "nvidia/Kimi-K2.5-NVFP4/o_proj", + "nvidia/Kimi-K2.6-NVFP4/o_proj" + ] }, { "type": "gemm", "args": { - "m": 8388608, + "m": 128, "n": 7168, - "k": 9216, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "k": 2048, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/o_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/o_proj" + ] }, { "type": "gemm", "args": { - "m": 1024, - "n": 9216, - "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 128, + "n": 7168, + "k": 2048, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 32 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 1, + 32 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "amd/DeepSeek-R1-0528-MXFP4-Preview/o_proj", + "amd/DeepSeek-R1-0528-MXFP4-v2/o_proj", + "amd/DeepSeek-R1-0528-MXFP4/o_proj" + ] }, { "type": "gemm", "args": { - "m": 1024, + "m": 128, "n": 7168, - "k": 9216, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "k": 2048, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/o_proj", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] }, { "type": "gemm", "args": { - "m": 2097152, - "n": 1536, - "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 128, + "n": 7168, + "k": 2304, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/down_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/down_proj", + "nvidia/Kimi-K2.5-NVFP4/down_proj" + ] }, { "type": "gemm", "args": { - "m": 2097152, - "n": 24576, - "k": 1536, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 128, + "n": 7168, + "k": 2304, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 32 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 1, + 32 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "amd/DeepSeek-R1-0528-MXFP4-Preview/down_proj", + "amd/DeepSeek-R1-0528-MXFP4-v2/down_proj", + "amd/DeepSeek-R1-0528-MXFP4/down_proj", + "amd/Kimi-K2.5-MXFP4/down_proj" + ] }, { "type": "gemm", "args": { - "m": 2097152, - "n": 576, - "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 128, + "n": 7168, + "k": 2304, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/down_proj" + ] }, { "type": "gemm", "args": { - "m": 2097152, - "n": 32768, - "k": 512, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 128, + "n": 7168, + "k": 4096, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/o_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/o_proj" + ] }, { "type": "gemm", "args": { - "m": 2097152, + "m": 128, "n": 7168, - "k": 16384, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "k": 4096, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 32 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 1, + 32 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "amd/DeepSeek-R1-0528-MXFP4-Preview/o_proj", + "amd/DeepSeek-R1-0528-MXFP4-v2/o_proj", + "amd/DeepSeek-R1-0528-MXFP4/o_proj" + ] }, { "type": "gemm", "args": { - "m": 2097152, - "n": 2304, - "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 128, + "n": 7168, + "k": 4608, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/down_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/down_proj", + "nvidia/Kimi-K2.5-NVFP4/down_proj" + ] }, { "type": "gemm", "args": { - "m": 2097152, + "m": 128, "n": 7168, - "k": 2304, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "k": 4608, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 32 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 1, + 32 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "amd/DeepSeek-R1-0528-MXFP4-Preview/down_proj", + "amd/DeepSeek-R1-0528-MXFP4-v2/down_proj", + "amd/DeepSeek-R1-0528-MXFP4/down_proj", + "amd/Kimi-K2.5-MXFP4/down_proj" + ] }, { "type": "gemm", "args": { - "m": 256, - "n": 1536, - "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 128, + "n": 7168, + "k": 8192, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/Kimi-K2.5-MXFP4/o_proj", + "moonshotai/Kimi-K2.5/o_proj", + "nvidia/Kimi-K2.5-NVFP4/o_proj", + "nvidia/Kimi-K2.6-NVFP4/o_proj" + ] }, { "type": "gemm", "args": { - "m": 256, - "n": 24576, - "k": 1536, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 128, + "n": 7168, + "k": 8192, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] }, { "type": "gemm", "args": { - "m": 256, - "n": 576, - "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 128, + "n": 7168, + "k": 16384, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/o_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/o_proj" + ] }, { "type": "gemm", "args": { - "m": 256, - "n": 32768, - "k": 512, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 128, + "n": 7168, + "k": 16384, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 32 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 1, + 32 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "amd/DeepSeek-R1-0528-MXFP4-Preview/o_proj", + "amd/DeepSeek-R1-0528-MXFP4-v2/o_proj", + "amd/DeepSeek-R1-0528-MXFP4/o_proj" + ] }, { "type": "gemm", "args": { - "m": 256, + "m": 128, "n": 7168, "k": 16384, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/o_proj", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] }, { "type": "gemm", "args": { - "m": 256, - "n": 2304, - "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 128, + "n": 7168, + "k": 16384, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] }, { "type": "gemm", "args": { - "m": 256, + "m": 128, "n": 7168, - "k": 2304, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "k": 65536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] }, { "type": "gemm", "args": { - "m": 2560, - "n": 1536, - "k": 7168, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 128, + "n": 8192, + "k": 512, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/Kimi-K2.5-MXFP4/kv_b_proj", + "moonshotai/Kimi-K2.5/kv_b_proj", + "nvidia/DeepSeek-R1-0528-FP4-V2/kv_b_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/kv_b_proj", + "nvidia/Kimi-K2.5-NVFP4/kv_b_proj", + "nvidia/Kimi-K2.6-NVFP4/kv_b_proj" + ] }, { "type": "gemm", "args": { - "m": 2560, - "n": 24576, - "k": 1536, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 128, + "n": 8192, + "k": 512, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 32 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 1, + 32 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "amd/DeepSeek-R1-0528-MXFP4-Preview/kv_b_proj", + "amd/DeepSeek-R1-0528-MXFP4-v2/kv_b_proj", + "amd/DeepSeek-R1-0528-MXFP4/kv_b_proj" + ] }, { "type": "gemm", "args": { - "m": 2560, - "n": 576, - "k": 7168, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 128, + "n": 8192, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_b" + ] }, { "type": "gemm", "args": { - "m": 2560, - "n": 32768, - "k": 512, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 128, + "n": 8192, + "k": 2048, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/GLM-5.1-MXFP4/q_b_proj", + "amd/GLM-5.2-MXFP4/q_b_proj", + "nvidia/GLM-5-NVFP4/q_b_proj", + "nvidia/GLM-5.1-NVFP4/q_b_proj", + "nvidia/GLM-5.2-NVFP4/q_b_proj" + ] }, { "type": "gemm", "args": { - "m": 2560, - "n": 7168, - "k": 16384, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 128, + "n": 12288, + "k": 1536, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/Kimi-K2.5-MXFP4/q_b_proj", + "moonshotai/Kimi-K2.5/q_b_proj", + "nvidia/DeepSeek-R1-0528-FP4-V2/q_b_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/q_b_proj", + "nvidia/Kimi-K2.5-NVFP4/q_b_proj", + "nvidia/Kimi-K2.6-NVFP4/q_b_proj" + ] }, { "type": "gemm", "args": { - "m": 2560, - "n": 4608, - "k": 7168, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 128, + "n": 14336, + "k": 512, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/GLM-5.1-MXFP4/kv_b_proj", + "amd/GLM-5.2-MXFP4/kv_b_proj", + "nvidia/GLM-5-NVFP4/kv_b_proj", + "nvidia/GLM-5.1-NVFP4/kv_b_proj", + "nvidia/GLM-5.2-NVFP4/kv_b_proj" + ] }, { "type": "gemm", "args": { - "m": 2560, - "n": 7168, - "k": 4608, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 128, + "n": 15360, + "k": 1024, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] }, { "type": "gemm", "args": { - "m": 5, - "n": 1536, - "k": 7168, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 128, + "n": 16384, + "k": 512, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/Kimi-K2.5-MXFP4/kv_b_proj", + "moonshotai/Kimi-K2.5/kv_b_proj", + "nvidia/DeepSeek-R1-0528-FP4-V2/kv_b_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/kv_b_proj", + "nvidia/Kimi-K2.5-NVFP4/kv_b_proj", + "nvidia/Kimi-K2.6-NVFP4/kv_b_proj" + ] }, { "type": "gemm", "args": { - "m": 5, - "n": 24576, + "m": 128, + "n": 16384, "k": 1536, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 5, - "n": 576, - "k": 7168, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_b" + ] }, { "type": "gemm", "args": { - "m": 5, - "n": 32768, - "k": 512, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 128, + "n": 24576, + "k": 1536, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/q_b_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/q_b_proj" + ] }, { "type": "gemm", "args": { - "m": 5, - "n": 7168, - "k": 16384, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 128, + "n": 24576, + "k": 1536, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 32 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 1, + 32 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "amd/DeepSeek-R1-0528-MXFP4-Preview/q_b_proj", + "amd/DeepSeek-R1-0528-MXFP4-v2/q_b_proj", + "amd/DeepSeek-R1-0528-MXFP4/q_b_proj" + ] }, { "type": "gemm", "args": { - "m": 5, - "n": 4608, - "k": 7168, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 128, + "n": 24576, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/q_b_proj" + ] }, { "type": "gemm", "args": { - "m": 5, - "n": 7168, - "k": 4608, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 128, + "n": 30720, + "k": 1024, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] }, { "type": "gemm", "args": { - "m": 10240, - "n": 1536, - "k": 7168, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 128, + "n": 32768, + "k": 512, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/kv_b_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/kv_b_proj" + ] }, { "type": "gemm", "args": { - "m": 10240, - "n": 3072, - "k": 1536, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 128, + "n": 32768, + "k": 512, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 32 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 1, + 32 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "amd/DeepSeek-R1-0528-MXFP4-Preview/kv_b_proj", + "amd/DeepSeek-R1-0528-MXFP4-v2/kv_b_proj", + "amd/DeepSeek-R1-0528-MXFP4/kv_b_proj" + ] }, { "type": "gemm", "args": { - "m": 10240, - "n": 576, - "k": 7168, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 128, + "n": 32768, + "k": 512, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/kv_b_proj" + ] }, { "type": "gemm", "args": { - "m": 10240, - "n": 4096, - "k": 512, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 128, + "n": 65536, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_b" + ] }, { "type": "gemm", "args": { - "m": 10240, - "n": 7168, - "k": 2048, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 128, + "n": 122880, + "k": 1024, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] }, { "type": "gemm", "args": { - "m": 10240, - "n": 2304, + "m": 132, + "n": 576, "k": 7168, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/Kimi-K2.5-MXFP4/kv_a_proj_with_mqa", + "moonshotai/Kimi-K2.5/kv_a_proj_with_mqa", + "nvidia/DeepSeek-R1-0528-FP4-V2/kv_a_proj_with_mqa", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/kv_a_proj_with_mqa", + "nvidia/Kimi-K2.5-NVFP4/kv_a_proj_with_mqa", + "nvidia/Kimi-K2.6-NVFP4/kv_a_proj_with_mqa" + ] }, { "type": "gemm", "args": { - "m": 10240, - "n": 7168, - "k": 2304, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 132, + "n": 1536, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/Kimi-K2.5-MXFP4/q_a_proj", + "moonshotai/Kimi-K2.5/q_a_proj", + "nvidia/DeepSeek-R1-0528-FP4-V2/q_a_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/q_a_proj", + "nvidia/Kimi-K2.5-NVFP4/q_a_proj", + "nvidia/Kimi-K2.6-NVFP4/q_a_proj" + ] }, { "type": "gemm", "args": { - "m": 20, - "n": 1536, + "m": 132, + "n": 2304, "k": 7168, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/gate_up_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/gate_up_proj", + "nvidia/Kimi-K2.5-NVFP4/gate_up_proj" + ] }, { "type": "gemm", "args": { - "m": 20, + "m": 132, "n": 3072, "k": 1536, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/Kimi-K2.5-MXFP4/q_b_proj", + "amd/MiniMax-M2.5-MXFP4/o_proj", + "moonshotai/Kimi-K2.5/q_b_proj", + "nvidia/DeepSeek-R1-0528-FP4-V2/q_b_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/q_b_proj", + "nvidia/Kimi-K2.5-NVFP4/q_b_proj", + "nvidia/Kimi-K2.6-NVFP4/q_b_proj", + "nvidia/MiniMax-M2.5-NVFP4/o_proj" + ] }, { "type": "gemm", "args": { - "m": 20, - "n": 576, - "k": 7168, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 132, + "n": 4096, + "k": 512, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "Qwen/Qwen3.5-397B-A17B/o_proj", + "Qwen/Qwen3.5-397B-A17B/shared_expert.down_proj", + "amd/Kimi-K2.5-MXFP4/kv_b_proj", + "amd/Qwen3.5-397B-A17B-MXFP4/o_proj", + "amd/Qwen3.5-397B-A17B-MXFP4/shared_expert.down_proj", + "moonshotai/Kimi-K2.5/kv_b_proj", + "nvidia/DeepSeek-R1-0528-FP4-V2/kv_b_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/kv_b_proj", + "nvidia/Kimi-K2.5-NVFP4/kv_b_proj", + "nvidia/Kimi-K2.6-NVFP4/kv_b_proj", + "nvidia/Qwen3.5-397B-A17B-NVFP4/o_proj", + "nvidia/Qwen3.5-397B-A17B-NVFP4/shared_expert.down_proj" + ] }, { "type": "gemm", "args": { - "m": 20, - "n": 4096, - "k": 512, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 132, + "n": 7168, + "k": 2048, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/o_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/o_proj" + ] }, { "type": "gemm", "args": { - "m": 20, + "m": 132, "n": 7168, - "k": 2048, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "k": 2304, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/down_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/down_proj", + "nvidia/Kimi-K2.5-NVFP4/down_proj" + ] }, { "type": "gemm", "args": { - "m": 20, - "n": 2304, + "m": 134, + "n": 576, "k": 7168, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/Kimi-K2.5-MXFP4/kv_a_proj_with_mqa", + "moonshotai/Kimi-K2.5/kv_a_proj_with_mqa", + "nvidia/DeepSeek-R1-0528-FP4-V2/kv_a_proj_with_mqa", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/kv_a_proj_with_mqa", + "nvidia/Kimi-K2.5-NVFP4/kv_a_proj_with_mqa", + "nvidia/Kimi-K2.6-NVFP4/kv_a_proj_with_mqa" + ] }, { "type": "gemm", "args": { - "m": 20, - "n": 7168, - "k": 2304, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 134, + "n": 576, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/kv_a_proj_with_mqa" + ] }, { "type": "gemm", "args": { - "m": 40960, - "n": 1536, + "m": 134, + "n": 1152, "k": 7168, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-NVFP4-v2/gate_up_proj" + ] }, { "type": "gemm", "args": { - "m": 40960, - "n": 24576, - "k": 1536, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 134, + "n": 1536, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/Kimi-K2.5-MXFP4/q_a_proj", + "moonshotai/Kimi-K2.5/q_a_proj", + "nvidia/DeepSeek-R1-0528-FP4-V2/q_a_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/q_a_proj", + "nvidia/Kimi-K2.5-NVFP4/q_a_proj", + "nvidia/Kimi-K2.6-NVFP4/q_a_proj" + ] }, { "type": "gemm", "args": { - "m": 40960, - "n": 576, + "m": 134, + "n": 1536, "k": 7168, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/gate_up_proj", + "deepseek-ai/DeepSeek-R1-0528/q_a_proj", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w3" + ] }, { "type": "gemm", "args": { - "m": 40960, - "n": 32768, - "k": 512, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 134, + "n": 2304, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/gate_up_proj" + ] }, { "type": "gemm", "args": { - "m": 40960, + "m": 134, "n": 7168, - "k": 16384, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "k": 576, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-NVFP4-v2/down_proj" + ] }, { "type": "gemm", "args": { - "m": 40960, - "n": 4608, - "k": 7168, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 134, + "n": 7168, + "k": 1152, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/down_proj" + ] }, { "type": "gemm", "args": { - "m": 40960, + "m": 134, "n": 7168, - "k": 4608, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "k": 2304, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/down_proj" + ] }, { "type": "gemm", "args": { - "m": 80, - "n": 4608, - "k": 7168, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 134, + "n": 7168, + "k": 8192, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/Kimi-K2.5-MXFP4/o_proj", + "moonshotai/Kimi-K2.5/o_proj", + "nvidia/Kimi-K2.5-NVFP4/o_proj", + "nvidia/Kimi-K2.6-NVFP4/o_proj" + ] }, { "type": "gemm", "args": { - "m": 80, + "m": 134, "n": 7168, - "k": 4608, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "k": 16384, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/o_proj", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] }, { "type": "gemm", "args": { - "m": 163840, - "n": 1536, - "k": 7168, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 134, + "n": 12288, + "k": 1536, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/Kimi-K2.5-MXFP4/q_b_proj", + "moonshotai/Kimi-K2.5/q_b_proj", + "nvidia/DeepSeek-R1-0528-FP4-V2/q_b_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/q_b_proj", + "nvidia/Kimi-K2.5-NVFP4/q_b_proj", + "nvidia/Kimi-K2.6-NVFP4/q_b_proj" + ] }, { "type": "gemm", "args": { - "m": 163840, - "n": 3072, - "k": 1536, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 134, + "n": 16384, + "k": 512, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/Kimi-K2.5-MXFP4/kv_b_proj", + "moonshotai/Kimi-K2.5/kv_b_proj", + "nvidia/DeepSeek-R1-0528-FP4-V2/kv_b_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/kv_b_proj", + "nvidia/Kimi-K2.5-NVFP4/kv_b_proj", + "nvidia/Kimi-K2.6-NVFP4/kv_b_proj" + ] }, { "type": "gemm", "args": { - "m": 163840, - "n": 576, - "k": 7168, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 134, + "n": 24576, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/q_b_proj" + ] }, { "type": "gemm", "args": { - "m": 163840, - "n": 4096, + "m": 134, + "n": 32768, "k": 512, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/kv_b_proj" + ] }, { "type": "gemm", "args": { - "m": 163840, - "n": 7168, - "k": 2048, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 136, + "n": 576, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/Kimi-K2.5-MXFP4/kv_a_proj_with_mqa", + "moonshotai/Kimi-K2.5/kv_a_proj_with_mqa", + "nvidia/DeepSeek-R1-0528-FP4-V2/kv_a_proj_with_mqa", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/kv_a_proj_with_mqa", + "nvidia/Kimi-K2.5-NVFP4/kv_a_proj_with_mqa", + "nvidia/Kimi-K2.6-NVFP4/kv_a_proj_with_mqa" + ] + }, + { + "type": "gemm", + "args": { + "m": 136, + "n": 1536, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/Kimi-K2.5-MXFP4/q_a_proj", + "moonshotai/Kimi-K2.5/q_a_proj", + "nvidia/DeepSeek-R1-0528-FP4-V2/q_a_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/q_a_proj", + "nvidia/Kimi-K2.5-NVFP4/q_a_proj", + "nvidia/Kimi-K2.6-NVFP4/q_a_proj" + ] }, { "type": "gemm", "args": { - "m": 163840, + "m": 136, "n": 2304, "k": 7168, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/gate_up_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/gate_up_proj", + "nvidia/Kimi-K2.5-NVFP4/gate_up_proj" + ] }, { "type": "gemm", "args": { - "m": 163840, + "m": 136, "n": 7168, "k": 2304, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/down_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/down_proj", + "nvidia/Kimi-K2.5-NVFP4/down_proj" + ] }, { "type": "gemm", "args": { - "m": 320, - "n": 1536, - "k": 7168, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 136, + "n": 7168, + "k": 16384, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/o_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/o_proj" + ] }, { "type": "gemm", "args": { - "m": 320, - "n": 3072, + "m": 136, + "n": 24576, "k": 1536, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/q_b_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/q_b_proj" + ] }, { "type": "gemm", "args": { - "m": 320, - "n": 576, - "k": 7168, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 136, + "n": 32768, + "k": 512, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/kv_b_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/kv_b_proj" + ] }, { "type": "gemm", "args": { - "m": 320, - "n": 4096, - "k": 512, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 137, + "n": 576, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/Kimi-K2.5-MXFP4/kv_a_proj_with_mqa", + "moonshotai/Kimi-K2.5/kv_a_proj_with_mqa", + "nvidia/DeepSeek-R1-0528-FP4-V2/kv_a_proj_with_mqa", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/kv_a_proj_with_mqa", + "nvidia/Kimi-K2.5-NVFP4/kv_a_proj_with_mqa", + "nvidia/Kimi-K2.6-NVFP4/kv_a_proj_with_mqa" + ] }, { "type": "gemm", "args": { - "m": 320, - "n": 7168, - "k": 2048, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 137, + "n": 1536, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/Kimi-K2.5-MXFP4/q_a_proj", + "moonshotai/Kimi-K2.5/q_a_proj", + "nvidia/DeepSeek-R1-0528-FP4-V2/q_a_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/q_a_proj", + "nvidia/Kimi-K2.5-NVFP4/q_a_proj", + "nvidia/Kimi-K2.6-NVFP4/q_a_proj" + ] }, { "type": "gemm", "args": { - "m": 320, + "m": 137, "n": 2304, "k": 7168, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/gate_up_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/gate_up_proj", + "nvidia/Kimi-K2.5-NVFP4/gate_up_proj" + ] }, { "type": "gemm", "args": { - "m": 320, + "m": 137, "n": 7168, "k": 2304, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/down_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/down_proj", + "nvidia/Kimi-K2.5-NVFP4/down_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 137, + "n": 7168, + "k": 16384, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/o_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/o_proj" + ] }, { "type": "gemm", "args": { - "m": 786432, + "m": 137, "n": 24576, "k": 1536, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/q_b_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/q_b_proj" + ] }, { "type": "gemm", "args": { - "m": 786432, + "m": 137, "n": 32768, "k": 512, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/kv_b_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/kv_b_proj" + ] }, { "type": "gemm", "args": { - "m": 786432, - "n": 7168, - "k": 16384, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 140, + "n": 576, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/Kimi-K2.5-MXFP4/kv_a_proj_with_mqa", + "moonshotai/Kimi-K2.5/kv_a_proj_with_mqa", + "nvidia/DeepSeek-R1-0528-FP4-V2/kv_a_proj_with_mqa", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/kv_a_proj_with_mqa", + "nvidia/Kimi-K2.5-NVFP4/kv_a_proj_with_mqa", + "nvidia/Kimi-K2.6-NVFP4/kv_a_proj_with_mqa" + ] }, { "type": "gemm", "args": { - "m": 786432, - "n": 4608, + "m": 140, + "n": 576, "k": 7168, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/kv_a_proj_with_mqa" + ] }, { "type": "gemm", "args": { - "m": 786432, - "n": 7168, - "k": 4608, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 140, + "n": 1152, + "k": 7168, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-NVFP4-v2/gate_up_proj" + ] }, { "type": "gemm", "args": { - "m": 1536, + "m": 140, "n": 1536, "k": 7168, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/Kimi-K2.5-MXFP4/q_a_proj", + "moonshotai/Kimi-K2.5/q_a_proj", + "nvidia/DeepSeek-R1-0528-FP4-V2/q_a_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/q_a_proj", + "nvidia/Kimi-K2.5-NVFP4/q_a_proj", + "nvidia/Kimi-K2.6-NVFP4/q_a_proj" + ] }, { "type": "gemm", "args": { - "m": 1536, - "n": 24576, - "k": 1536, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 140, + "n": 1536, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/gate_up_proj", + "deepseek-ai/DeepSeek-R1-0528/q_a_proj", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w3" + ] }, { "type": "gemm", "args": { - "m": 1536, - "n": 576, + "m": 140, + "n": 2304, "k": 7168, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/gate_up_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/gate_up_proj", + "nvidia/Kimi-K2.5-NVFP4/gate_up_proj" + ] }, { "type": "gemm", "args": { - "m": 1536, - "n": 32768, - "k": 512, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 140, + "n": 2304, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/gate_up_proj" + ] }, { "type": "gemm", "args": { - "m": 1536, + "m": 140, "n": 7168, - "k": 16384, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "k": 576, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-NVFP4-v2/down_proj" + ] }, { "type": "gemm", "args": { - "m": 1536, - "n": 4608, - "k": 7168, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 140, + "n": 7168, + "k": 1152, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/down_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/down_proj", + "nvidia/Kimi-K2.5-NVFP4/down_proj" + ] }, { "type": "gemm", "args": { - "m": 1536, + "m": 140, "n": 7168, - "k": 4608, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "k": 1152, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/down_proj" + ] }, { "type": "gemm", "args": { - "m": 393216, - "n": 24576, - "k": 1536, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 140, + "n": 7168, + "k": 2304, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/down_proj" + ] }, { "type": "gemm", "args": { - "m": 393216, - "n": 32768, - "k": 512, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 140, + "n": 7168, + "k": 8192, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/Kimi-K2.5-MXFP4/o_proj", + "moonshotai/Kimi-K2.5/o_proj", + "nvidia/Kimi-K2.5-NVFP4/o_proj", + "nvidia/Kimi-K2.6-NVFP4/o_proj" + ] }, { "type": "gemm", "args": { - "m": 393216, + "m": 140, "n": 7168, "k": 16384, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/o_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/o_proj" + ] }, { "type": "gemm", "args": { - "m": 768, - "n": 1536, - "k": 7168, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 140, + "n": 7168, + "k": 16384, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/o_proj", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] }, { "type": "gemm", "args": { - "m": 768, + "m": 140, + "n": 12288, + "k": 1536, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/Kimi-K2.5-MXFP4/q_b_proj", + "moonshotai/Kimi-K2.5/q_b_proj", + "nvidia/DeepSeek-R1-0528-FP4-V2/q_b_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/q_b_proj", + "nvidia/Kimi-K2.5-NVFP4/q_b_proj", + "nvidia/Kimi-K2.6-NVFP4/q_b_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 140, + "n": 16384, + "k": 512, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/Kimi-K2.5-MXFP4/kv_b_proj", + "moonshotai/Kimi-K2.5/kv_b_proj", + "nvidia/DeepSeek-R1-0528-FP4-V2/kv_b_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/kv_b_proj", + "nvidia/Kimi-K2.5-NVFP4/kv_b_proj", + "nvidia/Kimi-K2.6-NVFP4/kv_b_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 140, "n": 24576, "k": 1536, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/q_b_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/q_b_proj" + ] }, { "type": "gemm", "args": { - "m": 768, - "n": 576, - "k": 7168, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 140, + "n": 24576, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/q_b_proj" + ] }, { "type": "gemm", "args": { - "m": 768, + "m": 140, "n": 32768, "k": 512, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/kv_b_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/kv_b_proj" + ] }, { "type": "gemm", "args": { - "m": 768, - "n": 7168, - "k": 16384, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 140, + "n": 32768, + "k": 512, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/kv_b_proj" + ] }, { "type": "gemm", "args": { - "m": 768, - "n": 2304, + "m": 147, + "n": 576, "k": 7168, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/Kimi-K2.5-MXFP4/kv_a_proj_with_mqa", + "moonshotai/Kimi-K2.5/kv_a_proj_with_mqa", + "nvidia/DeepSeek-R1-0528-FP4-V2/kv_a_proj_with_mqa", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/kv_a_proj_with_mqa", + "nvidia/Kimi-K2.5-NVFP4/kv_a_proj_with_mqa", + "nvidia/Kimi-K2.6-NVFP4/kv_a_proj_with_mqa" + ] }, { "type": "gemm", "args": { - "m": 768, - "n": 7168, - "k": 2304, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 147, + "n": 1536, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/Kimi-K2.5-MXFP4/q_a_proj", + "moonshotai/Kimi-K2.5/q_a_proj", + "nvidia/DeepSeek-R1-0528-FP4-V2/q_a_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/q_a_proj", + "nvidia/Kimi-K2.5-NVFP4/q_a_proj", + "nvidia/Kimi-K2.6-NVFP4/q_a_proj" + ] }, { "type": "gemm", "args": { - "m": 655360, - "n": 4608, + "m": 147, + "n": 2304, "k": 7168, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/gate_up_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/gate_up_proj", + "nvidia/Kimi-K2.5-NVFP4/gate_up_proj" + ] }, { "type": "gemm", "args": { - "m": 655360, + "m": 147, "n": 7168, - "k": 4608, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "k": 2304, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/down_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/down_proj", + "nvidia/Kimi-K2.5-NVFP4/down_proj" + ] }, { "type": "gemm", "args": { - "m": 1280, - "n": 1536, - "k": 7168, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 147, + "n": 7168, + "k": 16384, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/o_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/o_proj" + ] }, { "type": "gemm", "args": { - "m": 1280, + "m": 147, "n": 24576, "k": 1536, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/q_b_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/q_b_proj" + ] }, { "type": "gemm", "args": { - "m": 1280, - "n": 576, - "k": 7168, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 147, + "n": 32768, + "k": 512, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/kv_b_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/kv_b_proj" + ] }, { "type": "gemm", "args": { - "m": 1280, - "n": 32768, - "k": 512, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 153, + "n": 576, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/Kimi-K2.5-MXFP4/kv_a_proj_with_mqa", + "moonshotai/Kimi-K2.5/kv_a_proj_with_mqa", + "nvidia/DeepSeek-R1-0528-FP4-V2/kv_a_proj_with_mqa", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/kv_a_proj_with_mqa", + "nvidia/Kimi-K2.5-NVFP4/kv_a_proj_with_mqa", + "nvidia/Kimi-K2.6-NVFP4/kv_a_proj_with_mqa" + ] }, { "type": "gemm", "args": { - "m": 1280, - "n": 7168, - "k": 16384, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 153, + "n": 576, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/kv_a_proj_with_mqa" + ] }, { "type": "gemm", "args": { - "m": 1280, - "n": 4608, + "m": 153, + "n": 1152, "k": 7168, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-NVFP4-v2/gate_up_proj" + ] }, { "type": "gemm", "args": { - "m": 1280, - "n": 7168, - "k": 4608, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 153, + "n": 1152, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/gate_up_proj" + ] }, { "type": "gemm", "args": { - "m": 327680, + "m": 153, "n": 1536, "k": 7168, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/Kimi-K2.5-MXFP4/q_a_proj", + "moonshotai/Kimi-K2.5/q_a_proj", + "nvidia/DeepSeek-R1-0528-FP4-V2/q_a_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/q_a_proj", + "nvidia/Kimi-K2.5-NVFP4/q_a_proj", + "nvidia/Kimi-K2.6-NVFP4/q_a_proj" + ] }, { "type": "gemm", "args": { - "m": 327680, - "n": 24576, - "k": 1536, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 153, + "n": 1536, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/gate_up_proj", + "deepseek-ai/DeepSeek-R1-0528/q_a_proj", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w3" + ] }, { "type": "gemm", "args": { - "m": 327680, - "n": 576, + "m": 153, + "n": 2304, "k": 7168, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/gate_up_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/gate_up_proj", + "nvidia/Kimi-K2.5-NVFP4/gate_up_proj" + ] }, { "type": "gemm", "args": { - "m": 327680, - "n": 32768, - "k": 512, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 153, + "n": 2304, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/gate_up_proj" + ] }, { "type": "gemm", "args": { - "m": 327680, - "n": 7168, - "k": 16384, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 153, + "n": 4608, + "k": 7168, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/gate_up_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/gate_up_proj", + "nvidia/Kimi-K2.5-NVFP4/gate_up_proj" + ] }, { "type": "gemm", "args": { - "m": 327680, - "n": 2304, + "m": 153, + "n": 4608, "k": 7168, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 32 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 1, + 32 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "amd/DeepSeek-R1-0528-MXFP4-Preview/gate_up_proj", + "amd/DeepSeek-R1-0528-MXFP4-v2/gate_up_proj", + "amd/DeepSeek-R1-0528-MXFP4/gate_up_proj", + "amd/Kimi-K2.5-MXFP4/gate_up_proj" + ] }, { "type": "gemm", "args": { - "m": 327680, + "m": 153, "n": 7168, - "k": 2304, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "k": 1152, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/down_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/down_proj", + "nvidia/Kimi-K2.5-NVFP4/down_proj" + ] }, { "type": "gemm", "args": { - "m": 640, - "n": 1536, - "k": 7168, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 153, + "n": 7168, + "k": 1152, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/down_proj" + ] }, { "type": "gemm", "args": { - "m": 640, - "n": 24576, - "k": 1536, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 153, + "n": 7168, + "k": 2304, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/down_proj" + ] }, { "type": "gemm", "args": { - "m": 640, - "n": 576, - "k": 7168, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 153, + "n": 7168, + "k": 4608, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/down_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/down_proj", + "nvidia/Kimi-K2.5-NVFP4/down_proj" + ] }, { "type": "gemm", "args": { - "m": 640, - "n": 32768, - "k": 512, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 153, + "n": 7168, + "k": 4608, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 32 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 1, + 32 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "amd/DeepSeek-R1-0528-MXFP4-Preview/down_proj", + "amd/DeepSeek-R1-0528-MXFP4-v2/down_proj", + "amd/DeepSeek-R1-0528-MXFP4/down_proj", + "amd/Kimi-K2.5-MXFP4/down_proj" + ] }, { "type": "gemm", "args": { - "m": 640, + "m": 153, "n": 7168, - "k": 16384, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "k": 8192, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/Kimi-K2.5-MXFP4/o_proj", + "moonshotai/Kimi-K2.5/o_proj", + "nvidia/Kimi-K2.5-NVFP4/o_proj", + "nvidia/Kimi-K2.6-NVFP4/o_proj" + ] }, { "type": "gemm", "args": { - "m": 640, - "n": 2304, - "k": 7168, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 153, + "n": 7168, + "k": 8192, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] }, { "type": "gemm", "args": { - "m": 640, + "m": 153, "n": 7168, - "k": 2304, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "k": 16384, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/o_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/o_proj" + ] }, { "type": "gemm", "args": { - "m": 184320, - "n": 1536, - "k": 7168, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 153, + "n": 7168, + "k": 16384, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/o_proj", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] }, { "type": "gemm", "args": { - "m": 184320, - "n": 24576, + "m": 153, + "n": 12288, "k": 1536, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/Kimi-K2.5-MXFP4/q_b_proj", + "moonshotai/Kimi-K2.5/q_b_proj", + "nvidia/DeepSeek-R1-0528-FP4-V2/q_b_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/q_b_proj", + "nvidia/Kimi-K2.5-NVFP4/q_b_proj", + "nvidia/Kimi-K2.6-NVFP4/q_b_proj" + ] }, { "type": "gemm", "args": { - "m": 184320, - "n": 576, - "k": 7168, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 153, + "n": 16384, + "k": 512, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/Kimi-K2.5-MXFP4/kv_b_proj", + "moonshotai/Kimi-K2.5/kv_b_proj", + "nvidia/DeepSeek-R1-0528-FP4-V2/kv_b_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/kv_b_proj", + "nvidia/Kimi-K2.5-NVFP4/kv_b_proj", + "nvidia/Kimi-K2.6-NVFP4/kv_b_proj" + ] }, { "type": "gemm", "args": { - "m": 184320, - "n": 32768, - "k": 512, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 153, + "n": 24576, + "k": 1536, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/q_b_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/q_b_proj" + ] }, { "type": "gemm", "args": { - "m": 184320, - "n": 7168, - "k": 16384, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 153, + "n": 24576, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/q_b_proj" + ] }, { "type": "gemm", "args": { - "m": 184320, - "n": 4608, - "k": 7168, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 153, + "n": 32768, + "k": 512, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/kv_b_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/kv_b_proj" + ] }, { "type": "gemm", "args": { - "m": 184320, - "n": 7168, - "k": 4608, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 153, + "n": 32768, + "k": 512, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/kv_b_proj" + ] }, { "type": "gemm", "args": { - "m": 360, - "n": 1536, + "m": 154, + "n": 576, "k": 7168, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/kv_a_proj_with_mqa" + ] }, { "type": "gemm", "args": { - "m": 360, - "n": 24576, + "m": 154, + "n": 1152, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/gate_up_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 154, + "n": 1536, "k": 1536, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/q_b_proj" + ] }, { "type": "gemm", "args": { - "m": 360, - "n": 576, + "m": 154, + "n": 1536, "k": 7168, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/gate_up_proj", + "deepseek-ai/DeepSeek-R1-0528/q_a_proj", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w3" + ] }, { "type": "gemm", "args": { - "m": 360, - "n": 32768, + "m": 154, + "n": 2048, "k": 512, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/kv_b_proj" + ] }, { "type": "gemm", "args": { - "m": 360, + "m": 154, "n": 7168, - "k": 16384, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "k": 1024, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/o_proj", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] }, { "type": "gemm", "args": { - "m": 360, - "n": 4608, - "k": 7168, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 154, + "n": 7168, + "k": 1152, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/down_proj" + ] }, { "type": "gemm", "args": { - "m": 360, - "n": 7168, - "k": 4608, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 156, + "n": 576, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/Kimi-K2.5-MXFP4/kv_a_proj_with_mqa", + "moonshotai/Kimi-K2.5/kv_a_proj_with_mqa", + "nvidia/DeepSeek-R1-0528-FP4-V2/kv_a_proj_with_mqa", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/kv_a_proj_with_mqa", + "nvidia/Kimi-K2.5-NVFP4/kv_a_proj_with_mqa", + "nvidia/Kimi-K2.6-NVFP4/kv_a_proj_with_mqa" + ] }, { "type": "gemm", "args": { - "m": 92160, + "m": 156, "n": 1536, "k": 7168, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/Kimi-K2.5-MXFP4/q_a_proj", + "moonshotai/Kimi-K2.5/q_a_proj", + "nvidia/DeepSeek-R1-0528-FP4-V2/q_a_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/q_a_proj", + "nvidia/Kimi-K2.5-NVFP4/q_a_proj", + "nvidia/Kimi-K2.6-NVFP4/q_a_proj" + ] }, { "type": "gemm", "args": { - "m": 92160, - "n": 24576, + "m": 156, + "n": 3072, "k": 1536, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 92160, - "n": 576, - "k": 7168, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/Kimi-K2.5-MXFP4/q_b_proj", + "amd/MiniMax-M2.5-MXFP4/o_proj", + "moonshotai/Kimi-K2.5/q_b_proj", + "nvidia/DeepSeek-R1-0528-FP4-V2/q_b_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/q_b_proj", + "nvidia/Kimi-K2.5-NVFP4/q_b_proj", + "nvidia/Kimi-K2.6-NVFP4/q_b_proj", + "nvidia/MiniMax-M2.5-NVFP4/o_proj" + ] }, { "type": "gemm", "args": { - "m": 92160, - "n": 32768, + "m": 156, + "n": 4096, "k": 512, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "Qwen/Qwen3.5-397B-A17B/o_proj", + "Qwen/Qwen3.5-397B-A17B/shared_expert.down_proj", + "amd/Kimi-K2.5-MXFP4/kv_b_proj", + "amd/Qwen3.5-397B-A17B-MXFP4/o_proj", + "amd/Qwen3.5-397B-A17B-MXFP4/shared_expert.down_proj", + "moonshotai/Kimi-K2.5/kv_b_proj", + "nvidia/DeepSeek-R1-0528-FP4-V2/kv_b_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/kv_b_proj", + "nvidia/Kimi-K2.5-NVFP4/kv_b_proj", + "nvidia/Kimi-K2.6-NVFP4/kv_b_proj", + "nvidia/Qwen3.5-397B-A17B-NVFP4/o_proj", + "nvidia/Qwen3.5-397B-A17B-NVFP4/shared_expert.down_proj" + ] }, { "type": "gemm", "args": { - "m": 92160, - "n": 7168, - "k": 16384, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 156, + "n": 4608, + "k": 7168, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/gate_up_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/gate_up_proj", + "nvidia/Kimi-K2.5-NVFP4/gate_up_proj" + ] }, { "type": "gemm", "args": { - "m": 92160, - "n": 2304, + "m": 156, + "n": 4608, "k": 7168, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 32 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 1, + 32 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "amd/DeepSeek-R1-0528-MXFP4-Preview/gate_up_proj", + "amd/DeepSeek-R1-0528-MXFP4-v2/gate_up_proj", + "amd/DeepSeek-R1-0528-MXFP4/gate_up_proj", + "amd/Kimi-K2.5-MXFP4/gate_up_proj" + ] }, { "type": "gemm", "args": { - "m": 92160, + "m": 156, "n": 7168, - "k": 2304, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 180, - "n": 1536, - "k": 7168, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "k": 2048, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/Kimi-K2.5-MXFP4/o_proj", + "moonshotai/Kimi-K2.5/o_proj", + "nvidia/Kimi-K2.5-NVFP4/o_proj", + "nvidia/Kimi-K2.6-NVFP4/o_proj" + ] }, { "type": "gemm", "args": { - "m": 180, - "n": 24576, - "k": 1536, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 156, + "n": 7168, + "k": 4608, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/down_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/down_proj", + "nvidia/Kimi-K2.5-NVFP4/down_proj" + ] }, { "type": "gemm", "args": { - "m": 180, + "m": 160, "n": 576, "k": 7168, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 180, - "n": 32768, - "k": 512, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/kv_a_proj_with_mqa" + ] }, { "type": "gemm", "args": { - "m": 180, - "n": 7168, - "k": 16384, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 160, + "n": 1536, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/gate_up_proj", + "deepseek-ai/DeepSeek-R1-0528/q_a_proj", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w3" + ] }, { "type": "gemm", "args": { - "m": 180, + "m": 160, "n": 2304, "k": 7168, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/gate_up_proj" + ] }, { "type": "gemm", "args": { - "m": 180, + "m": 160, "n": 7168, "k": 2304, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/down_proj" + ] }, { "type": "gemm", "args": { - "m": 2883584, - "n": 1536, - "k": 7168, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 160, + "n": 7168, + "k": 16384, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/o_proj", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] }, { "type": "gemm", "args": { - "m": 2883584, + "m": 160, "n": 24576, "k": 1536, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/q_b_proj" + ] }, { "type": "gemm", "args": { - "m": 2883584, - "n": 576, - "k": 7168, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 160, + "n": 32768, + "k": 512, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/kv_b_proj" + ] }, { "type": "gemm", "args": { - "m": 2883584, - "n": 32768, - "k": 512, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 163, + "n": 576, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/Kimi-K2.5-MXFP4/kv_a_proj_with_mqa", + "moonshotai/Kimi-K2.5/kv_a_proj_with_mqa", + "nvidia/DeepSeek-R1-0528-FP4-V2/kv_a_proj_with_mqa", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/kv_a_proj_with_mqa", + "nvidia/Kimi-K2.5-NVFP4/kv_a_proj_with_mqa", + "nvidia/Kimi-K2.6-NVFP4/kv_a_proj_with_mqa" + ] }, { "type": "gemm", "args": { - "m": 2883584, - "n": 7168, - "k": 16384, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 163, + "n": 1536, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/Kimi-K2.5-MXFP4/q_a_proj", + "moonshotai/Kimi-K2.5/q_a_proj", + "nvidia/DeepSeek-R1-0528-FP4-V2/q_a_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/q_a_proj", + "nvidia/Kimi-K2.5-NVFP4/q_a_proj", + "nvidia/Kimi-K2.6-NVFP4/q_a_proj" + ] }, { "type": "gemm", "args": { - "m": 2883584, - "n": 4608, + "m": 163, + "n": 2304, "k": 7168, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/gate_up_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/gate_up_proj", + "nvidia/Kimi-K2.5-NVFP4/gate_up_proj" + ] }, { "type": "gemm", "args": { - "m": 2883584, + "m": 163, "n": 7168, - "k": 4608, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "k": 2304, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/down_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/down_proj", + "nvidia/Kimi-K2.5-NVFP4/down_proj" + ] }, { "type": "gemm", "args": { - "m": 5632, - "n": 1536, - "k": 7168, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 163, + "n": 7168, + "k": 16384, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/o_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/o_proj" + ] }, { "type": "gemm", "args": { - "m": 5632, + "m": 163, "n": 24576, "k": 1536, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/q_b_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/q_b_proj" + ] }, { "type": "gemm", "args": { - "m": 5632, - "n": 576, - "k": 7168, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 163, + "n": 32768, + "k": 512, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/kv_b_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/kv_b_proj" + ] }, { "type": "gemm", "args": { - "m": 5632, - "n": 32768, - "k": 512, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 164, + "n": 576, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/Kimi-K2.5-MXFP4/kv_a_proj_with_mqa", + "moonshotai/Kimi-K2.5/kv_a_proj_with_mqa", + "nvidia/DeepSeek-R1-0528-FP4-V2/kv_a_proj_with_mqa", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/kv_a_proj_with_mqa", + "nvidia/Kimi-K2.5-NVFP4/kv_a_proj_with_mqa", + "nvidia/Kimi-K2.6-NVFP4/kv_a_proj_with_mqa" + ] }, { "type": "gemm", "args": { - "m": 5632, - "n": 7168, - "k": 16384, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 164, + "n": 1536, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/Kimi-K2.5-MXFP4/q_a_proj", + "moonshotai/Kimi-K2.5/q_a_proj", + "nvidia/DeepSeek-R1-0528-FP4-V2/q_a_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/q_a_proj", + "nvidia/Kimi-K2.5-NVFP4/q_a_proj", + "nvidia/Kimi-K2.6-NVFP4/q_a_proj" + ] }, { "type": "gemm", "args": { - "m": 5632, - "n": 4608, + "m": 164, + "n": 2304, "k": 7168, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/gate_up_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/gate_up_proj", + "nvidia/Kimi-K2.5-NVFP4/gate_up_proj" + ] }, { "type": "gemm", "args": { - "m": 5632, + "m": 164, "n": 7168, - "k": 4608, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "k": 2304, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/down_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/down_proj", + "nvidia/Kimi-K2.5-NVFP4/down_proj" + ] }, { "type": "gemm", "args": { - "m": 1441792, - "n": 1536, - "k": 7168, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 164, + "n": 7168, + "k": 16384, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/o_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/o_proj" + ] }, { "type": "gemm", "args": { - "m": 1441792, + "m": 164, "n": 24576, "k": 1536, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 1441792, - "n": 576, - "k": 7168, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/q_b_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/q_b_proj" + ] }, { "type": "gemm", "args": { - "m": 1441792, + "m": 164, "n": 32768, "k": 512, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/kv_b_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/kv_b_proj" + ] }, { "type": "gemm", "args": { - "m": 1441792, - "n": 7168, - "k": 16384, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 166, + "n": 576, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/Kimi-K2.5-MXFP4/kv_a_proj_with_mqa", + "moonshotai/Kimi-K2.5/kv_a_proj_with_mqa", + "nvidia/DeepSeek-R1-0528-FP4-V2/kv_a_proj_with_mqa", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/kv_a_proj_with_mqa", + "nvidia/Kimi-K2.5-NVFP4/kv_a_proj_with_mqa", + "nvidia/Kimi-K2.6-NVFP4/kv_a_proj_with_mqa" + ] }, { "type": "gemm", "args": { - "m": 1441792, - "n": 2304, + "m": 166, + "n": 576, "k": 7168, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 32 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 1, + 32 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "amd/DeepSeek-R1-0528-MXFP4-Preview/kv_a_proj_with_mqa", + "amd/DeepSeek-R1-0528-MXFP4-v2/kv_a_proj_with_mqa", + "amd/DeepSeek-R1-0528-MXFP4/kv_a_proj_with_mqa" + ] }, { "type": "gemm", "args": { - "m": 1441792, - "n": 7168, - "k": 2304, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 166, + "n": 576, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/kv_a_proj_with_mqa" + ] }, { "type": "gemm", "args": { - "m": 2816, + "m": 166, "n": 1536, "k": 7168, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 2816, - "n": 24576, - "k": 1536, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/Kimi-K2.5-MXFP4/q_a_proj", + "moonshotai/Kimi-K2.5/q_a_proj", + "nvidia/DeepSeek-R1-0528-FP4-V2/q_a_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/q_a_proj", + "nvidia/Kimi-K2.5-NVFP4/q_a_proj", + "nvidia/Kimi-K2.6-NVFP4/q_a_proj" + ] }, { "type": "gemm", "args": { - "m": 2816, - "n": 576, + "m": 166, + "n": 1536, "k": 7168, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 32 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 1, + 32 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "amd/DeepSeek-R1-0528-MXFP4-Preview/q_a_proj", + "amd/DeepSeek-R1-0528-MXFP4-v2/q_a_proj", + "amd/DeepSeek-R1-0528-MXFP4/q_a_proj" + ] }, { "type": "gemm", "args": { - "m": 2816, - "n": 32768, - "k": 512, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 166, + "n": 1536, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/gate_up_proj", + "deepseek-ai/DeepSeek-R1-0528/q_a_proj", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w3" + ] }, { "type": "gemm", "args": { - "m": 2816, - "n": 7168, - "k": 16384, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 166, + "n": 1536, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w3" + ] }, { "type": "gemm", "args": { - "m": 2816, + "m": 166, "n": 2304, "k": 7168, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/gate_up_proj" + ] }, { "type": "gemm", "args": { - "m": 2816, - "n": 7168, - "k": 2304, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 166, + "n": 4608, + "k": 7168, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/gate_up_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/gate_up_proj", + "nvidia/Kimi-K2.5-NVFP4/gate_up_proj" + ] }, { "type": "gemm", "args": { - "m": 540672, - "n": 1536, + "m": 166, + "n": 4608, "k": 7168, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 32 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 1, + 32 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "amd/DeepSeek-R1-0528-MXFP4-Preview/gate_up_proj", + "amd/DeepSeek-R1-0528-MXFP4-v2/gate_up_proj", + "amd/DeepSeek-R1-0528-MXFP4/gate_up_proj", + "amd/Kimi-K2.5-MXFP4/gate_up_proj" + ] }, { "type": "gemm", "args": { - "m": 540672, - "n": 24576, - "k": 1536, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 166, + "n": 4608, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/gate_up_proj" + ] }, { "type": "gemm", "args": { - "m": 540672, - "n": 576, - "k": 7168, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 166, + "n": 7168, + "k": 2304, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/down_proj" + ] }, { "type": "gemm", "args": { - "m": 540672, - "n": 32768, - "k": 512, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 166, + "n": 7168, + "k": 4608, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/down_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/down_proj", + "nvidia/Kimi-K2.5-NVFP4/down_proj" + ] }, { "type": "gemm", "args": { - "m": 540672, + "m": 166, "n": 7168, - "k": 16384, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "k": 4608, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 32 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 1, + 32 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "amd/DeepSeek-R1-0528-MXFP4-Preview/down_proj", + "amd/DeepSeek-R1-0528-MXFP4-v2/down_proj", + "amd/DeepSeek-R1-0528-MXFP4/down_proj", + "amd/Kimi-K2.5-MXFP4/down_proj" + ] }, { "type": "gemm", "args": { - "m": 540672, - "n": 4608, - "k": 7168, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 166, + "n": 7168, + "k": 4608, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/down_proj" + ] }, { "type": "gemm", "args": { - "m": 540672, + "m": 166, "n": 7168, - "k": 4608, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "k": 8192, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/Kimi-K2.5-MXFP4/o_proj", + "moonshotai/Kimi-K2.5/o_proj", + "nvidia/Kimi-K2.5-NVFP4/o_proj", + "nvidia/Kimi-K2.6-NVFP4/o_proj" + ] }, { "type": "gemm", "args": { - "m": 528, - "n": 1536, - "k": 7168, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 166, + "n": 7168, + "k": 8192, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] }, { "type": "gemm", "args": { - "m": 528, - "n": 24576, - "k": 1536, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 166, + "n": 7168, + "k": 16384, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/o_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/o_proj" + ] }, { "type": "gemm", "args": { - "m": 528, - "n": 576, - "k": 7168, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 166, + "n": 7168, + "k": 16384, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 32 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 1, + 32 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "amd/DeepSeek-R1-0528-MXFP4-Preview/o_proj", + "amd/DeepSeek-R1-0528-MXFP4-v2/o_proj", + "amd/DeepSeek-R1-0528-MXFP4/o_proj" + ] }, { "type": "gemm", "args": { - "m": 528, - "n": 32768, - "k": 512, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 166, + "n": 7168, + "k": 16384, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/o_proj", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] }, { "type": "gemm", "args": { - "m": 528, + "m": 166, "n": 7168, "k": 16384, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] }, { "type": "gemm", "args": { - "m": 528, - "n": 4608, - "k": 7168, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 166, + "n": 12288, + "k": 1536, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/Kimi-K2.5-MXFP4/q_b_proj", + "moonshotai/Kimi-K2.5/q_b_proj", + "nvidia/DeepSeek-R1-0528-FP4-V2/q_b_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/q_b_proj", + "nvidia/Kimi-K2.5-NVFP4/q_b_proj", + "nvidia/Kimi-K2.6-NVFP4/q_b_proj" + ] }, { "type": "gemm", "args": { - "m": 528, - "n": 7168, - "k": 4608, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 166, + "n": 16384, + "k": 512, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/Kimi-K2.5-MXFP4/kv_b_proj", + "moonshotai/Kimi-K2.5/kv_b_proj", + "nvidia/DeepSeek-R1-0528-FP4-V2/kv_b_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/kv_b_proj", + "nvidia/Kimi-K2.5-NVFP4/kv_b_proj", + "nvidia/Kimi-K2.6-NVFP4/kv_b_proj" + ] }, { "type": "gemm", "args": { - "m": 270336, - "n": 1536, - "k": 7168, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 166, + "n": 24576, + "k": 1536, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/q_b_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/q_b_proj" + ] }, { "type": "gemm", "args": { - "m": 270336, + "m": 166, "n": 24576, "k": 1536, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 32 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 1, + 32 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "amd/DeepSeek-R1-0528-MXFP4-Preview/q_b_proj", + "amd/DeepSeek-R1-0528-MXFP4-v2/q_b_proj", + "amd/DeepSeek-R1-0528-MXFP4/q_b_proj" + ] }, { "type": "gemm", "args": { - "m": 270336, - "n": 576, - "k": 7168, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 166, + "n": 24576, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/q_b_proj" + ] }, { "type": "gemm", "args": { - "m": 270336, + "m": 166, "n": 32768, "k": 512, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/kv_b_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/kv_b_proj" + ] }, { "type": "gemm", "args": { - "m": 270336, - "n": 7168, - "k": 16384, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 166, + "n": 32768, + "k": 512, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 32 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 1, + 32 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "amd/DeepSeek-R1-0528-MXFP4-Preview/kv_b_proj", + "amd/DeepSeek-R1-0528-MXFP4-v2/kv_b_proj", + "amd/DeepSeek-R1-0528-MXFP4/kv_b_proj" + ] }, { "type": "gemm", "args": { - "m": 270336, - "n": 2304, - "k": 7168, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 166, + "n": 32768, + "k": 512, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/kv_b_proj" + ] }, { "type": "gemm", "args": { - "m": 270336, - "n": 7168, - "k": 2304, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 167, + "n": 576, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/Kimi-K2.5-MXFP4/kv_a_proj_with_mqa", + "moonshotai/Kimi-K2.5/kv_a_proj_with_mqa", + "nvidia/DeepSeek-R1-0528-FP4-V2/kv_a_proj_with_mqa", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/kv_a_proj_with_mqa", + "nvidia/Kimi-K2.5-NVFP4/kv_a_proj_with_mqa", + "nvidia/Kimi-K2.6-NVFP4/kv_a_proj_with_mqa" + ] }, { "type": "gemm", "args": { - "m": 264, + "m": 167, "n": 1536, "k": 7168, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 264, - "n": 24576, - "k": 1536, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/Kimi-K2.5-MXFP4/q_a_proj", + "moonshotai/Kimi-K2.5/q_a_proj", + "nvidia/DeepSeek-R1-0528-FP4-V2/q_a_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/q_a_proj", + "nvidia/Kimi-K2.5-NVFP4/q_a_proj", + "nvidia/Kimi-K2.6-NVFP4/q_a_proj" + ] }, { "type": "gemm", "args": { - "m": 264, - "n": 576, + "m": 167, + "n": 4608, "k": 7168, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/gate_up_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/gate_up_proj", + "nvidia/Kimi-K2.5-NVFP4/gate_up_proj" + ] }, { "type": "gemm", "args": { - "m": 264, - "n": 32768, - "k": 512, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 167, + "n": 7168, + "k": 4608, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/down_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/down_proj", + "nvidia/Kimi-K2.5-NVFP4/down_proj" + ] }, { "type": "gemm", "args": { - "m": 264, + "m": 167, "n": 7168, - "k": 16384, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "k": 4608, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 32 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 1, + 32 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "amd/DeepSeek-R1-0528-MXFP4-Preview/down_proj", + "amd/DeepSeek-R1-0528-MXFP4-v2/down_proj", + "amd/DeepSeek-R1-0528-MXFP4/down_proj", + "amd/Kimi-K2.5-MXFP4/down_proj" + ] }, { "type": "gemm", "args": { - "m": 264, - "n": 2304, - "k": 7168, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 167, + "n": 7168, + "k": 8192, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/Kimi-K2.5-MXFP4/o_proj", + "moonshotai/Kimi-K2.5/o_proj", + "nvidia/Kimi-K2.5-NVFP4/o_proj", + "nvidia/Kimi-K2.6-NVFP4/o_proj" + ] }, { "type": "gemm", "args": { - "m": 264, + "m": 167, "n": 7168, - "k": 2304, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "k": 8192, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] }, { "type": "gemm", "args": { - "m": 768, - "n": 4608, - "k": 7168, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 167, + "n": 12288, + "k": 1536, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/Kimi-K2.5-MXFP4/q_b_proj", + "moonshotai/Kimi-K2.5/q_b_proj", + "nvidia/DeepSeek-R1-0528-FP4-V2/q_b_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/q_b_proj", + "nvidia/Kimi-K2.5-NVFP4/q_b_proj", + "nvidia/Kimi-K2.6-NVFP4/q_b_proj" + ] }, { "type": "gemm", "args": { - "m": 768, - "n": 7168, - "k": 4608, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 167, + "n": 16384, + "k": 512, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/Kimi-K2.5-MXFP4/kv_b_proj", + "moonshotai/Kimi-K2.5/kv_b_proj", + "nvidia/DeepSeek-R1-0528-FP4-V2/kv_b_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/kv_b_proj", + "nvidia/Kimi-K2.5-NVFP4/kv_b_proj", + "nvidia/Kimi-K2.6-NVFP4/kv_b_proj" + ] }, { "type": "gemm", "args": { - "m": 384, - "n": 1536, + "m": 170, + "n": 384, "k": 7168, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 384, - "n": 24576, - "k": 1536, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [] }, { "type": "gemm", "args": { - "m": 384, + "m": 170, "n": 576, "k": 7168, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/Kimi-K2.5-MXFP4/kv_a_proj_with_mqa", + "moonshotai/Kimi-K2.5/kv_a_proj_with_mqa", + "nvidia/DeepSeek-R1-0528-FP4-V2/kv_a_proj_with_mqa", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/kv_a_proj_with_mqa", + "nvidia/Kimi-K2.5-NVFP4/kv_a_proj_with_mqa", + "nvidia/Kimi-K2.6-NVFP4/kv_a_proj_with_mqa" + ] }, { "type": "gemm", "args": { - "m": 384, - "n": 32768, - "k": 512, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 170, + "n": 576, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/kv_a_proj_with_mqa" + ] }, { "type": "gemm", "args": { - "m": 384, - "n": 7168, - "k": 16384, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 170, + "n": 768, + "k": 7168, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-NVFP4-v2/gate_up_proj" + ] }, { "type": "gemm", "args": { - "m": 384, - "n": 2304, + "m": 170, + "n": 1536, "k": 7168, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/Kimi-K2.5-MXFP4/q_a_proj", + "moonshotai/Kimi-K2.5/q_a_proj", + "nvidia/DeepSeek-R1-0528-FP4-V2/q_a_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/q_a_proj", + "nvidia/Kimi-K2.5-NVFP4/q_a_proj", + "nvidia/Kimi-K2.6-NVFP4/q_a_proj" + ] }, { "type": "gemm", "args": { - "m": 384, - "n": 7168, - "k": 2304, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 170, + "n": 1536, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/gate_up_proj", + "deepseek-ai/DeepSeek-R1-0528/q_a_proj", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w3" + ] }, { "type": "gemm", "args": { - "m": 327680, - "n": 4608, + "m": 170, + "n": 2304, "k": 7168, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/gate_up_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/gate_up_proj", + "nvidia/Kimi-K2.5-NVFP4/gate_up_proj" + ] }, { "type": "gemm", "args": { - "m": 327680, + "m": 170, "n": 7168, - "k": 4608, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "k": 384, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-NVFP4-v2/down_proj" + ] }, { "type": "gemm", "args": { - "m": 320, - "n": 24576, - "k": 1536, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 170, + "n": 7168, + "k": 768, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/down_proj", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w2" + ] }, { "type": "gemm", "args": { - "m": 320, - "n": 32768, - "k": 512, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 170, + "n": 7168, + "k": 2304, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/down_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/down_proj", + "nvidia/Kimi-K2.5-NVFP4/down_proj" + ] }, { "type": "gemm", "args": { - "m": 320, + "m": 170, "n": 7168, "k": 16384, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/o_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/o_proj" + ] }, { "type": "gemm", "args": { - "m": 320, - "n": 4608, - "k": 7168, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 170, + "n": 7168, + "k": 16384, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/o_proj", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] }, { "type": "gemm", "args": { - "m": 320, - "n": 7168, - "k": 4608, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 170, + "n": 24576, + "k": 1536, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/q_b_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/q_b_proj" + ] }, { "type": "gemm", "args": { - "m": 163840, + "m": 170, "n": 24576, "k": 1536, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/q_b_proj" + ] }, { "type": "gemm", "args": { - "m": 163840, + "m": 170, "n": 32768, "k": 512, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/kv_b_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/kv_b_proj" + ] }, { "type": "gemm", "args": { - "m": 163840, - "n": 7168, - "k": 16384, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 170, + "n": 32768, + "k": 512, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/kv_b_proj" + ] }, { "type": "gemm", "args": { - "m": 160, - "n": 1536, + "m": 180, + "n": 576, "k": 7168, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 160, - "n": 24576, - "k": 1536, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/Kimi-K2.5-MXFP4/kv_a_proj_with_mqa", + "moonshotai/Kimi-K2.5/kv_a_proj_with_mqa", + "nvidia/DeepSeek-R1-0528-FP4-V2/kv_a_proj_with_mqa", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/kv_a_proj_with_mqa", + "nvidia/Kimi-K2.5-NVFP4/kv_a_proj_with_mqa", + "nvidia/Kimi-K2.6-NVFP4/kv_a_proj_with_mqa" + ] }, { "type": "gemm", "args": { - "m": 160, + "m": 180, "n": 576, "k": 7168, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/kv_a_proj_with_mqa" + ] }, { "type": "gemm", "args": { - "m": 160, - "n": 32768, - "k": 512, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 180, + "n": 1536, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/Kimi-K2.5-MXFP4/q_a_proj", + "moonshotai/Kimi-K2.5/q_a_proj", + "nvidia/DeepSeek-R1-0528-FP4-V2/q_a_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/q_a_proj", + "nvidia/Kimi-K2.5-NVFP4/q_a_proj", + "nvidia/Kimi-K2.6-NVFP4/q_a_proj" + ] }, { "type": "gemm", "args": { - "m": 160, - "n": 7168, - "k": 16384, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 180, + "n": 1536, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/gate_up_proj", + "deepseek-ai/DeepSeek-R1-0528/q_a_proj", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w3" + ] }, { "type": "gemm", "args": { - "m": 160, + "m": 180, "n": 2304, "k": 7168, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/gate_up_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/gate_up_proj", + "nvidia/Kimi-K2.5-NVFP4/gate_up_proj" + ] }, { "type": "gemm", "args": { - "m": 160, - "n": 7168, - "k": 2304, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 180, + "n": 2304, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/gate_up_proj" + ] }, { "type": "gemm", "args": { - "m": 3072, - "n": 1536, - "k": 7168, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 180, + "n": 3072, + "k": 1536, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/Kimi-K2.5-MXFP4/q_b_proj", + "amd/MiniMax-M2.5-MXFP4/o_proj", + "moonshotai/Kimi-K2.5/q_b_proj", + "nvidia/DeepSeek-R1-0528-FP4-V2/q_b_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/q_b_proj", + "nvidia/Kimi-K2.5-NVFP4/q_b_proj", + "nvidia/Kimi-K2.6-NVFP4/q_b_proj", + "nvidia/MiniMax-M2.5-NVFP4/o_proj" + ] }, { "type": "gemm", "args": { - "m": 3072, - "n": 24576, - "k": 1536, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 180, + "n": 4096, + "k": 512, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "Qwen/Qwen3.5-397B-A17B/o_proj", + "Qwen/Qwen3.5-397B-A17B/shared_expert.down_proj", + "amd/Kimi-K2.5-MXFP4/kv_b_proj", + "amd/Qwen3.5-397B-A17B-MXFP4/o_proj", + "amd/Qwen3.5-397B-A17B-MXFP4/shared_expert.down_proj", + "moonshotai/Kimi-K2.5/kv_b_proj", + "nvidia/DeepSeek-R1-0528-FP4-V2/kv_b_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/kv_b_proj", + "nvidia/Kimi-K2.5-NVFP4/kv_b_proj", + "nvidia/Kimi-K2.6-NVFP4/kv_b_proj", + "nvidia/Qwen3.5-397B-A17B-NVFP4/o_proj", + "nvidia/Qwen3.5-397B-A17B-NVFP4/shared_expert.down_proj" + ] }, { "type": "gemm", "args": { - "m": 3072, - "n": 576, - "k": 7168, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 180, + "n": 7168, + "k": 2048, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/o_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/o_proj" + ] }, { "type": "gemm", "args": { - "m": 3072, - "n": 32768, - "k": 512, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 180, + "n": 7168, + "k": 2304, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/down_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/down_proj", + "nvidia/Kimi-K2.5-NVFP4/down_proj" + ] }, { "type": "gemm", "args": { - "m": 3072, + "m": 180, "n": 7168, - "k": 16384, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "k": 2304, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/down_proj" + ] }, { "type": "gemm", "args": { - "m": 3072, - "n": 4608, - "k": 7168, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 180, + "n": 7168, + "k": 9216, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/down_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/down_proj", + "nvidia/Kimi-K2.5-NVFP4/down_proj" + ] }, { "type": "gemm", "args": { - "m": 3072, + "m": 180, "n": 7168, - "k": 4608, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "k": 16384, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/o_proj", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] }, { "type": "gemm", "args": { - "m": 3, - "n": 1536, + "m": 180, + "n": 9216, "k": 7168, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/gate_up_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/gate_up_proj", + "nvidia/Kimi-K2.5-NVFP4/gate_up_proj" + ] }, { "type": "gemm", "args": { - "m": 3, - "n": 24576, - "k": 1536, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 180, + "n": 9216, + "k": 7168, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 32 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 1, + 32 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "amd/DeepSeek-R1-0528-MXFP4-Preview/gate_up_proj", + "amd/DeepSeek-R1-0528-MXFP4-v2/gate_up_proj", + "amd/DeepSeek-R1-0528-MXFP4/gate_up_proj", + "amd/Kimi-K2.5-MXFP4/gate_up_proj" + ] }, { "type": "gemm", "args": { - "m": 3, - "n": 576, - "k": 7168, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 180, + "n": 24576, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/q_b_proj" + ] }, { "type": "gemm", "args": { - "m": 3, + "m": 180, "n": 32768, "k": 512, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/kv_b_proj" + ] }, { "type": "gemm", "args": { - "m": 3, - "n": 7168, - "k": 16384, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 181, + "n": 576, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/Kimi-K2.5-MXFP4/kv_a_proj_with_mqa", + "moonshotai/Kimi-K2.5/kv_a_proj_with_mqa", + "nvidia/DeepSeek-R1-0528-FP4-V2/kv_a_proj_with_mqa", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/kv_a_proj_with_mqa", + "nvidia/Kimi-K2.5-NVFP4/kv_a_proj_with_mqa", + "nvidia/Kimi-K2.6-NVFP4/kv_a_proj_with_mqa" + ] }, { "type": "gemm", "args": { - "m": 3, - "n": 4608, + "m": 181, + "n": 576, "k": 7168, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 32 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 1, + 32 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "amd/DeepSeek-R1-0528-MXFP4-Preview/kv_a_proj_with_mqa", + "amd/DeepSeek-R1-0528-MXFP4-v2/kv_a_proj_with_mqa", + "amd/DeepSeek-R1-0528-MXFP4/kv_a_proj_with_mqa" + ] }, { "type": "gemm", "args": { - "m": 3, - "n": 7168, - "k": 4608, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 181, + "n": 1536, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/Kimi-K2.5-MXFP4/q_a_proj", + "moonshotai/Kimi-K2.5/q_a_proj", + "nvidia/DeepSeek-R1-0528-FP4-V2/q_a_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/q_a_proj", + "nvidia/Kimi-K2.5-NVFP4/q_a_proj", + "nvidia/Kimi-K2.6-NVFP4/q_a_proj" + ] }, { "type": "gemm", "args": { - "m": 12288, + "m": 181, "n": 1536, "k": 7168, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 32 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 1, + 32 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "amd/DeepSeek-R1-0528-MXFP4-Preview/q_a_proj", + "amd/DeepSeek-R1-0528-MXFP4-v2/q_a_proj", + "amd/DeepSeek-R1-0528-MXFP4/q_a_proj" + ] }, { "type": "gemm", "args": { - "m": 12288, - "n": 3072, - "k": 1536, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 181, + "n": 1536, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w3" + ] }, { "type": "gemm", "args": { - "m": 12288, - "n": 576, - "k": 7168, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 181, + "n": 7168, + "k": 9216, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/down_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/down_proj", + "nvidia/Kimi-K2.5-NVFP4/down_proj" + ] }, { "type": "gemm", "args": { - "m": 12288, - "n": 4096, - "k": 512, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 181, + "n": 7168, + "k": 16384, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/o_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/o_proj" + ] }, { "type": "gemm", "args": { - "m": 12288, + "m": 181, "n": 7168, - "k": 2048, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "k": 16384, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 32 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 1, + 32 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "amd/DeepSeek-R1-0528-MXFP4-Preview/o_proj", + "amd/DeepSeek-R1-0528-MXFP4-v2/o_proj", + "amd/DeepSeek-R1-0528-MXFP4/o_proj" + ] }, { "type": "gemm", "args": { - "m": 12288, - "n": 2304, - "k": 7168, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 181, + "n": 7168, + "k": 16384, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] }, { "type": "gemm", "args": { - "m": 12288, - "n": 7168, - "k": 2304, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 181, + "n": 9216, + "k": 7168, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/gate_up_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/gate_up_proj", + "nvidia/Kimi-K2.5-NVFP4/gate_up_proj" + ] }, { "type": "gemm", "args": { - "m": 12, - "n": 3072, - "k": 1536, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 181, + "n": 9216, + "k": 7168, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 32 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 1, + 32 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "amd/DeepSeek-R1-0528-MXFP4-Preview/gate_up_proj", + "amd/DeepSeek-R1-0528-MXFP4-v2/gate_up_proj", + "amd/DeepSeek-R1-0528-MXFP4/gate_up_proj", + "amd/Kimi-K2.5-MXFP4/gate_up_proj" + ] }, { "type": "gemm", "args": { - "m": 12, - "n": 4096, - "k": 512, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 181, + "n": 24576, + "k": 1536, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/q_b_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/q_b_proj" + ] }, { "type": "gemm", "args": { - "m": 12, - "n": 7168, - "k": 2048, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 181, + "n": 24576, + "k": 1536, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 32 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 1, + 32 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "amd/DeepSeek-R1-0528-MXFP4-Preview/q_b_proj", + "amd/DeepSeek-R1-0528-MXFP4-v2/q_b_proj", + "amd/DeepSeek-R1-0528-MXFP4/q_b_proj" + ] }, { "type": "gemm", "args": { - "m": 32768, - "n": 4608, - "k": 7168, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 181, + "n": 32768, + "k": 512, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/kv_b_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/kv_b_proj" + ] }, { "type": "gemm", "args": { - "m": 32768, - "n": 7168, - "k": 4608, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 181, + "n": 32768, + "k": 512, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 32 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 1, + 32 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "amd/DeepSeek-R1-0528-MXFP4-Preview/kv_b_proj", + "amd/DeepSeek-R1-0528-MXFP4-v2/kv_b_proj", + "amd/DeepSeek-R1-0528-MXFP4/kv_b_proj" + ] }, { "type": "gemm", "args": { - "m": 32, - "n": 4608, + "m": 192, + "n": 576, "k": 7168, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/Kimi-K2.5-MXFP4/kv_a_proj_with_mqa", + "moonshotai/Kimi-K2.5/kv_a_proj_with_mqa", + "nvidia/DeepSeek-R1-0528-FP4-V2/kv_a_proj_with_mqa", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/kv_a_proj_with_mqa", + "nvidia/Kimi-K2.5-NVFP4/kv_a_proj_with_mqa", + "nvidia/Kimi-K2.6-NVFP4/kv_a_proj_with_mqa" + ] }, { "type": "gemm", "args": { - "m": 32, - "n": 7168, - "k": 4608, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 192, + "n": 576, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/kv_a_proj_with_mqa" + ] }, { "type": "gemm", "args": { - "m": 98304, - "n": 4608, + "m": 192, + "n": 1088, "k": 7168, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] }, { "type": "gemm", "args": { - "m": 98304, - "n": 7168, - "k": 4608, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 192, + "n": 1152, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/gate_up_proj" + ] }, { "type": "gemm", "args": { - "m": 96, - "n": 24576, + "m": 192, + "n": 1536, "k": 1536, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/Kimi-K2.5-MXFP4/q_b_proj", + "moonshotai/Kimi-K2.5/q_b_proj", + "nvidia/DeepSeek-R1-0528-FP4-V2/q_b_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/q_b_proj", + "nvidia/Kimi-K2.5-NVFP4/q_b_proj", + "nvidia/Kimi-K2.6-NVFP4/q_b_proj" + ] }, { "type": "gemm", "args": { - "m": 96, - "n": 32768, - "k": 512, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 192, + "n": 1536, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/Kimi-K2.5-MXFP4/q_a_proj", + "moonshotai/Kimi-K2.5/q_a_proj", + "nvidia/DeepSeek-R1-0528-FP4-V2/q_a_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/q_a_proj", + "nvidia/Kimi-K2.5-NVFP4/q_a_proj", + "nvidia/Kimi-K2.6-NVFP4/q_a_proj" + ] }, { "type": "gemm", "args": { - "m": 96, - "n": 7168, - "k": 16384, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 192, + "n": 1536, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/gate_up_proj", + "deepseek-ai/DeepSeek-R1-0528/q_a_proj", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w3" + ] }, { "type": "gemm", "args": { - "m": 96, - "n": 4608, - "k": 7168, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 192, + "n": 2048, + "k": 512, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/Kimi-K2.5-MXFP4/kv_b_proj", + "moonshotai/Kimi-K2.5/kv_b_proj", + "nvidia/DeepSeek-R1-0528-FP4-V2/kv_b_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/kv_b_proj", + "nvidia/Kimi-K2.5-NVFP4/kv_b_proj", + "nvidia/Kimi-K2.6-NVFP4/kv_b_proj" + ] }, { "type": "gemm", "args": { - "m": 96, - "n": 7168, - "k": 4608, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 192, + "n": 2304, + "k": 7168, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/gate_up_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/gate_up_proj", + "nvidia/Kimi-K2.5-NVFP4/gate_up_proj" + ] }, { "type": "gemm", "args": { - "m": 384, + "m": 192, "n": 3072, "k": 1536, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/Kimi-K2.5-MXFP4/q_b_proj", + "amd/MiniMax-M2.5-MXFP4/o_proj", + "moonshotai/Kimi-K2.5/q_b_proj", + "nvidia/DeepSeek-R1-0528-FP4-V2/q_b_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/q_b_proj", + "nvidia/Kimi-K2.5-NVFP4/q_b_proj", + "nvidia/Kimi-K2.6-NVFP4/q_b_proj", + "nvidia/MiniMax-M2.5-NVFP4/o_proj" + ] }, { "type": "gemm", "args": { - "m": 384, + "m": 192, "n": 4096, "k": 512, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "Qwen/Qwen3.5-397B-A17B/o_proj", + "Qwen/Qwen3.5-397B-A17B/shared_expert.down_proj", + "amd/Kimi-K2.5-MXFP4/kv_b_proj", + "amd/Qwen3.5-397B-A17B-MXFP4/o_proj", + "amd/Qwen3.5-397B-A17B-MXFP4/shared_expert.down_proj", + "moonshotai/Kimi-K2.5/kv_b_proj", + "nvidia/DeepSeek-R1-0528-FP4-V2/kv_b_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/kv_b_proj", + "nvidia/Kimi-K2.5-NVFP4/kv_b_proj", + "nvidia/Kimi-K2.6-NVFP4/kv_b_proj", + "nvidia/Qwen3.5-397B-A17B-NVFP4/o_proj", + "nvidia/Qwen3.5-397B-A17B-NVFP4/shared_expert.down_proj" + ] }, { "type": "gemm", "args": { - "m": 384, + "m": 192, "n": 7168, - "k": 2048, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "k": 576, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/down_proj" + ] }, { "type": "gemm", "args": { - "m": 4194304, - "n": 1536, - "k": 7168, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 192, + "n": 7168, + "k": 1024, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/Kimi-K2.5-MXFP4/o_proj", + "moonshotai/Kimi-K2.5/o_proj", + "nvidia/Kimi-K2.5-NVFP4/o_proj", + "nvidia/Kimi-K2.6-NVFP4/o_proj" + ] }, { "type": "gemm", "args": { - "m": 4194304, - "n": 24576, - "k": 1536, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 192, + "n": 7168, + "k": 1152, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/down_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/down_proj", + "nvidia/Kimi-K2.5-NVFP4/down_proj" + ] }, { "type": "gemm", "args": { - "m": 4194304, - "n": 576, - "k": 7168, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 192, + "n": 7168, + "k": 2048, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/o_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/o_proj" + ] }, { "type": "gemm", "args": { - "m": 4194304, - "n": 32768, - "k": 512, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 192, + "n": 7168, + "k": 2304, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/down_proj" + ] }, { "type": "gemm", "args": { - "m": 4194304, + "m": 192, "n": 7168, - "k": 16384, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "k": 8192, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/Kimi-K2.5-MXFP4/o_proj", + "moonshotai/Kimi-K2.5/o_proj", + "nvidia/Kimi-K2.5-NVFP4/o_proj", + "nvidia/Kimi-K2.6-NVFP4/o_proj" + ] }, { "type": "gemm", "args": { - "m": 4194304, - "n": 4608, - "k": 7168, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 192, + "n": 7168, + "k": 16384, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/o_proj", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] }, { "type": "gemm", "args": { - "m": 4194304, + "m": 192, "n": 7168, - "k": 4608, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "k": 65536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] }, { "type": "gemm", "args": { - "m": 4096, - "n": 4608, - "k": 7168, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 192, + "n": 12288, + "k": 1536, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/Kimi-K2.5-MXFP4/q_b_proj", + "moonshotai/Kimi-K2.5/q_b_proj", + "nvidia/DeepSeek-R1-0528-FP4-V2/q_b_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/q_b_proj", + "nvidia/Kimi-K2.5-NVFP4/q_b_proj", + "nvidia/Kimi-K2.6-NVFP4/q_b_proj" + ] }, { "type": "gemm", "args": { - "m": 4096, - "n": 7168, - "k": 4608, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 192, + "n": 16384, + "k": 512, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/Kimi-K2.5-MXFP4/kv_b_proj", + "moonshotai/Kimi-K2.5/kv_b_proj", + "nvidia/DeepSeek-R1-0528-FP4-V2/kv_b_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/kv_b_proj", + "nvidia/Kimi-K2.5-NVFP4/kv_b_proj", + "nvidia/Kimi-K2.6-NVFP4/kv_b_proj" + ] }, { "type": "gemm", "args": { - "m": 2097152, - "n": 1536, - "k": 7168, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 192, + "n": 16384, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_b" + ] }, { "type": "gemm", "args": { - "m": 2097152, + "m": 192, "n": 24576, "k": 1536, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/q_b_proj" + ] }, { "type": "gemm", "args": { - "m": 2097152, - "n": 576, - "k": 7168, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 192, + "n": 30720, + "k": 1024, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] }, { "type": "gemm", "args": { - "m": 2097152, + "m": 192, "n": 32768, "k": 512, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/kv_b_proj" + ] }, { "type": "gemm", "args": { - "m": 2097152, - "n": 7168, - "k": 16384, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 192, + "n": 65536, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_b" + ] }, { "type": "gemm", "args": { - "m": 2097152, - "n": 2304, - "k": 7168, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 192, + "n": 122880, + "k": 1024, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] }, { "type": "gemm", "args": { - "m": 2097152, - "n": 7168, - "k": 2304, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 195, + "n": 576, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/Kimi-K2.5-MXFP4/kv_a_proj_with_mqa", + "moonshotai/Kimi-K2.5/kv_a_proj_with_mqa", + "nvidia/DeepSeek-R1-0528-FP4-V2/kv_a_proj_with_mqa", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/kv_a_proj_with_mqa", + "nvidia/Kimi-K2.5-NVFP4/kv_a_proj_with_mqa", + "nvidia/Kimi-K2.6-NVFP4/kv_a_proj_with_mqa" + ] }, { "type": "gemm", "args": { - "m": 2048, + "m": 195, "n": 1536, "k": 7168, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/Kimi-K2.5-MXFP4/q_a_proj", + "moonshotai/Kimi-K2.5/q_a_proj", + "nvidia/DeepSeek-R1-0528-FP4-V2/q_a_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/q_a_proj", + "nvidia/Kimi-K2.5-NVFP4/q_a_proj", + "nvidia/Kimi-K2.6-NVFP4/q_a_proj" + ] }, { "type": "gemm", "args": { - "m": 2048, - "n": 24576, - "k": 1536, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 195, + "n": 2304, + "k": 7168, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/gate_up_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/gate_up_proj", + "nvidia/Kimi-K2.5-NVFP4/gate_up_proj" + ] }, { "type": "gemm", "args": { - "m": 2048, - "n": 576, - "k": 7168, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 195, + "n": 3072, + "k": 1536, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/Kimi-K2.5-MXFP4/q_b_proj", + "amd/MiniMax-M2.5-MXFP4/o_proj", + "moonshotai/Kimi-K2.5/q_b_proj", + "nvidia/DeepSeek-R1-0528-FP4-V2/q_b_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/q_b_proj", + "nvidia/Kimi-K2.5-NVFP4/q_b_proj", + "nvidia/Kimi-K2.6-NVFP4/q_b_proj", + "nvidia/MiniMax-M2.5-NVFP4/o_proj" + ] }, { "type": "gemm", "args": { - "m": 2048, - "n": 32768, + "m": 195, + "n": 4096, "k": 512, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "Qwen/Qwen3.5-397B-A17B/o_proj", + "Qwen/Qwen3.5-397B-A17B/shared_expert.down_proj", + "amd/Kimi-K2.5-MXFP4/kv_b_proj", + "amd/Qwen3.5-397B-A17B-MXFP4/o_proj", + "amd/Qwen3.5-397B-A17B-MXFP4/shared_expert.down_proj", + "moonshotai/Kimi-K2.5/kv_b_proj", + "nvidia/DeepSeek-R1-0528-FP4-V2/kv_b_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/kv_b_proj", + "nvidia/Kimi-K2.5-NVFP4/kv_b_proj", + "nvidia/Kimi-K2.6-NVFP4/kv_b_proj", + "nvidia/Qwen3.5-397B-A17B-NVFP4/o_proj", + "nvidia/Qwen3.5-397B-A17B-NVFP4/shared_expert.down_proj" + ] }, { "type": "gemm", "args": { - "m": 2048, + "m": 195, "n": 7168, - "k": 16384, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 2048, - "n": 2304, - "k": 7168, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "k": 2048, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/o_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/o_proj" + ] }, { "type": "gemm", "args": { - "m": 2048, + "m": 195, "n": 7168, "k": 2304, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/down_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/down_proj", + "nvidia/Kimi-K2.5-NVFP4/down_proj" + ] }, { "type": "gemm", "args": { - "m": 81920, - "n": 1536, + "m": 197, + "n": 576, "k": 7168, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/Kimi-K2.5-MXFP4/kv_a_proj_with_mqa", + "moonshotai/Kimi-K2.5/kv_a_proj_with_mqa", + "nvidia/DeepSeek-R1-0528-FP4-V2/kv_a_proj_with_mqa", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/kv_a_proj_with_mqa", + "nvidia/Kimi-K2.5-NVFP4/kv_a_proj_with_mqa", + "nvidia/Kimi-K2.6-NVFP4/kv_a_proj_with_mqa" + ] }, { "type": "gemm", "args": { - "m": 81920, - "n": 24576, - "k": 1536, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 81920, + "m": 197, "n": 576, "k": 7168, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 32 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 1, + 32 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "amd/DeepSeek-R1-0528-MXFP4-Preview/kv_a_proj_with_mqa", + "amd/DeepSeek-R1-0528-MXFP4-v2/kv_a_proj_with_mqa", + "amd/DeepSeek-R1-0528-MXFP4/kv_a_proj_with_mqa" + ] }, { "type": "gemm", "args": { - "m": 81920, - "n": 32768, - "k": 512, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 197, + "n": 1536, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/Kimi-K2.5-MXFP4/q_a_proj", + "moonshotai/Kimi-K2.5/q_a_proj", + "nvidia/DeepSeek-R1-0528-FP4-V2/q_a_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/q_a_proj", + "nvidia/Kimi-K2.5-NVFP4/q_a_proj", + "nvidia/Kimi-K2.6-NVFP4/q_a_proj" + ] }, { "type": "gemm", "args": { - "m": 81920, - "n": 7168, - "k": 16384, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 197, + "n": 1536, + "k": 7168, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 32 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 1, + 32 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "amd/DeepSeek-R1-0528-MXFP4-Preview/q_a_proj", + "amd/DeepSeek-R1-0528-MXFP4-v2/q_a_proj", + "amd/DeepSeek-R1-0528-MXFP4/q_a_proj" + ] }, { "type": "gemm", "args": { - "m": 81920, - "n": 4608, + "m": 197, + "n": 1536, "k": 7168, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w3" + ] }, { "type": "gemm", "args": { - "m": 81920, + "m": 197, "n": 7168, - "k": 4608, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "k": 9216, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/down_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/down_proj", + "nvidia/Kimi-K2.5-NVFP4/down_proj" + ] }, { "type": "gemm", "args": { - "m": 20, - "n": 24576, - "k": 1536, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 197, + "n": 7168, + "k": 16384, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/o_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/o_proj" + ] }, { "type": "gemm", "args": { - "m": 20, - "n": 32768, - "k": 512, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 197, + "n": 7168, + "k": 16384, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 32 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 1, + 32 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "amd/DeepSeek-R1-0528-MXFP4-Preview/o_proj", + "amd/DeepSeek-R1-0528-MXFP4-v2/o_proj", + "amd/DeepSeek-R1-0528-MXFP4/o_proj" + ] }, { "type": "gemm", "args": { - "m": 20, + "m": 197, "n": 7168, "k": 16384, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] }, { "type": "gemm", "args": { - "m": 20, - "n": 4608, + "m": 197, + "n": 9216, "k": 7168, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/gate_up_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/gate_up_proj", + "nvidia/Kimi-K2.5-NVFP4/gate_up_proj" + ] }, { "type": "gemm", "args": { - "m": 20, - "n": 7168, - "k": 4608, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 197, + "n": 9216, + "k": 7168, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 32 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 1, + 32 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "amd/DeepSeek-R1-0528-MXFP4-Preview/gate_up_proj", + "amd/DeepSeek-R1-0528-MXFP4-v2/gate_up_proj", + "amd/DeepSeek-R1-0528-MXFP4/gate_up_proj", + "amd/Kimi-K2.5-MXFP4/gate_up_proj" + ] }, { "type": "gemm", "args": { - "m": 327680, - "n": 3072, + "m": 197, + "n": 24576, "k": 1536, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/q_b_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/q_b_proj" + ] }, { "type": "gemm", "args": { - "m": 327680, - "n": 4096, - "k": 512, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 197, + "n": 24576, + "k": 1536, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 32 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 1, + 32 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "amd/DeepSeek-R1-0528-MXFP4-Preview/q_b_proj", + "amd/DeepSeek-R1-0528-MXFP4-v2/q_b_proj", + "amd/DeepSeek-R1-0528-MXFP4/q_b_proj" + ] }, { "type": "gemm", "args": { - "m": 327680, - "n": 7168, - "k": 2048, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 197, + "n": 32768, + "k": 512, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/kv_b_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/kv_b_proj" + ] }, { "type": "gemm", "args": { - "m": 80, - "n": 3072, - "k": 1536, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 197, + "n": 32768, + "k": 512, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 32 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 1, + 32 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "amd/DeepSeek-R1-0528-MXFP4-Preview/kv_b_proj", + "amd/DeepSeek-R1-0528-MXFP4-v2/kv_b_proj", + "amd/DeepSeek-R1-0528-MXFP4/kv_b_proj" + ] }, { "type": "gemm", "args": { - "m": 80, - "n": 4096, - "k": 512, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 200, + "n": 576, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/Kimi-K2.5-MXFP4/kv_a_proj_with_mqa", + "moonshotai/Kimi-K2.5/kv_a_proj_with_mqa", + "nvidia/DeepSeek-R1-0528-FP4-V2/kv_a_proj_with_mqa", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/kv_a_proj_with_mqa", + "nvidia/Kimi-K2.5-NVFP4/kv_a_proj_with_mqa", + "nvidia/Kimi-K2.6-NVFP4/kv_a_proj_with_mqa" + ] }, { "type": "gemm", "args": { - "m": 80, - "n": 7168, - "k": 2048, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 200, + "n": 1536, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/Kimi-K2.5-MXFP4/q_a_proj", + "moonshotai/Kimi-K2.5/q_a_proj", + "nvidia/DeepSeek-R1-0528-FP4-V2/q_a_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/q_a_proj", + "nvidia/Kimi-K2.5-NVFP4/q_a_proj", + "nvidia/Kimi-K2.6-NVFP4/q_a_proj" + ] }, { "type": "gemm", "args": { - "m": 16384, - "n": 4608, + "m": 200, + "n": 2304, "k": 7168, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/gate_up_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/gate_up_proj", + "nvidia/Kimi-K2.5-NVFP4/gate_up_proj" + ] }, { "type": "gemm", "args": { - "m": 16384, + "m": 200, "n": 7168, - "k": 4608, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "k": 2304, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/down_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/down_proj", + "nvidia/Kimi-K2.5-NVFP4/down_proj" + ] }, { "type": "gemm", "args": { - "m": 4, - "n": 4608, - "k": 7168, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 200, + "n": 7168, + "k": 9216, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/down_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/down_proj", + "nvidia/Kimi-K2.5-NVFP4/down_proj" + ] }, { "type": "gemm", "args": { - "m": 4, + "m": 200, "n": 7168, - "k": 4608, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "k": 16384, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/o_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/o_proj" + ] }, { "type": "gemm", "args": { - "m": 10, - "n": 1536, + "m": 200, + "n": 9216, "k": 7168, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/gate_up_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/gate_up_proj", + "nvidia/Kimi-K2.5-NVFP4/gate_up_proj" + ] }, { "type": "gemm", "args": { - "m": 10, - "n": 24576, - "k": 1536, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 200, + "n": 9216, + "k": 7168, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 32 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 1, + 32 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "amd/DeepSeek-R1-0528-MXFP4-Preview/gate_up_proj", + "amd/DeepSeek-R1-0528-MXFP4-v2/gate_up_proj", + "amd/DeepSeek-R1-0528-MXFP4/gate_up_proj", + "amd/Kimi-K2.5-MXFP4/gate_up_proj" + ] }, { "type": "gemm", "args": { - "m": 10, - "n": 576, - "k": 7168, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 200, + "n": 24576, + "k": 1536, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/q_b_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/q_b_proj" + ] }, { "type": "gemm", "args": { - "m": 10, + "m": 200, "n": 32768, "k": 512, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/kv_b_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/kv_b_proj" + ] }, { "type": "gemm", "args": { - "m": 10, - "n": 7168, - "k": 16384, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 209, + "n": 576, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/Kimi-K2.5-MXFP4/kv_a_proj_with_mqa", + "moonshotai/Kimi-K2.5/kv_a_proj_with_mqa", + "nvidia/DeepSeek-R1-0528-FP4-V2/kv_a_proj_with_mqa", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/kv_a_proj_with_mqa", + "nvidia/Kimi-K2.5-NVFP4/kv_a_proj_with_mqa", + "nvidia/Kimi-K2.6-NVFP4/kv_a_proj_with_mqa" + ] }, { "type": "gemm", "args": { - "m": 10, - "n": 4608, + "m": 209, + "n": 576, "k": 7168, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 32 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 1, + 32 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "amd/DeepSeek-R1-0528-MXFP4-Preview/kv_a_proj_with_mqa", + "amd/DeepSeek-R1-0528-MXFP4-v2/kv_a_proj_with_mqa", + "amd/DeepSeek-R1-0528-MXFP4/kv_a_proj_with_mqa" + ] }, { "type": "gemm", "args": { - "m": 10, - "n": 7168, - "k": 4608, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 209, + "n": 1536, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/Kimi-K2.5-MXFP4/q_a_proj", + "moonshotai/Kimi-K2.5/q_a_proj", + "nvidia/DeepSeek-R1-0528-FP4-V2/q_a_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/q_a_proj", + "nvidia/Kimi-K2.5-NVFP4/q_a_proj", + "nvidia/Kimi-K2.6-NVFP4/q_a_proj" + ] }, { "type": "gemm", "args": { - "m": 40, + "m": 209, "n": 1536, "k": 7168, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 32 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 1, + 32 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "amd/DeepSeek-R1-0528-MXFP4-Preview/q_a_proj", + "amd/DeepSeek-R1-0528-MXFP4-v2/q_a_proj", + "amd/DeepSeek-R1-0528-MXFP4/q_a_proj" + ] }, { "type": "gemm", "args": { - "m": 40, - "n": 3072, - "k": 1536, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 209, + "n": 1536, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w3" + ] }, { "type": "gemm", "args": { - "m": 40, - "n": 576, + "m": 209, + "n": 4608, "k": 7168, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/gate_up_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/gate_up_proj", + "nvidia/Kimi-K2.5-NVFP4/gate_up_proj" + ] }, { "type": "gemm", "args": { - "m": 40, - "n": 4096, - "k": 512, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 209, + "n": 4608, + "k": 7168, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 32 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 1, + 32 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "amd/DeepSeek-R1-0528-MXFP4-Preview/gate_up_proj", + "amd/DeepSeek-R1-0528-MXFP4-v2/gate_up_proj", + "amd/DeepSeek-R1-0528-MXFP4/gate_up_proj", + "amd/Kimi-K2.5-MXFP4/gate_up_proj" + ] }, { "type": "gemm", "args": { - "m": 40, + "m": 209, "n": 7168, - "k": 2048, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "k": 4608, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/down_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/down_proj", + "nvidia/Kimi-K2.5-NVFP4/down_proj" + ] }, { "type": "gemm", "args": { - "m": 40, - "n": 2304, - "k": 7168, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 209, + "n": 7168, + "k": 4608, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 32 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 1, + 32 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "amd/DeepSeek-R1-0528-MXFP4-Preview/down_proj", + "amd/DeepSeek-R1-0528-MXFP4-v2/down_proj", + "amd/DeepSeek-R1-0528-MXFP4/down_proj", + "amd/Kimi-K2.5-MXFP4/down_proj" + ] }, { "type": "gemm", "args": { - "m": 40, + "m": 209, "n": 7168, - "k": 2304, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "k": 16384, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/o_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/o_proj" + ] }, { "type": "gemm", "args": { - "m": 147456, - "n": 1536, - "k": 7168, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 209, + "n": 7168, + "k": 16384, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 32 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 1, + 32 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "amd/DeepSeek-R1-0528-MXFP4-Preview/o_proj", + "amd/DeepSeek-R1-0528-MXFP4-v2/o_proj", + "amd/DeepSeek-R1-0528-MXFP4/o_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 209, + "n": 7168, + "k": 16384, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] }, { "type": "gemm", "args": { - "m": 147456, + "m": 209, "n": 24576, "k": 1536, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/q_b_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/q_b_proj" + ] }, { "type": "gemm", "args": { - "m": 147456, - "n": 576, - "k": 7168, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 209, + "n": 24576, + "k": 1536, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 32 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 1, + 32 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "amd/DeepSeek-R1-0528-MXFP4-Preview/q_b_proj", + "amd/DeepSeek-R1-0528-MXFP4-v2/q_b_proj", + "amd/DeepSeek-R1-0528-MXFP4/q_b_proj" + ] }, { "type": "gemm", "args": { - "m": 147456, + "m": 209, "n": 32768, "k": 512, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/kv_b_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/kv_b_proj" + ] }, { "type": "gemm", "args": { - "m": 147456, - "n": 7168, - "k": 16384, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 209, + "n": 32768, + "k": 512, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 32 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 1, + 32 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "amd/DeepSeek-R1-0528-MXFP4-Preview/kv_b_proj", + "amd/DeepSeek-R1-0528-MXFP4-v2/kv_b_proj", + "amd/DeepSeek-R1-0528-MXFP4/kv_b_proj" + ] }, { "type": "gemm", "args": { - "m": 147456, - "n": 4608, + "m": 218, + "n": 576, "k": 7168, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 147456, - "n": 7168, - "k": 4608, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/Kimi-K2.5-MXFP4/kv_a_proj_with_mqa", + "moonshotai/Kimi-K2.5/kv_a_proj_with_mqa", + "nvidia/DeepSeek-R1-0528-FP4-V2/kv_a_proj_with_mqa", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/kv_a_proj_with_mqa", + "nvidia/Kimi-K2.5-NVFP4/kv_a_proj_with_mqa", + "nvidia/Kimi-K2.6-NVFP4/kv_a_proj_with_mqa" + ] }, { "type": "gemm", "args": { - "m": 36, + "m": 218, "n": 1536, "k": 7168, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 36, - "n": 24576, - "k": 1536, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/Kimi-K2.5-MXFP4/q_a_proj", + "moonshotai/Kimi-K2.5/q_a_proj", + "nvidia/DeepSeek-R1-0528-FP4-V2/q_a_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/q_a_proj", + "nvidia/Kimi-K2.5-NVFP4/q_a_proj", + "nvidia/Kimi-K2.6-NVFP4/q_a_proj" + ] }, { "type": "gemm", "args": { - "m": 36, - "n": 576, + "m": 218, + "n": 2304, "k": 7168, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/gate_up_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/gate_up_proj", + "nvidia/Kimi-K2.5-NVFP4/gate_up_proj" + ] }, { "type": "gemm", "args": { - "m": 36, - "n": 32768, - "k": 512, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 218, + "n": 7168, + "k": 2304, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/down_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/down_proj", + "nvidia/Kimi-K2.5-NVFP4/down_proj" + ] }, { "type": "gemm", "args": { - "m": 36, + "m": 218, "n": 7168, "k": 16384, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/o_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/o_proj" + ] }, { "type": "gemm", "args": { - "m": 36, - "n": 4608, - "k": 7168, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 218, + "n": 24576, + "k": 1536, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/q_b_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/q_b_proj" + ] }, { "type": "gemm", "args": { - "m": 36, - "n": 7168, - "k": 4608, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 218, + "n": 32768, + "k": 512, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/kv_b_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/kv_b_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 224, + "n": 576, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/kv_a_proj_with_mqa" + ] }, { "type": "gemm", "args": { - "m": 73728, + "m": 224, "n": 1536, "k": 7168, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/gate_up_proj", + "deepseek-ai/DeepSeek-R1-0528/q_a_proj", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w3" + ] }, { "type": "gemm", "args": { - "m": 73728, - "n": 24576, - "k": 1536, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 224, + "n": 1536, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w3" + ] }, { "type": "gemm", "args": { - "m": 73728, - "n": 576, + "m": 224, + "n": 2304, "k": 7168, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/gate_up_proj" + ] }, { "type": "gemm", "args": { - "m": 73728, - "n": 32768, - "k": 512, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 224, + "n": 7168, + "k": 2304, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/down_proj" + ] }, { "type": "gemm", "args": { - "m": 73728, + "m": 224, "n": 7168, "k": 16384, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/o_proj", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] }, { "type": "gemm", "args": { - "m": 73728, - "n": 2304, - "k": 7168, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 224, + "n": 24576, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/q_b_proj" + ] }, { "type": "gemm", "args": { - "m": 73728, - "n": 7168, - "k": 2304, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 224, + "n": 32768, + "k": 512, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/kv_b_proj" + ] }, { "type": "gemm", "args": { - "m": 18, - "n": 1536, + "m": 234, + "n": 576, "k": 7168, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/kv_a_proj_with_mqa" + ] }, { "type": "gemm", "args": { - "m": 18, - "n": 24576, + "m": 234, + "n": 1152, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/gate_up_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 234, + "n": 1536, "k": 1536, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/q_b_proj" + ] }, { "type": "gemm", "args": { - "m": 18, - "n": 576, + "m": 234, + "n": 1536, "k": 7168, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/gate_up_proj", + "deepseek-ai/DeepSeek-R1-0528/q_a_proj", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w3" + ] }, { "type": "gemm", "args": { - "m": 18, - "n": 32768, + "m": 234, + "n": 2048, "k": 512, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/kv_b_proj" + ] }, { "type": "gemm", "args": { - "m": 18, + "m": 234, "n": 7168, - "k": 16384, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "k": 1024, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/o_proj", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] }, { "type": "gemm", "args": { - "m": 18, - "n": 2304, - "k": 7168, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 234, + "n": 7168, + "k": 1152, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/down_proj" + ] }, { "type": "gemm", "args": { - "m": 18, - "n": 7168, - "k": 2304, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 240, + "n": 576, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/Kimi-K2.5-MXFP4/kv_a_proj_with_mqa", + "moonshotai/Kimi-K2.5/kv_a_proj_with_mqa", + "nvidia/DeepSeek-R1-0528-FP4-V2/kv_a_proj_with_mqa", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/kv_a_proj_with_mqa", + "nvidia/Kimi-K2.5-NVFP4/kv_a_proj_with_mqa", + "nvidia/Kimi-K2.6-NVFP4/kv_a_proj_with_mqa" + ] }, { "type": "gemm", "args": { - "m": 294912, - "n": 4608, + "m": 240, + "n": 1536, "k": 7168, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/Kimi-K2.5-MXFP4/q_a_proj", + "moonshotai/Kimi-K2.5/q_a_proj", + "nvidia/DeepSeek-R1-0528-FP4-V2/q_a_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/q_a_proj", + "nvidia/Kimi-K2.5-NVFP4/q_a_proj", + "nvidia/Kimi-K2.6-NVFP4/q_a_proj" + ] }, { "type": "gemm", "args": { - "m": 294912, - "n": 7168, - "k": 4608, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 240, + "n": 2304, + "k": 7168, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/gate_up_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/gate_up_proj", + "nvidia/Kimi-K2.5-NVFP4/gate_up_proj" + ] }, { "type": "gemm", "args": { - "m": 72, - "n": 4608, + "m": 240, + "n": 2304, "k": 7168, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/gate_up_proj" + ] }, { "type": "gemm", "args": { - "m": 72, - "n": 7168, - "k": 4608, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 240, + "n": 3072, + "k": 1536, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/Kimi-K2.5-MXFP4/q_b_proj", + "amd/MiniMax-M2.5-MXFP4/o_proj", + "moonshotai/Kimi-K2.5/q_b_proj", + "nvidia/DeepSeek-R1-0528-FP4-V2/q_b_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/q_b_proj", + "nvidia/Kimi-K2.5-NVFP4/q_b_proj", + "nvidia/Kimi-K2.6-NVFP4/q_b_proj", + "nvidia/MiniMax-M2.5-NVFP4/o_proj" + ] }, { "type": "gemm", "args": { - "m": 147456, - "n": 2304, - "k": 7168, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 240, + "n": 4096, + "k": 512, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "Qwen/Qwen3.5-397B-A17B/o_proj", + "Qwen/Qwen3.5-397B-A17B/shared_expert.down_proj", + "amd/Kimi-K2.5-MXFP4/kv_b_proj", + "amd/Qwen3.5-397B-A17B-MXFP4/o_proj", + "amd/Qwen3.5-397B-A17B-MXFP4/shared_expert.down_proj", + "moonshotai/Kimi-K2.5/kv_b_proj", + "nvidia/DeepSeek-R1-0528-FP4-V2/kv_b_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/kv_b_proj", + "nvidia/Kimi-K2.5-NVFP4/kv_b_proj", + "nvidia/Kimi-K2.6-NVFP4/kv_b_proj", + "nvidia/Qwen3.5-397B-A17B-NVFP4/o_proj", + "nvidia/Qwen3.5-397B-A17B-NVFP4/shared_expert.down_proj" + ] }, { "type": "gemm", "args": { - "m": 147456, + "m": 240, "n": 7168, - "k": 2304, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "k": 2048, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/o_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/o_proj" + ] }, { "type": "gemm", "args": { - "m": 36, - "n": 2304, - "k": 7168, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 240, + "n": 7168, + "k": 2304, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/down_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/down_proj", + "nvidia/Kimi-K2.5-NVFP4/down_proj" + ] }, { "type": "gemm", "args": { - "m": 36, + "m": 240, "n": 7168, - "k": 2304, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "k": 16384, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/o_proj", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] }, { "type": "gemm", "args": { - "m": 1048576, + "m": 240, "n": 24576, "k": 1536, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/q_b_proj" + ] }, { "type": "gemm", "args": { - "m": 1048576, + "m": 240, "n": 32768, "k": 512, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/kv_b_proj" + ] }, { "type": "gemm", "args": { - "m": 1048576, - "n": 7168, - "k": 16384, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 256, + "n": 128, + "k": 3072, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "MiniMaxAI/MiniMax-M2.5/k_proj", + "MiniMaxAI/MiniMax-M2.5/v_proj" + ] }, { "type": "gemm", "args": { - "m": 1048576, - "n": 4608, - "k": 7168, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 256, + "n": 128, + "k": 4096, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "Qwen/Qwen3.5-397B-A17B/shared_expert.gate_up_proj", + "amd/Qwen3.5-397B-A17B-MXFP4/shared_expert.gate_up_proj", + "nvidia/Qwen3.5-397B-A17B-NVFP4/shared_expert.gate_up_proj" + ] }, { "type": "gemm", "args": { - "m": 1048576, - "n": 7168, - "k": 4608, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 256, + "n": 128, + "k": 4096, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "Qwen/Qwen3.5-397B-A17B-FP8/shared_expert.gate_up_proj" + ] }, { "type": "gemm", "args": { "m": 256, - "n": 4608, - "k": 7168, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "n": 192, + "k": 3072, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "MiniMaxAI/MiniMax-M2.5/experts.gate_up_proj" + ] }, { "type": "gemm", "args": { "m": 256, - "n": 7168, - "k": 4608, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "n": 256, + "k": 2880, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/gpt-oss-120b-w-mxfp4-a-fp8/k_proj", + "amd/gpt-oss-120b-w-mxfp4-a-fp8/q_proj", + "amd/gpt-oss-120b-w-mxfp4-a-fp8/v_proj", + "openai/gpt-oss-120b/k_proj", + "openai/gpt-oss-120b/q_proj", + "openai/gpt-oss-120b/v_proj" + ] }, { "type": "gemm", "args": { - "m": 128, - "n": 24576, - "k": 1536, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 256, + "n": 256, + "k": 3072, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "MiniMaxAI/MiniMax-M2.5/k_proj", + "MiniMaxAI/MiniMax-M2.5/v_proj" + ] }, { "type": "gemm", "args": { - "m": 128, - "n": 32768, - "k": 512, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 256, + "n": 256, + "k": 4096, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "Qwen/Qwen3.5-397B-A17B/k_proj", + "Qwen/Qwen3.5-397B-A17B/shared_expert.gate_up_proj", + "Qwen/Qwen3.5-397B-A17B/v_proj", + "amd/Qwen3.5-397B-A17B-MXFP4/k_proj", + "amd/Qwen3.5-397B-A17B-MXFP4/shared_expert.gate_up_proj", + "amd/Qwen3.5-397B-A17B-MXFP4/v_proj", + "nvidia/Qwen3.5-397B-A17B-NVFP4/k_proj", + "nvidia/Qwen3.5-397B-A17B-NVFP4/shared_expert.gate_up_proj", + "nvidia/Qwen3.5-397B-A17B-NVFP4/v_proj" + ] }, { "type": "gemm", "args": { - "m": 128, - "n": 7168, - "k": 16384, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 256, + "n": 256, + "k": 4096, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "Qwen/Qwen3.5-397B-A17B-FP8/k_proj", + "Qwen/Qwen3.5-397B-A17B-FP8/shared_expert.gate_up_proj", + "Qwen/Qwen3.5-397B-A17B-FP8/v_proj" + ] }, { "type": "gemm", "args": { - "m": 131072, - "n": 4608, - "k": 7168, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 256, + "n": 384, + "k": 3072, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "MiniMaxAI/MiniMax-M2.5/experts.gate_up_proj", + "MiniMaxAI/MiniMax-M2.5/q_proj" + ] }, { "type": "gemm", "args": { - "m": 131072, - "n": 7168, - "k": 4608, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 256, + "n": 512, + "k": 2880, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/gpt-oss-120b-w-mxfp4-a-fp8/k_proj", + "amd/gpt-oss-120b-w-mxfp4-a-fp8/q_proj", + "amd/gpt-oss-120b-w-mxfp4-a-fp8/v_proj", + "openai/gpt-oss-120b/k_proj", + "openai/gpt-oss-120b/q_proj", + "openai/gpt-oss-120b/v_proj" + ] }, { "type": "gemm", "args": { - "m": 16, - "n": 4608, - "k": 7168, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 256, + "n": 512, + "k": 3072, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/MiniMax-M2.5-MXFP4/k_proj", + "amd/MiniMax-M2.5-MXFP4/v_proj", + "nvidia/MiniMax-M2.5-NVFP4/k_proj", + "nvidia/MiniMax-M2.5-NVFP4/v_proj" + ] }, { "type": "gemm", "args": { - "m": 16, - "n": 7168, - "k": 4608, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 256, + "n": 512, + "k": 3072, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "MiniMaxAI/MiniMax-M2.5/k_proj", + "MiniMaxAI/MiniMax-M2.5/v_proj" + ] }, { "type": "gemm", "args": { - "m": 524288, - "n": 4608, - "k": 7168, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 256, + "n": 512, + "k": 4096, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "Qwen/Qwen3.5-397B-A17B/k_proj", + "Qwen/Qwen3.5-397B-A17B/q_proj", + "Qwen/Qwen3.5-397B-A17B/shared_expert.gate_up_proj", + "Qwen/Qwen3.5-397B-A17B/v_proj", + "amd/Qwen3.5-397B-A17B-MXFP4/k_proj", + "amd/Qwen3.5-397B-A17B-MXFP4/q_proj", + "amd/Qwen3.5-397B-A17B-MXFP4/shared_expert.gate_up_proj", + "amd/Qwen3.5-397B-A17B-MXFP4/v_proj", + "nvidia/Qwen3.5-397B-A17B-NVFP4/k_proj", + "nvidia/Qwen3.5-397B-A17B-NVFP4/q_proj", + "nvidia/Qwen3.5-397B-A17B-NVFP4/shared_expert.gate_up_proj", + "nvidia/Qwen3.5-397B-A17B-NVFP4/v_proj" + ] }, { "type": "gemm", "args": { - "m": 524288, - "n": 7168, - "k": 4608, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 256, + "n": 512, + "k": 4096, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "Qwen/Qwen3.5-397B-A17B-FP8/k_proj", + "Qwen/Qwen3.5-397B-A17B-FP8/q_proj", + "Qwen/Qwen3.5-397B-A17B-FP8/shared_expert.gate_up_proj", + "Qwen/Qwen3.5-397B-A17B-FP8/v_proj", + "deepseek-ai/DeepSeek-V4.1-Flash/attn.wo_a" + ] }, { "type": "gemm", "args": { - "m": 64, - "n": 4608, - "k": 7168, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 256, + "n": 576, + "k": 6144, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/GLM-5.1-MXFP4/kv_a_proj_with_mqa", + "amd/GLM-5.2-MXFP4/kv_a_proj_with_mqa", + "nvidia/GLM-5-NVFP4/kv_a_proj_with_mqa", + "nvidia/GLM-5.1-NVFP4/kv_a_proj_with_mqa", + "nvidia/GLM-5.2-NVFP4/kv_a_proj_with_mqa" + ] }, { "type": "gemm", "args": { - "m": 64, - "n": 7168, - "k": 4608, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 256, + "n": 576, + "k": 6144, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "zai-org/GLM-5-FP8/kv_a_proj_with_mqa", + "zai-org/GLM-5.1-FP8/kv_a_proj_with_mqa", + "zai-org/GLM-5.2-FP8/kv_a_proj_with_mqa" + ] }, { "type": "gemm", "args": { - "m": 128, - "n": 4608, + "m": 256, + "n": 576, "k": 7168, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/Kimi-K2.5-MXFP4/kv_a_proj_with_mqa", + "moonshotai/Kimi-K2.5/kv_a_proj_with_mqa", + "nvidia/DeepSeek-R1-0528-FP4-V2/kv_a_proj_with_mqa", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/kv_a_proj_with_mqa", + "nvidia/Kimi-K2.5-NVFP4/kv_a_proj_with_mqa", + "nvidia/Kimi-K2.6-NVFP4/kv_a_proj_with_mqa" + ] }, { "type": "gemm", "args": { - "m": 128, - "n": 7168, - "k": 4608, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 256, + "n": 576, + "k": 7168, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 32 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 1, + 32 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "amd/DeepSeek-R1-0528-MXFP4-Preview/kv_a_proj_with_mqa", + "amd/DeepSeek-R1-0528-MXFP4-v2/kv_a_proj_with_mqa", + "amd/DeepSeek-R1-0528-MXFP4/kv_a_proj_with_mqa" + ] }, { "type": "gemm", "args": { - "m": 2097152, - "n": 3072, - "k": 1536, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 256, + "n": 576, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/kv_a_proj_with_mqa" + ] }, { "type": "gemm", "args": { - "m": 2097152, - "n": 4096, - "k": 512, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 256, + "n": 768, + "k": 3072, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "MiniMaxAI/MiniMax-M2.5/experts.gate_up_proj", + "MiniMaxAI/MiniMax-M2.5/q_proj" + ] }, { "type": "gemm", "args": { - "m": 2097152, - "n": 7168, - "k": 2048, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 256, + "n": 768, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w3" + ] }, { "type": "gemm", "args": { "m": 256, - "n": 3072, - "k": 1536, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "n": 1024, + "k": 3072, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "MiniMaxAI/MiniMax-M2.5/k_proj", + "MiniMaxAI/MiniMax-M2.5/qkv_proj", + "MiniMaxAI/MiniMax-M2.5/v_proj" + ] }, { "type": "gemm", "args": { "m": 256, - "n": 4096, - "k": 512, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "n": 1024, + "k": 4096, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "Qwen/Qwen3.5-397B-A17B/q_proj", + "Qwen/Qwen3.5-397B-A17B/shared_expert.gate_up_proj", + "amd/Qwen3.5-397B-A17B-MXFP4/q_proj", + "amd/Qwen3.5-397B-A17B-MXFP4/shared_expert.gate_up_proj", + "nvidia/Qwen3.5-397B-A17B-NVFP4/q_proj", + "nvidia/Qwen3.5-397B-A17B-NVFP4/shared_expert.gate_up_proj" + ] }, { "type": "gemm", "args": { "m": 256, - "n": 7168, - "k": 2048, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "n": 1024, + "k": 4096, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "Qwen/Qwen3.5-397B-A17B-FP8/q_proj", + "Qwen/Qwen3.5-397B-A17B-FP8/shared_expert.gate_up_proj", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_a", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_a", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_a" + ] }, { "type": "gemm", "args": { - "m": 2201600, - "n": 1536, + "m": 256, + "n": 1088, "k": 7168, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 2201600, - "n": 24576, - "k": 1536, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] }, { "type": "gemm", "args": { - "m": 2201600, - "n": 576, + "m": 256, + "n": 1152, "k": 7168, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-NVFP4-v2/gate_up_proj" + ] }, { "type": "gemm", "args": { - "m": 2201600, - "n": 32768, - "k": 512, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 256, + "n": 1536, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/q_b_proj" + ] }, { "type": "gemm", "args": { - "m": 2201600, - "n": 7168, - "k": 16384, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 256, + "n": 1536, + "k": 3072, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/MiniMax-M2.5-NVFP4/experts.gate_up_proj" + ] }, { "type": "gemm", "args": { - "m": 2201600, - "n": 4608, - "k": 7168, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 256, + "n": 1536, + "k": 3072, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 32 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 1, + 32 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "amd/MiniMax-M2.5-MXFP4/experts.gate_up_proj" + ] }, { "type": "gemm", "args": { - "m": 2201600, - "n": 7168, - "k": 4608, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 256, + "n": 1536, + "k": 3072, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "MiniMaxAI/MiniMax-M2.5/experts.gate_up_proj", + "MiniMaxAI/MiniMax-M2.5/q_proj" + ] }, { "type": "gemm", "args": { - "m": 268, + "m": 256, "n": 1536, - "k": 7168, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "k": 6144, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "zai-org/GLM-5-FP8/gate_up_proj", + "zai-org/GLM-5.1-FP8/gate_up_proj", + "zai-org/GLM-5.2-FP8/gate_up_proj" + ] }, { "type": "gemm", "args": { - "m": 268, - "n": 24576, - "k": 1536, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 256, + "n": 1536, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/Kimi-K2.5-MXFP4/q_a_proj", + "moonshotai/Kimi-K2.5/q_a_proj", + "nvidia/DeepSeek-R1-0528-FP4-V2/q_a_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/q_a_proj", + "nvidia/Kimi-K2.5-NVFP4/q_a_proj", + "nvidia/Kimi-K2.6-NVFP4/q_a_proj" + ] }, { "type": "gemm", "args": { - "m": 268, - "n": 576, + "m": 256, + "n": 1536, "k": 7168, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 32 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 1, + 32 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "amd/DeepSeek-R1-0528-MXFP4-Preview/q_a_proj", + "amd/DeepSeek-R1-0528-MXFP4-v2/q_a_proj", + "amd/DeepSeek-R1-0528-MXFP4/q_a_proj" + ] }, { "type": "gemm", "args": { - "m": 268, - "n": 32768, - "k": 512, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 256, + "n": 1536, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/gate_up_proj", + "deepseek-ai/DeepSeek-R1-0528/q_a_proj", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w3" + ] }, { "type": "gemm", "args": { - "m": 268, - "n": 7168, - "k": 16384, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 256, + "n": 1536, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w3" + ] }, { "type": "gemm", "args": { - "m": 268, - "n": 4608, - "k": 7168, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 256, + "n": 2048, + "k": 512, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/kv_b_proj" + ] }, { "type": "gemm", "args": { - "m": 268, - "n": 7168, - "k": 4608, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 256, + "n": 2048, + "k": 2048, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "zai-org/GLM-5-FP8/q_b_proj", + "zai-org/GLM-5.1-FP8/q_b_proj", + "zai-org/GLM-5.2-FP8/q_b_proj" + ] }, { "type": "gemm", "args": { - "m": 1100800, - "n": 1536, - "k": 7168, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 256, + "n": 2048, + "k": 2880, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/gpt-oss-120b-w-mxfp4-a-fp8/q_proj", + "openai/gpt-oss-120b/q_proj" + ] }, { "type": "gemm", "args": { - "m": 1100800, - "n": 24576, - "k": 1536, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 256, + "n": 2048, + "k": 4096, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "Qwen/Qwen3.5-397B-A17B/q_proj", + "Qwen/Qwen3.5-397B-A17B/shared_expert.gate_up_proj", + "amd/Qwen3.5-397B-A17B-MXFP4/q_proj", + "amd/Qwen3.5-397B-A17B-MXFP4/shared_expert.gate_up_proj", + "nvidia/Qwen3.5-397B-A17B-NVFP4/q_proj", + "nvidia/Qwen3.5-397B-A17B-NVFP4/shared_expert.gate_up_proj" + ] }, { "type": "gemm", "args": { - "m": 1100800, - "n": 576, - "k": 7168, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 256, + "n": 2048, + "k": 4096, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "Qwen/Qwen3.5-397B-A17B-FP8/q_proj", + "Qwen/Qwen3.5-397B-A17B-FP8/shared_expert.gate_up_proj" + ] }, { "type": "gemm", "args": { - "m": 1100800, - "n": 32768, - "k": 512, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 256, + "n": 2048, + "k": 6144, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/GLM-5.1-MXFP4/q_a_proj", + "amd/GLM-5.2-MXFP4/q_a_proj", + "amd/MiniMax-M3-MXFP4/q_proj", + "nvidia/GLM-5-NVFP4/q_a_proj", + "nvidia/GLM-5.1-NVFP4/q_a_proj", + "nvidia/GLM-5.2-NVFP4/q_a_proj" + ] }, { "type": "gemm", "args": { - "m": 1100800, - "n": 7168, - "k": 16384, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 256, + "n": 2048, + "k": 6144, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "zai-org/GLM-5-FP8/q_a_proj", + "zai-org/GLM-5.1-FP8/q_a_proj", + "zai-org/GLM-5.2-FP8/q_a_proj" + ] }, { "type": "gemm", "args": { - "m": 1100800, + "m": 256, "n": 2304, "k": 7168, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/gate_up_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/gate_up_proj", + "nvidia/Kimi-K2.5-NVFP4/gate_up_proj" + ] }, { "type": "gemm", "args": { - "m": 1100800, - "n": 7168, - "k": 2304, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 256, + "n": 2880, + "k": 2048, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/gpt-oss-120b-w-mxfp4-a-fp8/o_proj", + "openai/gpt-oss-120b/o_proj" + ] }, { "type": "gemm", "args": { - "m": 134, - "n": 1536, - "k": 7168, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 256, + "n": 2880, + "k": 4096, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/gpt-oss-120b-w-mxfp4-a-fp8/o_proj", + "openai/gpt-oss-120b/o_proj" + ] }, { "type": "gemm", "args": { - "m": 134, - "n": 24576, - "k": 1536, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 256, + "n": 3072, + "k": 192, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "MiniMaxAI/MiniMax-M2.5/experts.down_proj" + ] }, { "type": "gemm", "args": { - "m": 134, - "n": 576, - "k": 7168, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 256, + "n": 3072, + "k": 384, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "MiniMaxAI/MiniMax-M2.5/experts.down_proj", + "MiniMaxAI/MiniMax-M2.5/o_proj" + ] }, { "type": "gemm", "args": { - "m": 134, - "n": 32768, - "k": 512, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 256, + "n": 3072, + "k": 768, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/MiniMax-M2.5-NVFP4/experts.down_proj" + ] }, { "type": "gemm", "args": { - "m": 134, - "n": 7168, - "k": 16384, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 256, + "n": 3072, + "k": 768, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 32 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 1, + 32 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "amd/MiniMax-M2.5-MXFP4/experts.down_proj" + ] }, { "type": "gemm", "args": { - "m": 134, - "n": 2304, - "k": 7168, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 256, + "n": 3072, + "k": 768, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "MiniMaxAI/MiniMax-M2.5/experts.down_proj", + "MiniMaxAI/MiniMax-M2.5/o_proj" + ] }, { "type": "gemm", "args": { - "m": 134, - "n": 7168, - "k": 2304, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 256, + "n": 3072, + "k": 1536, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/Kimi-K2.5-MXFP4/q_b_proj", + "amd/MiniMax-M2.5-MXFP4/o_proj", + "moonshotai/Kimi-K2.5/q_b_proj", + "nvidia/DeepSeek-R1-0528-FP4-V2/q_b_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/q_b_proj", + "nvidia/Kimi-K2.5-NVFP4/q_b_proj", + "nvidia/Kimi-K2.6-NVFP4/q_b_proj", + "nvidia/MiniMax-M2.5-NVFP4/o_proj" + ] }, { "type": "gemm", "args": { - "m": 6291456, - "n": 1536, - "k": 7168, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 256, + "n": 3072, + "k": 1536, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/MiniMax-M2.5-NVFP4/experts.down_proj" + ] }, { "type": "gemm", "args": { - "m": 6291456, - "n": 24576, + "m": 256, + "n": 3072, "k": 1536, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 32 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 1, + 32 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "amd/DeepSeek-R1-0528-MXFP4-Preview/q_b_proj", + "amd/DeepSeek-R1-0528-MXFP4-v2/q_b_proj", + "amd/DeepSeek-R1-0528-MXFP4/q_b_proj", + "amd/MiniMax-M2.5-MXFP4/experts.down_proj" + ] }, { "type": "gemm", "args": { - "m": 6291456, - "n": 576, - "k": 7168, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 256, + "n": 3072, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "MiniMaxAI/MiniMax-M2.5/experts.down_proj", + "MiniMaxAI/MiniMax-M2.5/o_proj", + "deepseek-ai/DeepSeek-R1-0528/q_b_proj" + ] }, { "type": "gemm", "args": { - "m": 6291456, - "n": 32768, - "k": 512, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 256, + "n": 3072, + "k": 3072, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/MiniMax-M2.5-MXFP4/o_proj", + "amd/MiniMax-M2.5-MXFP4/q_proj", + "nvidia/MiniMax-M2.5-NVFP4/o_proj", + "nvidia/MiniMax-M2.5-NVFP4/q_proj" + ] }, { "type": "gemm", "args": { - "m": 6291456, - "n": 7168, - "k": 16384, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 256, + "n": 3072, + "k": 3072, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/MiniMax-M2.5-NVFP4/experts.gate_up_proj" + ] }, { "type": "gemm", "args": { - "m": 6291456, - "n": 4608, - "k": 7168, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 256, + "n": 3072, + "k": 3072, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 32 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 1, + 32 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "amd/MiniMax-M2.5-MXFP4/experts.gate_up_proj" + ] }, { "type": "gemm", "args": { - "m": 6291456, - "n": 7168, - "k": 4608, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 256, + "n": 3072, + "k": 3072, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "MiniMaxAI/MiniMax-M2.5/experts.gate_up_proj", + "MiniMaxAI/MiniMax-M2.5/o_proj", + "MiniMaxAI/MiniMax-M2.5/q_proj" + ] }, { "type": "gemm", "args": { - "m": 3145728, - "n": 1536, - "k": 7168, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 256, + "n": 3072, + "k": 6144, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/GLM-5.1-MXFP4/gate_up_proj", + "amd/GLM-5.2-MXFP4/gate_up_proj", + "amd/MiniMax-M2.5-MXFP4/o_proj", + "amd/MiniMax-M3-MXFP4/gate_up_proj", + "nvidia/GLM-5.1-NVFP4/gate_up_proj", + "nvidia/GLM-5.2-NVFP4/gate_up_proj", + "nvidia/MiniMax-M2.5-NVFP4/o_proj" + ] }, { "type": "gemm", "args": { - "m": 3145728, - "n": 24576, - "k": 1536, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 256, + "n": 3072, + "k": 6144, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "MiniMaxAI/MiniMax-M2.5/o_proj", + "zai-org/GLM-5-FP8/gate_up_proj", + "zai-org/GLM-5.1-FP8/gate_up_proj", + "zai-org/GLM-5.2-FP8/gate_up_proj" + ] }, { "type": "gemm", "args": { - "m": 3145728, - "n": 576, - "k": 7168, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 256, + "n": 3584, + "k": 512, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "zai-org/GLM-5-FP8/kv_b_proj", + "zai-org/GLM-5.1-FP8/kv_b_proj", + "zai-org/GLM-5.2-FP8/kv_b_proj" + ] }, { "type": "gemm", "args": { - "m": 3145728, - "n": 32768, - "k": 512, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 256, + "n": 4096, + "k": 128, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "Qwen/Qwen3.5-397B-A17B/shared_expert.down_proj", + "amd/Qwen3.5-397B-A17B-MXFP4/shared_expert.down_proj", + "nvidia/Qwen3.5-397B-A17B-NVFP4/shared_expert.down_proj" + ] }, { "type": "gemm", "args": { - "m": 3145728, - "n": 7168, - "k": 16384, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 256, + "n": 4096, + "k": 128, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "Qwen/Qwen3.5-397B-A17B-FP8/shared_expert.down_proj" + ] }, { "type": "gemm", "args": { - "m": 3145728, - "n": 2304, - "k": 7168, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 256, + "n": 4096, + "k": 256, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "Qwen/Qwen3.5-397B-A17B/shared_expert.down_proj", + "amd/Qwen3.5-397B-A17B-MXFP4/shared_expert.down_proj", + "nvidia/Qwen3.5-397B-A17B-NVFP4/shared_expert.down_proj" + ] }, { "type": "gemm", "args": { - "m": 3145728, - "n": 7168, - "k": 2304, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 256, + "n": 4096, + "k": 256, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "Qwen/Qwen3.5-397B-A17B-FP8/shared_expert.down_proj" + ] }, { "type": "gemm", "args": { - "m": 4096, - "n": 6144, - "k": 1536, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 256, + "n": 4096, + "k": 512, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "Qwen/Qwen3.5-397B-A17B/o_proj", + "Qwen/Qwen3.5-397B-A17B/shared_expert.down_proj", + "amd/Kimi-K2.5-MXFP4/kv_b_proj", + "amd/Qwen3.5-397B-A17B-MXFP4/o_proj", + "amd/Qwen3.5-397B-A17B-MXFP4/shared_expert.down_proj", + "moonshotai/Kimi-K2.5/kv_b_proj", + "nvidia/DeepSeek-R1-0528-FP4-V2/kv_b_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/kv_b_proj", + "nvidia/Kimi-K2.5-NVFP4/kv_b_proj", + "nvidia/Kimi-K2.6-NVFP4/kv_b_proj", + "nvidia/Qwen3.5-397B-A17B-NVFP4/o_proj", + "nvidia/Qwen3.5-397B-A17B-NVFP4/shared_expert.down_proj" + ] }, { "type": "gemm", "args": { - "m": 4096, - "n": 8192, + "m": 256, + "n": 4096, "k": 512, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 32 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 1, + 32 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "amd/DeepSeek-R1-0528-MXFP4-Preview/kv_b_proj", + "amd/DeepSeek-R1-0528-MXFP4-v2/kv_b_proj", + "amd/DeepSeek-R1-0528-MXFP4/kv_b_proj", + "amd/Qwen3.5-397B-A17B-MXFP4-AttnFP8-V2/shared_expert.down_proj" + ] }, { "type": "gemm", "args": { - "m": 4096, - "n": 7168, - "k": 4096, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 256, + "n": 4096, + "k": 512, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "Qwen/Qwen3.5-397B-A17B-FP8/o_proj", + "Qwen/Qwen3.5-397B-A17B-FP8/shared_expert.down_proj", + "deepseek-ai/DeepSeek-R1-0528/kv_b_proj" + ] }, { "type": "gemm", "args": { - "m": 4, - "n": 6144, - "k": 1536, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 256, + "n": 4096, + "k": 1024, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "Qwen/Qwen3.5-397B-A17B/o_proj", + "Qwen/Qwen3.5-397B-A17B/shared_expert.down_proj", + "amd/Qwen3.5-397B-A17B-MXFP4/o_proj", + "amd/Qwen3.5-397B-A17B-MXFP4/shared_expert.down_proj", + "nvidia/Qwen3.5-397B-A17B-NVFP4/o_proj", + "nvidia/Qwen3.5-397B-A17B-NVFP4/shared_expert.down_proj" + ] }, { "type": "gemm", "args": { - "m": 4, - "n": 8192, - "k": 512, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 256, + "n": 4096, + "k": 1024, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "Qwen/Qwen3.5-397B-A17B-FP8/o_proj", + "Qwen/Qwen3.5-397B-A17B-FP8/shared_expert.down_proj" + ] }, { "type": "gemm", "args": { - "m": 4, - "n": 7168, - "k": 4096, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 256, + "n": 4096, + "k": 2048, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "Qwen/Qwen3.5-397B-A17B/o_proj", + "amd/GLM-5.1-MXFP4/q_b_proj", + "amd/GLM-5.2-MXFP4/q_b_proj", + "amd/Qwen3.5-397B-A17B-MXFP4/o_proj", + "nvidia/GLM-5-NVFP4/q_b_proj", + "nvidia/GLM-5.1-NVFP4/q_b_proj", + "nvidia/GLM-5.2-NVFP4/q_b_proj", + "nvidia/Qwen3.5-397B-A17B-NVFP4/o_proj" + ] }, { "type": "gemm", "args": { - "m": 8192, - "n": 6144, - "k": 1536, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 256, + "n": 4096, + "k": 2048, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "Qwen/Qwen3.5-397B-A17B-FP8/o_proj", + "zai-org/GLM-5-FP8/q_b_proj", + "zai-org/GLM-5.1-FP8/q_b_proj", + "zai-org/GLM-5.2-FP8/q_b_proj" + ] }, { "type": "gemm", "args": { - "m": 8192, - "n": 8192, - "k": 512, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 256, + "n": 4096, + "k": 2880, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/gpt-oss-120b-w-mxfp4-a-fp8/q_proj", + "openai/gpt-oss-120b/q_proj" + ] }, { "type": "gemm", "args": { - "m": 8192, - "n": 7168, + "m": 256, + "n": 4096, "k": 4096, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "Qwen/Qwen3.5-397B-A17B/o_proj", + "Qwen/Qwen3.5-397B-A17B/q_proj", + "amd/Qwen3.5-397B-A17B-MXFP4/o_proj", + "amd/Qwen3.5-397B-A17B-MXFP4/q_proj", + "nvidia/Qwen3.5-397B-A17B-NVFP4/o_proj", + "nvidia/Qwen3.5-397B-A17B-NVFP4/q_proj" + ] }, { "type": "gemm", "args": { - "m": 8, - "n": 6144, - "k": 1536, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 256, + "n": 4096, + "k": 4096, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "Qwen/Qwen3.5-397B-A17B-FP8/o_proj", + "Qwen/Qwen3.5-397B-A17B-FP8/q_proj" + ] }, { "type": "gemm", "args": { - "m": 8, - "n": 8192, - "k": 512, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 256, + "n": 4608, + "k": 7168, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/gate_up_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/gate_up_proj", + "nvidia/Kimi-K2.5-NVFP4/gate_up_proj" + ] }, { "type": "gemm", "args": { - "m": 8, - "n": 7168, - "k": 4096, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 256, + "n": 4608, + "k": 7168, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 32 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 1, + 32 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "amd/DeepSeek-R1-0528-MXFP4-Preview/gate_up_proj", + "amd/DeepSeek-R1-0528-MXFP4-v2/gate_up_proj", + "amd/DeepSeek-R1-0528-MXFP4/gate_up_proj", + "amd/Kimi-K2.5-MXFP4/gate_up_proj" + ] }, { "type": "gemm", "args": { - "m": 16384, - "n": 6144, - "k": 1536, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 256, + "n": 4608, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/gate_up_proj" + ] }, { "type": "gemm", "args": { - "m": 16384, - "n": 8192, - "k": 512, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 256, + "n": 6144, + "k": 1536, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/GLM-5.1-MXFP4/down_proj", + "amd/GLM-5.2-MXFP4/down_proj", + "amd/Kimi-K2.5-MXFP4/q_b_proj", + "amd/MiniMax-M3-MXFP4/down_proj", + "moonshotai/Kimi-K2.5/q_b_proj", + "nvidia/DeepSeek-R1-0528-FP4-V2/q_b_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/q_b_proj", + "nvidia/GLM-5.1-NVFP4/down_proj", + "nvidia/GLM-5.2-NVFP4/down_proj", + "nvidia/Kimi-K2.5-NVFP4/q_b_proj", + "nvidia/Kimi-K2.6-NVFP4/q_b_proj" + ] }, { "type": "gemm", "args": { - "m": 16384, - "n": 7168, - "k": 4096, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 256, + "n": 6144, + "k": 1536, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 32 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 1, + 32 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "amd/DeepSeek-R1-0528-MXFP4-Preview/q_b_proj", + "amd/DeepSeek-R1-0528-MXFP4-v2/q_b_proj", + "amd/DeepSeek-R1-0528-MXFP4/q_b_proj" + ] }, { "type": "gemm", "args": { - "m": 16, + "m": 256, "n": 6144, "k": 1536, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/q_b_proj", + "zai-org/GLM-5-FP8/down_proj", + "zai-org/GLM-5.1-FP8/down_proj", + "zai-org/GLM-5.2-FP8/down_proj" + ] }, { "type": "gemm", "args": { - "m": 16, - "n": 8192, - "k": 512, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 256, + "n": 6144, + "k": 2048, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "zai-org/GLM-5-FP8/o_proj", + "zai-org/GLM-5.1-FP8/o_proj", + "zai-org/GLM-5.2-FP8/o_proj" + ] }, { "type": "gemm", "args": { - "m": 16, - "n": 7168, - "k": 4096, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 256, + "n": 6144, + "k": 3072, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/GLM-5.1-MXFP4/down_proj", + "amd/GLM-5.2-MXFP4/down_proj", + "amd/MiniMax-M2.5-MXFP4/q_proj", + "amd/MiniMax-M3-MXFP4/down_proj", + "nvidia/GLM-5.1-NVFP4/down_proj", + "nvidia/GLM-5.2-NVFP4/down_proj", + "nvidia/MiniMax-M2.5-NVFP4/q_proj" + ] }, { "type": "gemm", "args": { - "m": 16, - "n": 4608, - "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 256, + "n": 6144, + "k": 3072, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/GLM-5-NVFP4/down_proj" + ] }, { "type": "gemm", "args": { - "m": 16, - "n": 7168, - "k": 4608, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 256, + "n": 6144, + "k": 3072, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "MiniMaxAI/MiniMax-M2.5/q_proj", + "zai-org/GLM-5-FP8/down_proj", + "zai-org/GLM-5.1-FP8/down_proj", + "zai-org/GLM-5.2-FP8/down_proj" + ] }, { "type": "gemm", "args": { - "m": 32768, + "m": 256, "n": 6144, - "k": 1536, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "k": 4096, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/GLM-5.1-MXFP4/o_proj", + "amd/GLM-5.2-MXFP4/o_proj", + "amd/MiniMax-M3-MXFP4/o_proj", + "nvidia/GLM-5-NVFP4/o_proj", + "nvidia/GLM-5.1-NVFP4/o_proj", + "nvidia/GLM-5.2-NVFP4/o_proj" + ] }, { "type": "gemm", "args": { - "m": 32768, - "n": 8192, - "k": 512, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 256, + "n": 6144, + "k": 4096, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "zai-org/GLM-5-FP8/o_proj", + "zai-org/GLM-5.1-FP8/o_proj", + "zai-org/GLM-5.2-FP8/o_proj" + ] }, { "type": "gemm", "args": { - "m": 32768, - "n": 7168, - "k": 4096, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 256, + "n": 6144, + "k": 6144, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/GLM-5.1-MXFP4/down_proj", + "amd/GLM-5.1-MXFP4/gate_up_proj", + "amd/GLM-5.2-MXFP4/down_proj", + "amd/GLM-5.2-MXFP4/gate_up_proj", + "amd/MiniMax-M3-MXFP4/gate_up_proj", + "nvidia/GLM-5.1-NVFP4/down_proj", + "nvidia/GLM-5.1-NVFP4/gate_up_proj", + "nvidia/GLM-5.2-NVFP4/down_proj", + "nvidia/GLM-5.2-NVFP4/gate_up_proj" + ] }, { "type": "gemm", "args": { - "m": 32, + "m": 256, "n": 6144, - "k": 1536, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "k": 6144, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/GLM-5-NVFP4/down_proj", + "nvidia/GLM-5-NVFP4/gate_up_proj" + ] }, { "type": "gemm", "args": { - "m": 32, - "n": 8192, - "k": 512, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 256, + "n": 6144, + "k": 6144, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "zai-org/GLM-5-FP8/down_proj", + "zai-org/GLM-5-FP8/gate_up_proj", + "zai-org/GLM-5.1-FP8/down_proj", + "zai-org/GLM-5.1-FP8/gate_up_proj", + "zai-org/GLM-5.2-FP8/down_proj", + "zai-org/GLM-5.2-FP8/gate_up_proj" + ] }, { "type": "gemm", "args": { - "m": 32, - "n": 7168, - "k": 4096, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 256, + "n": 6144, + "k": 8192, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/GLM-5.1-MXFP4/o_proj", + "amd/GLM-5.2-MXFP4/o_proj", + "amd/MiniMax-M3-MXFP4/o_proj", + "nvidia/GLM-5-NVFP4/o_proj", + "nvidia/GLM-5.1-NVFP4/o_proj", + "nvidia/GLM-5.2-NVFP4/o_proj" + ] }, { "type": "gemm", "args": { - "m": 32, - "n": 4608, - "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 256, + "n": 7168, + "k": 512, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/GLM-5.1-MXFP4/kv_b_proj", + "amd/GLM-5.2-MXFP4/kv_b_proj", + "amd/Kimi-K2.5-MXFP4/o_proj", + "moonshotai/Kimi-K2.5/o_proj", + "nvidia/GLM-5-NVFP4/kv_b_proj", + "nvidia/GLM-5.1-NVFP4/kv_b_proj", + "nvidia/GLM-5.2-NVFP4/kv_b_proj", + "nvidia/Kimi-K2.5-NVFP4/o_proj", + "nvidia/Kimi-K2.6-NVFP4/o_proj" + ] }, { "type": "gemm", "args": { - "m": 32, + "m": 256, "n": 7168, - "k": 4608, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "k": 512, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "zai-org/GLM-5-FP8/kv_b_proj", + "zai-org/GLM-5.1-FP8/kv_b_proj", + "zai-org/GLM-5.2-FP8/kv_b_proj" + ] }, { "type": "gemm", "args": { - "m": 65536, - "n": 6144, - "k": 1536, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 256, + "n": 7168, + "k": 576, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-NVFP4-v2/down_proj" + ] }, { "type": "gemm", "args": { - "m": 65536, - "n": 8192, - "k": 512, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 256, + "n": 7168, + "k": 768, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/down_proj", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w2" + ] }, { "type": "gemm", "args": { - "m": 65536, + "m": 256, "n": 7168, - "k": 4096, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "k": 1024, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/o_proj", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] }, { "type": "gemm", "args": { - "m": 64, - "n": 6144, - "k": 1536, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 256, + "n": 7168, + "k": 1152, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/down_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/down_proj", + "nvidia/Kimi-K2.5-NVFP4/down_proj" + ] }, { "type": "gemm", "args": { - "m": 64, - "n": 8192, - "k": 512, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 256, + "n": 7168, + "k": 2048, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/o_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/o_proj" + ] }, { "type": "gemm", "args": { - "m": 64, + "m": 256, "n": 7168, - "k": 4096, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "k": 2048, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 32 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 1, + 32 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "amd/DeepSeek-R1-0528-MXFP4-Preview/o_proj", + "amd/DeepSeek-R1-0528-MXFP4-v2/o_proj", + "amd/DeepSeek-R1-0528-MXFP4/o_proj" + ] }, { "type": "gemm", "args": { - "m": 64, - "n": 4608, - "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 256, + "n": 7168, + "k": 2048, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/o_proj", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] }, { "type": "gemm", "args": { - "m": 64, + "m": 256, "n": 7168, - "k": 4608, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "k": 2304, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/down_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/down_proj", + "nvidia/Kimi-K2.5-NVFP4/down_proj" + ] }, { "type": "gemm", "args": { - "m": 131072, - "n": 6144, - "k": 1536, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 256, + "n": 7168, + "k": 2304, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 32 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 1, + 32 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "amd/DeepSeek-R1-0528-MXFP4-Preview/down_proj", + "amd/DeepSeek-R1-0528-MXFP4-v2/down_proj", + "amd/DeepSeek-R1-0528-MXFP4/down_proj", + "amd/Kimi-K2.5-MXFP4/down_proj" + ] }, { "type": "gemm", "args": { - "m": 131072, - "n": 8192, - "k": 512, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 256, + "n": 7168, + "k": 2304, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/down_proj" + ] }, { "type": "gemm", "args": { - "m": 131072, + "m": 256, "n": 7168, "k": 4096, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/o_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/o_proj" + ] }, { "type": "gemm", "args": { - "m": 131072, - "n": 4608, - "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 256, + "n": 7168, + "k": 4096, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 32 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 1, + 32 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "amd/DeepSeek-R1-0528-MXFP4-Preview/o_proj", + "amd/DeepSeek-R1-0528-MXFP4-v2/o_proj", + "amd/DeepSeek-R1-0528-MXFP4/o_proj" + ] }, { "type": "gemm", "args": { - "m": 131072, + "m": 256, "n": 7168, "k": 4608, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/down_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/down_proj", + "nvidia/Kimi-K2.5-NVFP4/down_proj" + ] }, { "type": "gemm", "args": { - "m": 128, - "n": 1536, - "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 256, + "n": 7168, + "k": 4608, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 32 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 1, + 32 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "amd/DeepSeek-R1-0528-MXFP4-Preview/down_proj", + "amd/DeepSeek-R1-0528-MXFP4-v2/down_proj", + "amd/DeepSeek-R1-0528-MXFP4/down_proj", + "amd/Kimi-K2.5-MXFP4/down_proj" + ] }, { "type": "gemm", "args": { - "m": 128, - "n": 6144, - "k": 1536, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 256, + "n": 7168, + "k": 8192, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/Kimi-K2.5-MXFP4/o_proj", + "moonshotai/Kimi-K2.5/o_proj", + "nvidia/Kimi-K2.5-NVFP4/o_proj", + "nvidia/Kimi-K2.6-NVFP4/o_proj" + ] }, { "type": "gemm", "args": { - "m": 128, - "n": 576, - "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 256, + "n": 7168, + "k": 8192, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] }, { "type": "gemm", "args": { - "m": 128, - "n": 8192, - "k": 512, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 256, + "n": 7168, + "k": 16384, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/o_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/o_proj" + ] }, { "type": "gemm", "args": { - "m": 128, + "m": 256, "n": 7168, - "k": 4096, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "k": 16384, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 32 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 1, + 32 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "amd/DeepSeek-R1-0528-MXFP4-Preview/o_proj", + "amd/DeepSeek-R1-0528-MXFP4-v2/o_proj", + "amd/DeepSeek-R1-0528-MXFP4/o_proj" + ] }, { "type": "gemm", "args": { - "m": 128, - "n": 4608, - "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 256, + "n": 7168, + "k": 16384, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/o_proj", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] }, { "type": "gemm", "args": { - "m": 128, + "m": 256, "n": 7168, - "k": 4608, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "k": 16384, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] }, { "type": "gemm", "args": { - "m": 4096, - "n": 3072, - "k": 1536, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 256, + "n": 7168, + "k": 65536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] }, { "type": "gemm", "args": { - "m": 4096, - "n": 4096, + "m": 256, + "n": 8192, "k": 512, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/Kimi-K2.5-MXFP4/kv_b_proj", + "moonshotai/Kimi-K2.5/kv_b_proj", + "nvidia/DeepSeek-R1-0528-FP4-V2/kv_b_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/kv_b_proj", + "nvidia/Kimi-K2.5-NVFP4/kv_b_proj", + "nvidia/Kimi-K2.6-NVFP4/kv_b_proj" + ] }, { "type": "gemm", "args": { - "m": 4096, - "n": 7168, - "k": 2048, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 256, + "n": 8192, + "k": 512, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 32 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 1, + 32 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "amd/DeepSeek-R1-0528-MXFP4-Preview/kv_b_proj", + "amd/DeepSeek-R1-0528-MXFP4-v2/kv_b_proj", + "amd/DeepSeek-R1-0528-MXFP4/kv_b_proj" + ] }, { "type": "gemm", "args": { - "m": 4096, - "n": 2304, - "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 256, + "n": 8192, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_b" + ] }, { "type": "gemm", "args": { - "m": 4096, - "n": 7168, - "k": 2304, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 256, + "n": 8192, + "k": 2048, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/GLM-5.1-MXFP4/q_b_proj", + "amd/GLM-5.2-MXFP4/q_b_proj", + "nvidia/GLM-5-NVFP4/q_b_proj", + "nvidia/GLM-5.1-NVFP4/q_b_proj", + "nvidia/GLM-5.2-NVFP4/q_b_proj" + ] }, { "type": "gemm", "args": { - "m": 128, - "n": 3072, + "m": 256, + "n": 12288, "k": 1536, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/Kimi-K2.5-MXFP4/q_b_proj", + "moonshotai/Kimi-K2.5/q_b_proj", + "nvidia/DeepSeek-R1-0528-FP4-V2/q_b_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/q_b_proj", + "nvidia/Kimi-K2.5-NVFP4/q_b_proj", + "nvidia/Kimi-K2.6-NVFP4/q_b_proj" + ] }, { "type": "gemm", "args": { - "m": 128, - "n": 4096, + "m": 256, + "n": 14336, "k": 512, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/GLM-5.1-MXFP4/kv_b_proj", + "amd/GLM-5.2-MXFP4/kv_b_proj", + "nvidia/GLM-5-NVFP4/kv_b_proj", + "nvidia/GLM-5.1-NVFP4/kv_b_proj", + "nvidia/GLM-5.2-NVFP4/kv_b_proj" + ] }, { "type": "gemm", "args": { - "m": 128, - "n": 7168, - "k": 2048, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 256, + "n": 15360, + "k": 1024, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] }, { "type": "gemm", "args": { - "m": 128, - "n": 2304, - "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 256, + "n": 16384, + "k": 512, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/Kimi-K2.5-MXFP4/kv_b_proj", + "moonshotai/Kimi-K2.5/kv_b_proj", + "nvidia/DeepSeek-R1-0528-FP4-V2/kv_b_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/kv_b_proj", + "nvidia/Kimi-K2.5-NVFP4/kv_b_proj", + "nvidia/Kimi-K2.6-NVFP4/kv_b_proj" + ] }, { "type": "gemm", "args": { - "m": 128, - "n": 7168, - "k": 2304, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 256, + "n": 16384, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_b" + ] }, { "type": "gemm", "args": { - "m": 262144, - "n": 6144, + "m": 256, + "n": 24576, "k": 1536, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/q_b_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/q_b_proj" + ] }, { "type": "gemm", "args": { - "m": 262144, - "n": 8192, - "k": 512, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 256, + "n": 24576, + "k": 1536, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 32 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 1, + 32 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "amd/DeepSeek-R1-0528-MXFP4-Preview/q_b_proj", + "amd/DeepSeek-R1-0528-MXFP4-v2/q_b_proj", + "amd/DeepSeek-R1-0528-MXFP4/q_b_proj" + ] }, { "type": "gemm", "args": { - "m": 262144, - "n": 7168, - "k": 4096, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 256, + "n": 24576, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/q_b_proj" + ] }, { "type": "gemm", "args": { - "m": 262144, - "n": 4608, - "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 256, + "n": 30720, + "k": 1024, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] }, { "type": "gemm", "args": { - "m": 262144, - "n": 7168, - "k": 4608, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 256, + "n": 32768, + "k": 512, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/kv_b_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/kv_b_proj" + ] }, { "type": "gemm", "args": { - "m": 524288, - "n": 6144, - "k": 1536, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 256, + "n": 32768, + "k": 512, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 32 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 1, + 32 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "amd/DeepSeek-R1-0528-MXFP4-Preview/kv_b_proj", + "amd/DeepSeek-R1-0528-MXFP4-v2/kv_b_proj", + "amd/DeepSeek-R1-0528-MXFP4/kv_b_proj" + ] }, { "type": "gemm", "args": { - "m": 524288, - "n": 8192, + "m": 256, + "n": 32768, "k": 512, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/kv_b_proj" + ] }, { "type": "gemm", "args": { - "m": 524288, - "n": 7168, - "k": 4096, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 256, + "n": 65536, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_b" + ] }, { "type": "gemm", "args": { - "m": 524288, - "n": 4608, - "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 256, + "n": 122880, + "k": 1024, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] }, { "type": "gemm", "args": { - "m": 524288, - "n": 7168, - "k": 4608, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 264, + "n": 576, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/kv_a_proj_with_mqa" + ] }, { "type": "gemm", "args": { - "m": 1048576, - "n": 6144, - "k": 1536, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 264, + "n": 1536, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/gate_up_proj", + "deepseek-ai/DeepSeek-R1-0528/q_a_proj", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w3" + ] }, { "type": "gemm", "args": { - "m": 1048576, - "n": 8192, - "k": 512, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 264, + "n": 2304, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/gate_up_proj" + ] }, { "type": "gemm", "args": { - "m": 1048576, + "m": 264, "n": 7168, - "k": 4096, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "k": 2304, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/down_proj" + ] }, { "type": "gemm", "args": { - "m": 1, - "n": 6144, - "k": 1536, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 264, + "n": 7168, + "k": 16384, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/o_proj", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] }, { "type": "gemm", "args": { - "m": 1, - "n": 8192, - "k": 512, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 264, + "n": 24576, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/q_b_proj" + ] }, { "type": "gemm", "args": { - "m": 1, - "n": 7168, - "k": 4096, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 264, + "n": 32768, + "k": 512, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/kv_b_proj" + ] }, { "type": "gemm", "args": { - "m": 2, - "n": 6144, - "k": 1536, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 268, + "n": 576, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/Kimi-K2.5-MXFP4/kv_a_proj_with_mqa", + "moonshotai/Kimi-K2.5/kv_a_proj_with_mqa", + "nvidia/DeepSeek-R1-0528-FP4-V2/kv_a_proj_with_mqa", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/kv_a_proj_with_mqa", + "nvidia/Kimi-K2.5-NVFP4/kv_a_proj_with_mqa", + "nvidia/Kimi-K2.6-NVFP4/kv_a_proj_with_mqa" + ] }, { "type": "gemm", "args": { - "m": 2, - "n": 8192, - "k": 512, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 268, + "n": 576, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/kv_a_proj_with_mqa" + ] }, { "type": "gemm", "args": { - "m": 2, - "n": 7168, - "k": 4096, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 268, + "n": 1152, + "k": 7168, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-NVFP4-v2/gate_up_proj" + ] }, { "type": "gemm", "args": { - "m": 196608, + "m": 268, "n": 1536, "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 196608, - "n": 6144, - "k": 1536, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/Kimi-K2.5-MXFP4/q_a_proj", + "moonshotai/Kimi-K2.5/q_a_proj", + "nvidia/DeepSeek-R1-0528-FP4-V2/q_a_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/q_a_proj", + "nvidia/Kimi-K2.5-NVFP4/q_a_proj", + "nvidia/Kimi-K2.6-NVFP4/q_a_proj" + ] }, { "type": "gemm", "args": { - "m": 196608, - "n": 576, + "m": 268, + "n": 1536, "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/gate_up_proj", + "deepseek-ai/DeepSeek-R1-0528/q_a_proj", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w3" + ] }, { "type": "gemm", "args": { - "m": 196608, - "n": 8192, - "k": 512, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 268, + "n": 2304, + "k": 7168, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/gate_up_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/gate_up_proj", + "nvidia/Kimi-K2.5-NVFP4/gate_up_proj" + ] }, { "type": "gemm", "args": { - "m": 196608, - "n": 7168, - "k": 4096, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 268, + "n": 4608, + "k": 7168, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/gate_up_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/gate_up_proj", + "nvidia/Kimi-K2.5-NVFP4/gate_up_proj" + ] }, { "type": "gemm", "args": { - "m": 196608, + "m": 268, "n": 4608, "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/gate_up_proj" + ] }, { "type": "gemm", "args": { - "m": 196608, + "m": 268, "n": 7168, - "k": 4608, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "k": 1152, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/down_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/down_proj", + "nvidia/Kimi-K2.5-NVFP4/down_proj" + ] }, { "type": "gemm", "args": { - "m": 24, - "n": 1536, - "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 268, + "n": 7168, + "k": 2304, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/down_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/down_proj", + "nvidia/Kimi-K2.5-NVFP4/down_proj" + ] }, { "type": "gemm", "args": { - "m": 24, - "n": 6144, - "k": 1536, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 268, + "n": 7168, + "k": 2304, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/down_proj" + ] }, { "type": "gemm", "args": { - "m": 24, - "n": 576, - "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 268, + "n": 7168, + "k": 4608, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/down_proj" + ] }, { "type": "gemm", "args": { - "m": 24, - "n": 8192, - "k": 512, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 268, + "n": 7168, + "k": 8192, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/Kimi-K2.5-MXFP4/o_proj", + "moonshotai/Kimi-K2.5/o_proj", + "nvidia/Kimi-K2.5-NVFP4/o_proj", + "nvidia/Kimi-K2.6-NVFP4/o_proj" + ] }, { "type": "gemm", "args": { - "m": 24, + "m": 268, "n": 7168, - "k": 4096, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "k": 16384, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/o_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/o_proj" + ] }, { "type": "gemm", "args": { - "m": 24, - "n": 4608, - "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 268, + "n": 7168, + "k": 16384, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/o_proj", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] }, { "type": "gemm", "args": { - "m": 24, - "n": 7168, - "k": 4608, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 268, + "n": 12288, + "k": 1536, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/Kimi-K2.5-MXFP4/q_b_proj", + "moonshotai/Kimi-K2.5/q_b_proj", + "nvidia/DeepSeek-R1-0528-FP4-V2/q_b_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/q_b_proj", + "nvidia/Kimi-K2.5-NVFP4/q_b_proj", + "nvidia/Kimi-K2.6-NVFP4/q_b_proj" + ] }, { "type": "gemm", "args": { - "m": 393216, - "n": 1536, - "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 268, + "n": 16384, + "k": 512, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/Kimi-K2.5-MXFP4/kv_b_proj", + "moonshotai/Kimi-K2.5/kv_b_proj", + "nvidia/DeepSeek-R1-0528-FP4-V2/kv_b_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/kv_b_proj", + "nvidia/Kimi-K2.5-NVFP4/kv_b_proj", + "nvidia/Kimi-K2.6-NVFP4/kv_b_proj" + ] }, { "type": "gemm", "args": { - "m": 393216, - "n": 6144, + "m": 268, + "n": 24576, "k": 1536, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/q_b_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/q_b_proj" + ] }, { "type": "gemm", "args": { - "m": 393216, - "n": 576, - "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 268, + "n": 24576, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/q_b_proj" + ] }, { "type": "gemm", "args": { - "m": 393216, - "n": 8192, + "m": 268, + "n": 32768, "k": 512, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/kv_b_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/kv_b_proj" + ] }, { "type": "gemm", "args": { - "m": 393216, - "n": 7168, - "k": 4096, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 268, + "n": 32768, + "k": 512, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/kv_b_proj" + ] }, { "type": "gemm", "args": { - "m": 393216, - "n": 4608, + "m": 271, + "n": 576, "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/Kimi-K2.5-MXFP4/kv_a_proj_with_mqa", + "moonshotai/Kimi-K2.5/kv_a_proj_with_mqa", + "nvidia/DeepSeek-R1-0528-FP4-V2/kv_a_proj_with_mqa", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/kv_a_proj_with_mqa", + "nvidia/Kimi-K2.5-NVFP4/kv_a_proj_with_mqa", + "nvidia/Kimi-K2.6-NVFP4/kv_a_proj_with_mqa" + ] }, { "type": "gemm", "args": { - "m": 393216, - "n": 7168, - "k": 4608, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 271, + "n": 576, + "k": 7168, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 32 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 1, + 32 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "amd/DeepSeek-R1-0528-MXFP4-Preview/kv_a_proj_with_mqa", + "amd/DeepSeek-R1-0528-MXFP4-v2/kv_a_proj_with_mqa", + "amd/DeepSeek-R1-0528-MXFP4/kv_a_proj_with_mqa" + ] }, { "type": "gemm", "args": { - "m": 48, + "m": 271, "n": 1536, "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 48, - "n": 6144, - "k": 1536, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/Kimi-K2.5-MXFP4/q_a_proj", + "moonshotai/Kimi-K2.5/q_a_proj", + "nvidia/DeepSeek-R1-0528-FP4-V2/q_a_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/q_a_proj", + "nvidia/Kimi-K2.5-NVFP4/q_a_proj", + "nvidia/Kimi-K2.6-NVFP4/q_a_proj" + ] }, { "type": "gemm", "args": { - "m": 48, - "n": 576, + "m": 271, + "n": 1536, "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 32 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 1, + 32 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "amd/DeepSeek-R1-0528-MXFP4-Preview/q_a_proj", + "amd/DeepSeek-R1-0528-MXFP4-v2/q_a_proj", + "amd/DeepSeek-R1-0528-MXFP4/q_a_proj" + ] }, { "type": "gemm", "args": { - "m": 48, - "n": 8192, - "k": 512, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 271, + "n": 1536, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w3" + ] }, { "type": "gemm", "args": { - "m": 48, + "m": 271, "n": 7168, - "k": 4096, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 48, - "n": 4608, - "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "k": 9216, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/down_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/down_proj", + "nvidia/Kimi-K2.5-NVFP4/down_proj" + ] }, { "type": "gemm", "args": { - "m": 48, + "m": 271, "n": 7168, - "k": 4608, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "k": 16384, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/o_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/o_proj" + ] }, { "type": "gemm", "args": { - "m": 2097152, - "n": 6144, - "k": 1536, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 271, + "n": 7168, + "k": 16384, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 32 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 1, + 32 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "amd/DeepSeek-R1-0528-MXFP4-Preview/o_proj", + "amd/DeepSeek-R1-0528-MXFP4-v2/o_proj", + "amd/DeepSeek-R1-0528-MXFP4/o_proj" + ] }, { "type": "gemm", "args": { - "m": 2097152, - "n": 8192, - "k": 512, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 271, + "n": 7168, + "k": 16384, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] }, { "type": "gemm", "args": { - "m": 2097152, - "n": 7168, - "k": 4096, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 271, + "n": 9216, + "k": 7168, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/gate_up_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/gate_up_proj", + "nvidia/Kimi-K2.5-NVFP4/gate_up_proj" + ] }, { "type": "gemm", "args": { - "m": 2097152, - "n": 4608, + "m": 271, + "n": 9216, "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 32 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 1, + 32 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "amd/DeepSeek-R1-0528-MXFP4-Preview/gate_up_proj", + "amd/DeepSeek-R1-0528-MXFP4-v2/gate_up_proj", + "amd/DeepSeek-R1-0528-MXFP4/gate_up_proj", + "amd/Kimi-K2.5-MXFP4/gate_up_proj" + ] }, { "type": "gemm", "args": { - "m": 2097152, - "n": 7168, - "k": 4608, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 271, + "n": 24576, + "k": 1536, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/q_b_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/q_b_proj" + ] }, { "type": "gemm", "args": { - "m": 256, - "n": 6144, + "m": 271, + "n": 24576, "k": 1536, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 32 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 1, + 32 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "amd/DeepSeek-R1-0528-MXFP4-Preview/q_b_proj", + "amd/DeepSeek-R1-0528-MXFP4-v2/q_b_proj", + "amd/DeepSeek-R1-0528-MXFP4/q_b_proj" + ] }, { "type": "gemm", "args": { - "m": 256, - "n": 8192, + "m": 271, + "n": 32768, "k": 512, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/kv_b_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/kv_b_proj" + ] }, { "type": "gemm", "args": { - "m": 256, - "n": 7168, - "k": 4096, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 271, + "n": 32768, + "k": 512, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 32 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 1, + 32 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "amd/DeepSeek-R1-0528-MXFP4-Preview/kv_b_proj", + "amd/DeepSeek-R1-0528-MXFP4-v2/kv_b_proj", + "amd/DeepSeek-R1-0528-MXFP4/kv_b_proj" + ] }, { "type": "gemm", "args": { - "m": 256, - "n": 4608, + "m": 272, + "n": 576, "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/kv_a_proj_with_mqa" + ] }, { "type": "gemm", "args": { - "m": 256, - "n": 7168, - "k": 4608, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 272, + "n": 1536, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/gate_up_proj", + "deepseek-ai/DeepSeek-R1-0528/q_a_proj", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w3" + ] }, { "type": "gemm", "args": { - "m": 98304, - "n": 3072, - "k": 1536, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 272, + "n": 2304, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/gate_up_proj" + ] }, { "type": "gemm", "args": { - "m": 98304, - "n": 4096, - "k": 512, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 272, + "n": 7168, + "k": 2304, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/down_proj" + ] }, { "type": "gemm", "args": { - "m": 98304, + "m": 272, "n": 7168, - "k": 2048, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "k": 16384, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/o_proj", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] }, { "type": "gemm", "args": { - "m": 98304, - "n": 2304, + "m": 272, + "n": 24576, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/q_b_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 272, + "n": 32768, + "k": 512, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/kv_b_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 274, + "n": 576, "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/Kimi-K2.5-MXFP4/kv_a_proj_with_mqa", + "moonshotai/Kimi-K2.5/kv_a_proj_with_mqa", + "nvidia/DeepSeek-R1-0528-FP4-V2/kv_a_proj_with_mqa", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/kv_a_proj_with_mqa", + "nvidia/Kimi-K2.5-NVFP4/kv_a_proj_with_mqa", + "nvidia/Kimi-K2.6-NVFP4/kv_a_proj_with_mqa" + ] }, { "type": "gemm", "args": { - "m": 98304, - "n": 7168, - "k": 2304, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 274, + "n": 1536, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/Kimi-K2.5-MXFP4/q_a_proj", + "moonshotai/Kimi-K2.5/q_a_proj", + "nvidia/DeepSeek-R1-0528-FP4-V2/q_a_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/q_a_proj", + "nvidia/Kimi-K2.5-NVFP4/q_a_proj", + "nvidia/Kimi-K2.6-NVFP4/q_a_proj" + ] }, { "type": "gemm", "args": { - "m": 524288, - "n": 3072, - "k": 1536, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 274, + "n": 2304, + "k": 7168, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/gate_up_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/gate_up_proj", + "nvidia/Kimi-K2.5-NVFP4/gate_up_proj" + ] }, { "type": "gemm", "args": { - "m": 524288, - "n": 4096, - "k": 512, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 274, + "n": 4608, + "k": 7168, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/gate_up_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/gate_up_proj", + "nvidia/Kimi-K2.5-NVFP4/gate_up_proj" + ] }, { "type": "gemm", "args": { - "m": 524288, + "m": 274, "n": 7168, - "k": 2048, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "k": 2304, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/down_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/down_proj", + "nvidia/Kimi-K2.5-NVFP4/down_proj" + ] }, { "type": "gemm", "args": { - "m": 1048576, - "n": 3072, - "k": 1536, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 274, + "n": 7168, + "k": 4608, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/down_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/down_proj", + "nvidia/Kimi-K2.5-NVFP4/down_proj" + ] }, { "type": "gemm", "args": { - "m": 1048576, - "n": 4096, - "k": 512, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 274, + "n": 7168, + "k": 4608, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 32 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 1, + 32 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "amd/DeepSeek-R1-0528-MXFP4-Preview/down_proj", + "amd/DeepSeek-R1-0528-MXFP4-v2/down_proj", + "amd/DeepSeek-R1-0528-MXFP4/down_proj", + "amd/Kimi-K2.5-MXFP4/down_proj" + ] }, { "type": "gemm", "args": { - "m": 1048576, + "m": 274, "n": 7168, - "k": 2048, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "k": 16384, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/o_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/o_proj" + ] }, { "type": "gemm", "args": { - "m": 2097152, - "n": 3072, + "m": 274, + "n": 24576, "k": 1536, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/q_b_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/q_b_proj" + ] }, { "type": "gemm", "args": { - "m": 2097152, - "n": 4096, + "m": 274, + "n": 32768, "k": 512, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/kv_b_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/kv_b_proj" + ] }, { "type": "gemm", "args": { - "m": 2097152, - "n": 7168, - "k": 2048, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 277, + "n": 576, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/Kimi-K2.5-MXFP4/kv_a_proj_with_mqa", + "moonshotai/Kimi-K2.5/kv_a_proj_with_mqa", + "nvidia/DeepSeek-R1-0528-FP4-V2/kv_a_proj_with_mqa", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/kv_a_proj_with_mqa", + "nvidia/Kimi-K2.5-NVFP4/kv_a_proj_with_mqa", + "nvidia/Kimi-K2.6-NVFP4/kv_a_proj_with_mqa" + ] }, { "type": "gemm", "args": { - "m": 256, - "n": 3072, - "k": 1536, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 277, + "n": 1536, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/Kimi-K2.5-MXFP4/q_a_proj", + "moonshotai/Kimi-K2.5/q_a_proj", + "nvidia/DeepSeek-R1-0528-FP4-V2/q_a_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/q_a_proj", + "nvidia/Kimi-K2.5-NVFP4/q_a_proj", + "nvidia/Kimi-K2.6-NVFP4/q_a_proj" + ] }, { "type": "gemm", "args": { - "m": 256, - "n": 4096, - "k": 512, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 277, + "n": 2304, + "k": 7168, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/gate_up_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/gate_up_proj", + "nvidia/Kimi-K2.5-NVFP4/gate_up_proj" + ] }, { "type": "gemm", "args": { - "m": 256, + "m": 277, "n": 7168, - "k": 2048, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "k": 2304, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/down_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/down_proj", + "nvidia/Kimi-K2.5-NVFP4/down_proj" + ] }, { "type": "gemm", "args": { - "m": 4194304, - "n": 3072, + "m": 277, + "n": 7168, + "k": 16384, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/o_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/o_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 277, + "n": 24576, "k": 1536, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/q_b_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/q_b_proj" + ] }, { "type": "gemm", "args": { - "m": 4194304, - "n": 4096, + "m": 277, + "n": 32768, "k": 512, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/kv_b_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/kv_b_proj" + ] }, { "type": "gemm", "args": { - "m": 4194304, - "n": 7168, - "k": 2048, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 281, + "n": 576, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/Kimi-K2.5-MXFP4/kv_a_proj_with_mqa", + "moonshotai/Kimi-K2.5/kv_a_proj_with_mqa", + "nvidia/DeepSeek-R1-0528-FP4-V2/kv_a_proj_with_mqa", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/kv_a_proj_with_mqa", + "nvidia/Kimi-K2.5-NVFP4/kv_a_proj_with_mqa", + "nvidia/Kimi-K2.6-NVFP4/kv_a_proj_with_mqa" + ] }, { "type": "gemm", "args": { - "m": 4194304, - "n": 2304, + "m": 281, + "n": 576, "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/kv_a_proj_with_mqa" + ] }, { "type": "gemm", "args": { - "m": 4194304, - "n": 7168, - "k": 2304, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 281, + "n": 1152, + "k": 7168, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-NVFP4-v2/gate_up_proj" + ] }, { "type": "gemm", "args": { - "m": 512, - "n": 3072, - "k": 1536, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 281, + "n": 1152, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/gate_up_proj" + ] }, { "type": "gemm", "args": { - "m": 512, - "n": 4096, - "k": 512, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 281, + "n": 1536, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/Kimi-K2.5-MXFP4/q_a_proj", + "moonshotai/Kimi-K2.5/q_a_proj", + "nvidia/DeepSeek-R1-0528-FP4-V2/q_a_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/q_a_proj", + "nvidia/Kimi-K2.5-NVFP4/q_a_proj", + "nvidia/Kimi-K2.6-NVFP4/q_a_proj" + ] }, { "type": "gemm", "args": { - "m": 512, - "n": 7168, - "k": 2048, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 281, + "n": 1536, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/gate_up_proj", + "deepseek-ai/DeepSeek-R1-0528/q_a_proj", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w3" + ] }, { "type": "gemm", "args": { - "m": 1024, - "n": 8192, - "k": 1536, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 281, + "n": 7168, + "k": 1152, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/down_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/down_proj", + "nvidia/Kimi-K2.5-NVFP4/down_proj" + ] }, { "type": "gemm", "args": { - "m": 1024, - "n": 1088, - "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 281, + "n": 7168, + "k": 1152, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/down_proj" + ] }, { "type": "gemm", "args": { - "m": 1024, - "n": 15360, - "k": 1024, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 281, + "n": 7168, + "k": 16384, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/o_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/o_proj" + ] }, { "type": "gemm", "args": { - "m": 1024, + "m": 281, "n": 7168, - "k": 8192, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "k": 16384, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/o_proj", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] }, { "type": "gemm", "args": { - "m": 1, - "n": 8192, + "m": 281, + "n": 24576, "k": 1536, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/q_b_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/q_b_proj" + ] }, { "type": "gemm", "args": { - "m": 1, - "n": 1088, - "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 281, + "n": 24576, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/q_b_proj" + ] }, { "type": "gemm", "args": { - "m": 1, - "n": 15360, - "k": 1024, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 281, + "n": 32768, + "k": 512, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/kv_b_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/kv_b_proj" + ] }, { "type": "gemm", "args": { - "m": 1, - "n": 7168, - "k": 8192, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 281, + "n": 32768, + "k": 512, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/kv_b_proj" + ] }, { "type": "gemm", "args": { - "m": 2048, - "n": 8192, - "k": 1536, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 282, + "n": 576, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/kv_a_proj_with_mqa" + ] }, { "type": "gemm", "args": { - "m": 2048, - "n": 1088, + "m": 282, + "n": 1536, "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/gate_up_proj", + "deepseek-ai/DeepSeek-R1-0528/q_a_proj", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w3" + ] }, { "type": "gemm", "args": { - "m": 2048, - "n": 15360, - "k": 1024, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 282, + "n": 1536, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w3" + ] }, { "type": "gemm", "args": { - "m": 2048, - "n": 7168, - "k": 8192, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 282, + "n": 4608, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/gate_up_proj" + ] }, { "type": "gemm", "args": { - "m": 2, - "n": 8192, - "k": 1536, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 282, + "n": 7168, + "k": 4608, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/down_proj" + ] }, { "type": "gemm", "args": { - "m": 2, - "n": 1088, - "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 282, + "n": 7168, + "k": 16384, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/o_proj", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] }, { "type": "gemm", "args": { - "m": 2, - "n": 15360, - "k": 1024, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 282, + "n": 24576, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/q_b_proj" + ] }, { "type": "gemm", "args": { - "m": 2, - "n": 7168, - "k": 8192, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 282, + "n": 32768, + "k": 512, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/kv_b_proj" + ] }, { "type": "gemm", "args": { - "m": 4096, - "n": 8192, - "k": 1536, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 294, + "n": 576, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/Kimi-K2.5-MXFP4/kv_a_proj_with_mqa", + "moonshotai/Kimi-K2.5/kv_a_proj_with_mqa", + "nvidia/DeepSeek-R1-0528-FP4-V2/kv_a_proj_with_mqa", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/kv_a_proj_with_mqa", + "nvidia/Kimi-K2.5-NVFP4/kv_a_proj_with_mqa", + "nvidia/Kimi-K2.6-NVFP4/kv_a_proj_with_mqa" + ] }, { "type": "gemm", "args": { - "m": 4096, - "n": 1088, + "m": 294, + "n": 576, "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 32 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 1, + 32 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "amd/DeepSeek-R1-0528-MXFP4-Preview/kv_a_proj_with_mqa", + "amd/DeepSeek-R1-0528-MXFP4-v2/kv_a_proj_with_mqa", + "amd/DeepSeek-R1-0528-MXFP4/kv_a_proj_with_mqa" + ] }, { "type": "gemm", "args": { - "m": 4096, - "n": 15360, - "k": 1024, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 294, + "n": 1536, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/Kimi-K2.5-MXFP4/q_a_proj", + "moonshotai/Kimi-K2.5/q_a_proj", + "nvidia/DeepSeek-R1-0528-FP4-V2/q_a_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/q_a_proj", + "nvidia/Kimi-K2.5-NVFP4/q_a_proj", + "nvidia/Kimi-K2.6-NVFP4/q_a_proj" + ] }, { "type": "gemm", "args": { - "m": 4096, - "n": 7168, - "k": 8192, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 294, + "n": 1536, + "k": 7168, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 32 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 1, + 32 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "amd/DeepSeek-R1-0528-MXFP4-Preview/q_a_proj", + "amd/DeepSeek-R1-0528-MXFP4-v2/q_a_proj", + "amd/DeepSeek-R1-0528-MXFP4/q_a_proj" + ] }, { "type": "gemm", "args": { - "m": 4, - "n": 8192, - "k": 1536, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 294, + "n": 1536, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w3" + ] }, { "type": "gemm", "args": { - "m": 4, - "n": 1088, - "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 294, + "n": 7168, + "k": 9216, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/down_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/down_proj", + "nvidia/Kimi-K2.5-NVFP4/down_proj" + ] }, { "type": "gemm", "args": { - "m": 4, - "n": 15360, - "k": 1024, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 294, + "n": 7168, + "k": 16384, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/o_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/o_proj" + ] }, { "type": "gemm", "args": { - "m": 4, + "m": 294, "n": 7168, - "k": 8192, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "k": 16384, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 32 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 1, + 32 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "amd/DeepSeek-R1-0528-MXFP4-Preview/o_proj", + "amd/DeepSeek-R1-0528-MXFP4-v2/o_proj", + "amd/DeepSeek-R1-0528-MXFP4/o_proj" + ] }, { "type": "gemm", "args": { - "m": 8192, - "n": 8192, - "k": 1536, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 294, + "n": 7168, + "k": 16384, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] }, { "type": "gemm", "args": { - "m": 8192, - "n": 1088, + "m": 294, + "n": 9216, "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/gate_up_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/gate_up_proj", + "nvidia/Kimi-K2.5-NVFP4/gate_up_proj" + ] }, { "type": "gemm", "args": { - "m": 8192, - "n": 15360, - "k": 1024, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 294, + "n": 9216, + "k": 7168, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 32 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 1, + 32 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "amd/DeepSeek-R1-0528-MXFP4-Preview/gate_up_proj", + "amd/DeepSeek-R1-0528-MXFP4-v2/gate_up_proj", + "amd/DeepSeek-R1-0528-MXFP4/gate_up_proj", + "amd/Kimi-K2.5-MXFP4/gate_up_proj" + ] }, { "type": "gemm", "args": { - "m": 8192, - "n": 7168, - "k": 8192, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 294, + "n": 24576, + "k": 1536, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/q_b_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/q_b_proj" + ] }, { "type": "gemm", "args": { - "m": 8, - "n": 8192, + "m": 294, + "n": 24576, "k": 1536, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 32 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 1, + 32 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "amd/DeepSeek-R1-0528-MXFP4-Preview/q_b_proj", + "amd/DeepSeek-R1-0528-MXFP4-v2/q_b_proj", + "amd/DeepSeek-R1-0528-MXFP4/q_b_proj" + ] }, { "type": "gemm", "args": { - "m": 8, - "n": 1088, - "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 294, + "n": 32768, + "k": 512, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/kv_b_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/kv_b_proj" + ] }, { "type": "gemm", "args": { - "m": 8, - "n": 15360, - "k": 1024, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 294, + "n": 32768, + "k": 512, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 32 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 1, + 32 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "amd/DeepSeek-R1-0528-MXFP4-Preview/kv_b_proj", + "amd/DeepSeek-R1-0528-MXFP4-v2/kv_b_proj", + "amd/DeepSeek-R1-0528-MXFP4/kv_b_proj" + ] }, { "type": "gemm", "args": { - "m": 8, - "n": 7168, - "k": 8192, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 296, + "n": 576, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/kv_a_proj_with_mqa" + ] }, { "type": "gemm", "args": { - "m": 16384, - "n": 8192, - "k": 1536, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 296, + "n": 1536, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/gate_up_proj", + "deepseek-ai/DeepSeek-R1-0528/q_a_proj", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w3" + ] }, { "type": "gemm", "args": { - "m": 16384, - "n": 1088, + "m": 296, + "n": 1536, "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w3" + ] }, { "type": "gemm", "args": { - "m": 16384, - "n": 15360, - "k": 1024, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 296, + "n": 2304, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/gate_up_proj" + ] }, { "type": "gemm", "args": { - "m": 16384, + "m": 296, "n": 7168, - "k": 8192, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "k": 2304, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/down_proj" + ] }, { "type": "gemm", "args": { - "m": 16, - "n": 8192, - "k": 1536, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 296, + "n": 7168, + "k": 16384, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/o_proj", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] }, { "type": "gemm", "args": { - "m": 16, - "n": 1088, - "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 296, + "n": 24576, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/q_b_proj" + ] }, { "type": "gemm", "args": { - "m": 16, - "n": 15360, - "k": 1024, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 296, + "n": 32768, + "k": 512, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/kv_b_proj" + ] }, { "type": "gemm", "args": { - "m": 16, - "n": 7168, - "k": 8192, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 299, + "n": 576, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/Kimi-K2.5-MXFP4/kv_a_proj_with_mqa", + "moonshotai/Kimi-K2.5/kv_a_proj_with_mqa", + "nvidia/DeepSeek-R1-0528-FP4-V2/kv_a_proj_with_mqa", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/kv_a_proj_with_mqa", + "nvidia/Kimi-K2.5-NVFP4/kv_a_proj_with_mqa", + "nvidia/Kimi-K2.6-NVFP4/kv_a_proj_with_mqa" + ] }, { "type": "gemm", "args": { - "m": 32768, - "n": 8192, - "k": 1536, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 299, + "n": 1536, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/Kimi-K2.5-MXFP4/q_a_proj", + "moonshotai/Kimi-K2.5/q_a_proj", + "nvidia/DeepSeek-R1-0528-FP4-V2/q_a_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/q_a_proj", + "nvidia/Kimi-K2.5-NVFP4/q_a_proj", + "nvidia/Kimi-K2.6-NVFP4/q_a_proj" + ] }, { "type": "gemm", "args": { - "m": 32768, - "n": 1088, + "m": 299, + "n": 2304, "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/gate_up_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/gate_up_proj", + "nvidia/Kimi-K2.5-NVFP4/gate_up_proj" + ] }, { "type": "gemm", "args": { - "m": 32768, - "n": 15360, - "k": 1024, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 299, + "n": 7168, + "k": 2304, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/down_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/down_proj", + "nvidia/Kimi-K2.5-NVFP4/down_proj" + ] }, { "type": "gemm", "args": { - "m": 32768, + "m": 299, "n": 7168, - "k": 8192, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "k": 16384, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/o_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/o_proj" + ] }, { "type": "gemm", "args": { - "m": 32, - "n": 8192, + "m": 299, + "n": 24576, "k": 1536, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/q_b_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/q_b_proj" + ] }, { "type": "gemm", "args": { - "m": 32, - "n": 1088, - "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 299, + "n": 32768, + "k": 512, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/kv_b_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/kv_b_proj" + ] }, { "type": "gemm", "args": { - "m": 32, - "n": 15360, - "k": 1024, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 303, + "n": 576, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/Kimi-K2.5-MXFP4/kv_a_proj_with_mqa", + "moonshotai/Kimi-K2.5/kv_a_proj_with_mqa", + "nvidia/DeepSeek-R1-0528-FP4-V2/kv_a_proj_with_mqa", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/kv_a_proj_with_mqa", + "nvidia/Kimi-K2.5-NVFP4/kv_a_proj_with_mqa", + "nvidia/Kimi-K2.6-NVFP4/kv_a_proj_with_mqa" + ] }, { "type": "gemm", "args": { - "m": 32, - "n": 7168, - "k": 8192, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 303, + "n": 1536, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/Kimi-K2.5-MXFP4/q_a_proj", + "moonshotai/Kimi-K2.5/q_a_proj", + "nvidia/DeepSeek-R1-0528-FP4-V2/q_a_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/q_a_proj", + "nvidia/Kimi-K2.5-NVFP4/q_a_proj", + "nvidia/Kimi-K2.6-NVFP4/q_a_proj" + ] }, { "type": "gemm", "args": { - "m": 8192, - "n": 65536, - "k": 1536, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 303, + "n": 2304, + "k": 7168, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/gate_up_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/gate_up_proj", + "nvidia/Kimi-K2.5-NVFP4/gate_up_proj" + ] }, { "type": "gemm", "args": { - "m": 8192, - "n": 122880, - "k": 1024, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 303, + "n": 7168, + "k": 2304, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/down_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/down_proj", + "nvidia/Kimi-K2.5-NVFP4/down_proj" + ] }, { "type": "gemm", "args": { - "m": 8192, + "m": 303, "n": 7168, - "k": 65536, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "k": 16384, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/o_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/o_proj" + ] }, { "type": "gemm", "args": { - "m": 8, - "n": 65536, + "m": 303, + "n": 24576, "k": 1536, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/q_b_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/q_b_proj" + ] }, { "type": "gemm", "args": { - "m": 8, - "n": 122880, - "k": 1024, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 303, + "n": 32768, + "k": 512, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/kv_b_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/kv_b_proj" + ] }, { "type": "gemm", "args": { - "m": 8, - "n": 7168, - "k": 65536, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 307, + "n": 576, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/kv_a_proj_with_mqa" + ] }, { "type": "gemm", "args": { - "m": 16384, - "n": 65536, - "k": 1536, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 307, + "n": 1152, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/gate_up_proj" + ] }, { "type": "gemm", "args": { - "m": 16384, - "n": 122880, - "k": 1024, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 307, + "n": 1536, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/gate_up_proj", + "deepseek-ai/DeepSeek-R1-0528/q_a_proj", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w3" + ] }, { "type": "gemm", "args": { - "m": 16384, - "n": 7168, - "k": 65536, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 307, + "n": 2304, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/gate_up_proj" + ] }, { "type": "gemm", "args": { - "m": 16, - "n": 65536, - "k": 1536, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 307, + "n": 4608, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/gate_up_proj" + ] }, { "type": "gemm", "args": { - "m": 16, - "n": 122880, - "k": 1024, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 307, + "n": 7168, + "k": 1152, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/down_proj" + ] }, { "type": "gemm", "args": { - "m": 16, + "m": 307, "n": 7168, - "k": 65536, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "k": 2304, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/down_proj" + ] }, { "type": "gemm", "args": { - "m": 32768, - "n": 65536, - "k": 1536, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 307, + "n": 7168, + "k": 4608, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/down_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/down_proj", + "nvidia/Kimi-K2.5-NVFP4/down_proj" + ] }, { "type": "gemm", "args": { - "m": 32768, - "n": 122880, - "k": 1024, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 307, + "n": 7168, + "k": 4608, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 32 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 1, + 32 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "amd/DeepSeek-R1-0528-MXFP4-Preview/down_proj", + "amd/DeepSeek-R1-0528-MXFP4-v2/down_proj", + "amd/DeepSeek-R1-0528-MXFP4/down_proj", + "amd/Kimi-K2.5-MXFP4/down_proj" + ] }, { "type": "gemm", "args": { - "m": 32768, + "m": 307, "n": 7168, - "k": 65536, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "k": 4608, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/down_proj" + ] }, { "type": "gemm", "args": { - "m": 32, - "n": 65536, - "k": 1536, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 307, + "n": 7168, + "k": 16384, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/o_proj", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] }, { "type": "gemm", "args": { - "m": 32, - "n": 122880, - "k": 1024, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 307, + "n": 24576, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/q_b_proj" + ] }, { "type": "gemm", "args": { - "m": 32, - "n": 7168, - "k": 65536, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 307, + "n": 32768, + "k": 512, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/kv_b_proj" + ] }, { "type": "gemm", "args": { - "m": 65536, - "n": 65536, - "k": 1536, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 308, + "n": 576, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/kv_a_proj_with_mqa" + ] }, { "type": "gemm", "args": { - "m": 65536, - "n": 1088, + "m": 308, + "n": 1536, "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/gate_up_proj", + "deepseek-ai/DeepSeek-R1-0528/q_a_proj", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w3" + ] }, { "type": "gemm", "args": { - "m": 65536, - "n": 122880, - "k": 1024, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 308, + "n": 7168, + "k": 9216, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/down_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/down_proj", + "nvidia/Kimi-K2.5-NVFP4/down_proj" + ] }, { "type": "gemm", "args": { - "m": 65536, + "m": 308, "n": 7168, - "k": 65536, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "k": 16384, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/o_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/o_proj" + ] }, { "type": "gemm", "args": { - "m": 64, - "n": 65536, - "k": 1536, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 308, + "n": 7168, + "k": 16384, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 32 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 1, + 32 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "amd/DeepSeek-R1-0528-MXFP4-Preview/o_proj", + "amd/DeepSeek-R1-0528-MXFP4-v2/o_proj", + "amd/DeepSeek-R1-0528-MXFP4/o_proj" + ] }, { "type": "gemm", "args": { - "m": 64, - "n": 1088, - "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 308, + "n": 7168, + "k": 16384, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] }, { "type": "gemm", "args": { - "m": 64, - "n": 122880, - "k": 1024, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 308, + "n": 9216, + "k": 7168, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/gate_up_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/gate_up_proj", + "nvidia/Kimi-K2.5-NVFP4/gate_up_proj" + ] }, { "type": "gemm", "args": { - "m": 64, - "n": 7168, - "k": 65536, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 308, + "n": 9216, + "k": 7168, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 32 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 1, + 32 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "amd/DeepSeek-R1-0528-MXFP4-Preview/gate_up_proj", + "amd/DeepSeek-R1-0528-MXFP4-v2/gate_up_proj", + "amd/DeepSeek-R1-0528-MXFP4/gate_up_proj", + "amd/Kimi-K2.5-MXFP4/gate_up_proj" + ] }, { "type": "gemm", "args": { - "m": 131072, - "n": 65536, + "m": 308, + "n": 24576, "k": 1536, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/q_b_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/q_b_proj" + ] }, { "type": "gemm", "args": { - "m": 131072, - "n": 1088, - "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 308, + "n": 24576, + "k": 1536, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 32 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 1, + 32 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "amd/DeepSeek-R1-0528-MXFP4-Preview/q_b_proj", + "amd/DeepSeek-R1-0528-MXFP4-v2/q_b_proj", + "amd/DeepSeek-R1-0528-MXFP4/q_b_proj" + ] }, { "type": "gemm", "args": { - "m": 131072, - "n": 122880, - "k": 1024, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 308, + "n": 32768, + "k": 512, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/kv_b_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/kv_b_proj" + ] }, { "type": "gemm", "args": { - "m": 131072, - "n": 7168, - "k": 65536, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 308, + "n": 32768, + "k": 512, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 32 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 1, + 32 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "amd/DeepSeek-R1-0528-MXFP4-Preview/kv_b_proj", + "amd/DeepSeek-R1-0528-MXFP4-v2/kv_b_proj", + "amd/DeepSeek-R1-0528-MXFP4/kv_b_proj" + ] }, { "type": "gemm", "args": { - "m": 128, - "n": 65536, - "k": 1536, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 320, + "n": 576, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/kv_a_proj_with_mqa" + ] }, { "type": "gemm", "args": { - "m": 128, - "n": 1088, + "m": 320, + "n": 1536, "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/gate_up_proj", + "deepseek-ai/DeepSeek-R1-0528/q_a_proj", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w3" + ] }, { "type": "gemm", "args": { - "m": 128, - "n": 122880, - "k": 1024, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 320, + "n": 2304, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/gate_up_proj" + ] }, { "type": "gemm", "args": { - "m": 128, - "n": 7168, - "k": 65536, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 320, + "n": 3072, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "MiniMaxAI/MiniMax-M2.5/experts.down_proj", + "MiniMaxAI/MiniMax-M2.5/o_proj", + "deepseek-ai/DeepSeek-R1-0528/q_b_proj" + ] }, { "type": "gemm", "args": { - "m": 65536, - "n": 8192, - "k": 1536, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 320, + "n": 4096, + "k": 512, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "Qwen/Qwen3.5-397B-A17B-FP8/o_proj", + "Qwen/Qwen3.5-397B-A17B-FP8/shared_expert.down_proj", + "deepseek-ai/DeepSeek-R1-0528/kv_b_proj" + ] }, { "type": "gemm", "args": { - "m": 65536, - "n": 15360, - "k": 1024, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 320, + "n": 4608, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/gate_up_proj" + ] }, { "type": "gemm", "args": { - "m": 65536, + "m": 320, "n": 7168, - "k": 8192, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "k": 2048, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/o_proj", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] }, { "type": "gemm", "args": { - "m": 131072, - "n": 8192, - "k": 1536, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 320, + "n": 7168, + "k": 2304, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/down_proj" + ] }, { "type": "gemm", "args": { - "m": 131072, - "n": 15360, - "k": 1024, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 320, + "n": 7168, + "k": 4608, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/down_proj" + ] }, { "type": "gemm", "args": { - "m": 131072, + "m": 320, "n": 7168, - "k": 8192, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "k": 16384, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/o_proj", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] }, { "type": "gemm", "args": { - "m": 262144, - "n": 8192, + "m": 320, + "n": 24576, "k": 1536, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/q_b_proj" + ] }, { "type": "gemm", "args": { - "m": 262144, - "n": 1088, - "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 320, + "n": 32768, + "k": 512, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/kv_b_proj" + ] }, { "type": "gemm", "args": { - "m": 262144, - "n": 15360, - "k": 1024, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 329, + "n": 576, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/Kimi-K2.5-MXFP4/kv_a_proj_with_mqa", + "moonshotai/Kimi-K2.5/kv_a_proj_with_mqa", + "nvidia/DeepSeek-R1-0528-FP4-V2/kv_a_proj_with_mqa", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/kv_a_proj_with_mqa", + "nvidia/Kimi-K2.5-NVFP4/kv_a_proj_with_mqa", + "nvidia/Kimi-K2.6-NVFP4/kv_a_proj_with_mqa" + ] }, { "type": "gemm", "args": { - "m": 262144, - "n": 7168, - "k": 8192, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 329, + "n": 576, + "k": 7168, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 32 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 1, + 32 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "amd/DeepSeek-R1-0528-MXFP4-Preview/kv_a_proj_with_mqa", + "amd/DeepSeek-R1-0528-MXFP4-v2/kv_a_proj_with_mqa", + "amd/DeepSeek-R1-0528-MXFP4/kv_a_proj_with_mqa" + ] }, { "type": "gemm", "args": { - "m": 262144, - "n": 65536, - "k": 1536, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 329, + "n": 1536, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/Kimi-K2.5-MXFP4/q_a_proj", + "moonshotai/Kimi-K2.5/q_a_proj", + "nvidia/DeepSeek-R1-0528-FP4-V2/q_a_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/q_a_proj", + "nvidia/Kimi-K2.5-NVFP4/q_a_proj", + "nvidia/Kimi-K2.6-NVFP4/q_a_proj" + ] }, { "type": "gemm", "args": { - "m": 262144, - "n": 122880, - "k": 1024, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 329, + "n": 1536, + "k": 7168, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 32 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 1, + 32 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "amd/DeepSeek-R1-0528-MXFP4-Preview/q_a_proj", + "amd/DeepSeek-R1-0528-MXFP4-v2/q_a_proj", + "amd/DeepSeek-R1-0528-MXFP4/q_a_proj" + ] }, { "type": "gemm", "args": { - "m": 262144, - "n": 7168, - "k": 65536, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 329, + "n": 1536, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w3" + ] }, { "type": "gemm", "args": { - "m": 524288, - "n": 65536, - "k": 1536, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 329, + "n": 4608, + "k": 7168, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/gate_up_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/gate_up_proj", + "nvidia/Kimi-K2.5-NVFP4/gate_up_proj" + ] }, { "type": "gemm", "args": { - "m": 524288, - "n": 1088, + "m": 329, + "n": 4608, "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 32 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 1, + 32 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "amd/DeepSeek-R1-0528-MXFP4-Preview/gate_up_proj", + "amd/DeepSeek-R1-0528-MXFP4-v2/gate_up_proj", + "amd/DeepSeek-R1-0528-MXFP4/gate_up_proj", + "amd/Kimi-K2.5-MXFP4/gate_up_proj" + ] }, { "type": "gemm", "args": { - "m": 524288, - "n": 122880, - "k": 1024, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 329, + "n": 7168, + "k": 4608, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/down_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/down_proj", + "nvidia/Kimi-K2.5-NVFP4/down_proj" + ] }, { "type": "gemm", "args": { - "m": 524288, + "m": 329, "n": 7168, - "k": 65536, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "k": 4608, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 32 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 1, + 32 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "amd/DeepSeek-R1-0528-MXFP4-Preview/down_proj", + "amd/DeepSeek-R1-0528-MXFP4-v2/down_proj", + "amd/DeepSeek-R1-0528-MXFP4/down_proj", + "amd/Kimi-K2.5-MXFP4/down_proj" + ] }, { "type": "gemm", "args": { - "m": 64, - "n": 8192, - "k": 1536, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 329, + "n": 7168, + "k": 16384, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/o_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/o_proj" + ] }, { "type": "gemm", "args": { - "m": 64, - "n": 15360, - "k": 1024, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 329, + "n": 7168, + "k": 16384, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 32 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 1, + 32 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "amd/DeepSeek-R1-0528-MXFP4-Preview/o_proj", + "amd/DeepSeek-R1-0528-MXFP4-v2/o_proj", + "amd/DeepSeek-R1-0528-MXFP4/o_proj" + ] }, { "type": "gemm", "args": { - "m": 64, + "m": 329, "n": 7168, - "k": 8192, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "k": 16384, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] }, { "type": "gemm", "args": { - "m": 128, - "n": 8192, + "m": 329, + "n": 24576, "k": 1536, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/q_b_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/q_b_proj" + ] }, { "type": "gemm", "args": { - "m": 128, - "n": 15360, - "k": 1024, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 329, + "n": 24576, + "k": 1536, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 32 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 1, + 32 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "amd/DeepSeek-R1-0528-MXFP4-Preview/q_b_proj", + "amd/DeepSeek-R1-0528-MXFP4-v2/q_b_proj", + "amd/DeepSeek-R1-0528-MXFP4/q_b_proj" + ] }, { "type": "gemm", "args": { - "m": 128, - "n": 7168, - "k": 8192, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 329, + "n": 32768, + "k": 512, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/kv_b_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/kv_b_proj" + ] }, { "type": "gemm", "args": { - "m": 256, - "n": 65536, - "k": 1536, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 329, + "n": 32768, + "k": 512, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 32 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 1, + 32 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "amd/DeepSeek-R1-0528-MXFP4-Preview/kv_b_proj", + "amd/DeepSeek-R1-0528-MXFP4-v2/kv_b_proj", + "amd/DeepSeek-R1-0528-MXFP4/kv_b_proj" + ] }, { "type": "gemm", "args": { - "m": 256, - "n": 1088, + "m": 333, + "n": 576, "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/Kimi-K2.5-MXFP4/kv_a_proj_with_mqa", + "moonshotai/Kimi-K2.5/kv_a_proj_with_mqa", + "nvidia/DeepSeek-R1-0528-FP4-V2/kv_a_proj_with_mqa", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/kv_a_proj_with_mqa", + "nvidia/Kimi-K2.5-NVFP4/kv_a_proj_with_mqa", + "nvidia/Kimi-K2.6-NVFP4/kv_a_proj_with_mqa" + ] }, { "type": "gemm", "args": { - "m": 256, - "n": 122880, - "k": 1024, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 333, + "n": 576, + "k": 7168, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 32 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 1, + 32 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "amd/DeepSeek-R1-0528-MXFP4-Preview/kv_a_proj_with_mqa", + "amd/DeepSeek-R1-0528-MXFP4-v2/kv_a_proj_with_mqa", + "amd/DeepSeek-R1-0528-MXFP4/kv_a_proj_with_mqa" + ] }, { "type": "gemm", "args": { - "m": 256, - "n": 7168, - "k": 65536, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 333, + "n": 576, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/kv_a_proj_with_mqa" + ] }, { "type": "gemm", "args": { - "m": 512, - "n": 65536, - "k": 1536, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 333, + "n": 1536, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/Kimi-K2.5-MXFP4/q_a_proj", + "moonshotai/Kimi-K2.5/q_a_proj", + "nvidia/DeepSeek-R1-0528-FP4-V2/q_a_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/q_a_proj", + "nvidia/Kimi-K2.5-NVFP4/q_a_proj", + "nvidia/Kimi-K2.6-NVFP4/q_a_proj" + ] }, { "type": "gemm", "args": { - "m": 512, - "n": 1088, + "m": 333, + "n": 1536, "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 32 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 1, + 32 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "amd/DeepSeek-R1-0528-MXFP4-Preview/q_a_proj", + "amd/DeepSeek-R1-0528-MXFP4-v2/q_a_proj", + "amd/DeepSeek-R1-0528-MXFP4/q_a_proj" + ] }, { "type": "gemm", "args": { - "m": 512, - "n": 122880, - "k": 1024, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 333, + "n": 1536, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/gate_up_proj", + "deepseek-ai/DeepSeek-R1-0528/q_a_proj", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w3" + ] }, { "type": "gemm", "args": { - "m": 512, - "n": 7168, - "k": 65536, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 333, + "n": 1536, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w3" + ] }, { "type": "gemm", "args": { - "m": 1048576, - "n": 65536, - "k": 1536, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 333, + "n": 4608, + "k": 7168, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/gate_up_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/gate_up_proj", + "nvidia/Kimi-K2.5-NVFP4/gate_up_proj" + ] }, { "type": "gemm", "args": { - "m": 1048576, - "n": 1088, + "m": 333, + "n": 4608, "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 32 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 1, + 32 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "amd/DeepSeek-R1-0528-MXFP4-Preview/gate_up_proj", + "amd/DeepSeek-R1-0528-MXFP4-v2/gate_up_proj", + "amd/DeepSeek-R1-0528-MXFP4/gate_up_proj", + "amd/Kimi-K2.5-MXFP4/gate_up_proj" + ] }, { "type": "gemm", "args": { - "m": 1048576, - "n": 122880, - "k": 1024, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 333, + "n": 4608, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/gate_up_proj" + ] }, { "type": "gemm", "args": { - "m": 1048576, + "m": 333, "n": 7168, - "k": 65536, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "k": 4608, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/down_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/down_proj", + "nvidia/Kimi-K2.5-NVFP4/down_proj" + ] }, { "type": "gemm", "args": { - "m": 4096, - "n": 65536, - "k": 1536, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 333, + "n": 7168, + "k": 4608, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 32 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 1, + 32 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "amd/DeepSeek-R1-0528-MXFP4-Preview/down_proj", + "amd/DeepSeek-R1-0528-MXFP4-v2/down_proj", + "amd/DeepSeek-R1-0528-MXFP4/down_proj", + "amd/Kimi-K2.5-MXFP4/down_proj" + ] }, { "type": "gemm", "args": { - "m": 4096, - "n": 122880, - "k": 1024, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 333, + "n": 7168, + "k": 4608, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/down_proj" + ] }, { "type": "gemm", "args": { - "m": 4096, + "m": 333, "n": 7168, - "k": 65536, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "k": 9216, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/down_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/down_proj", + "nvidia/Kimi-K2.5-NVFP4/down_proj" + ] }, { "type": "gemm", "args": { - "m": 4, - "n": 65536, - "k": 1536, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 333, + "n": 7168, + "k": 16384, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/o_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/o_proj" + ] }, { "type": "gemm", "args": { - "m": 4, - "n": 122880, - "k": 1024, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 333, + "n": 7168, + "k": 16384, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 32 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 1, + 32 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "amd/DeepSeek-R1-0528-MXFP4-Preview/o_proj", + "amd/DeepSeek-R1-0528-MXFP4-v2/o_proj", + "amd/DeepSeek-R1-0528-MXFP4/o_proj" + ] }, { "type": "gemm", "args": { - "m": 4, + "m": 333, "n": 7168, - "k": 65536, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "k": 16384, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/o_proj", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] }, { "type": "gemm", "args": { - "m": 256, - "n": 8192, - "k": 1536, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 333, + "n": 7168, + "k": 16384, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] }, { "type": "gemm", "args": { - "m": 256, - "n": 15360, - "k": 1024, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 333, + "n": 9216, + "k": 7168, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/gate_up_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/gate_up_proj", + "nvidia/Kimi-K2.5-NVFP4/gate_up_proj" + ] }, { "type": "gemm", "args": { - "m": 256, - "n": 7168, - "k": 8192, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 333, + "n": 9216, + "k": 7168, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 32 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 1, + 32 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "amd/DeepSeek-R1-0528-MXFP4-Preview/gate_up_proj", + "amd/DeepSeek-R1-0528-MXFP4-v2/gate_up_proj", + "amd/DeepSeek-R1-0528-MXFP4/gate_up_proj", + "amd/Kimi-K2.5-MXFP4/gate_up_proj" + ] }, { "type": "gemm", "args": { - "m": 1024, - "n": 65536, + "m": 333, + "n": 24576, "k": 1536, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/q_b_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/q_b_proj" + ] }, { "type": "gemm", "args": { - "m": 1024, - "n": 122880, - "k": 1024, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 333, + "n": 24576, + "k": 1536, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 32 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 1, + 32 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "amd/DeepSeek-R1-0528-MXFP4-Preview/q_b_proj", + "amd/DeepSeek-R1-0528-MXFP4-v2/q_b_proj", + "amd/DeepSeek-R1-0528-MXFP4/q_b_proj" + ] }, { "type": "gemm", "args": { - "m": 1024, - "n": 7168, - "k": 65536, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 333, + "n": 24576, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/q_b_proj" + ] }, { "type": "gemm", "args": { - "m": 64, - "n": 24576, - "k": 1536, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 333, + "n": 32768, + "k": 512, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/kv_b_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/kv_b_proj" + ] }, { "type": "gemm", "args": { - "m": 64, + "m": 333, "n": 32768, "k": 512, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 32 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 1, + 32 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "amd/DeepSeek-R1-0528-MXFP4-Preview/kv_b_proj", + "amd/DeepSeek-R1-0528-MXFP4-v2/kv_b_proj", + "amd/DeepSeek-R1-0528-MXFP4/kv_b_proj" + ] }, { "type": "gemm", "args": { - "m": 64, - "n": 7168, - "k": 16384, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 333, + "n": 32768, + "k": 512, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/kv_b_proj" + ] }, { "type": "gemm", "args": { - "m": 16, - "n": 24576, - "k": 1536, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 341, + "n": 576, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/Kimi-K2.5-MXFP4/kv_a_proj_with_mqa", + "moonshotai/Kimi-K2.5/kv_a_proj_with_mqa", + "nvidia/DeepSeek-R1-0528-FP4-V2/kv_a_proj_with_mqa", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/kv_a_proj_with_mqa", + "nvidia/Kimi-K2.5-NVFP4/kv_a_proj_with_mqa", + "nvidia/Kimi-K2.6-NVFP4/kv_a_proj_with_mqa" + ] }, { "type": "gemm", "args": { - "m": 16, - "n": 32768, - "k": 512, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 341, + "n": 576, + "k": 7168, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 32 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 1, + 32 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "amd/DeepSeek-R1-0528-MXFP4-Preview/kv_a_proj_with_mqa", + "amd/DeepSeek-R1-0528-MXFP4-v2/kv_a_proj_with_mqa", + "amd/DeepSeek-R1-0528-MXFP4/kv_a_proj_with_mqa" + ] }, { "type": "gemm", "args": { - "m": 16, - "n": 7168, - "k": 16384, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 341, + "n": 1536, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/Kimi-K2.5-MXFP4/q_a_proj", + "moonshotai/Kimi-K2.5/q_a_proj", + "nvidia/DeepSeek-R1-0528-FP4-V2/q_a_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/q_a_proj", + "nvidia/Kimi-K2.5-NVFP4/q_a_proj", + "nvidia/Kimi-K2.6-NVFP4/q_a_proj" + ] }, { "type": "gemm", "args": { - "m": 262144, - "n": 24576, - "k": 1536, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 341, + "n": 1536, + "k": 7168, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 32 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 1, + 32 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "amd/DeepSeek-R1-0528-MXFP4-Preview/q_a_proj", + "amd/DeepSeek-R1-0528-MXFP4-v2/q_a_proj", + "amd/DeepSeek-R1-0528-MXFP4/q_a_proj" + ] }, { "type": "gemm", "args": { - "m": 262144, - "n": 32768, - "k": 512, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 341, + "n": 1536, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w3" + ] }, { "type": "gemm", "args": { - "m": 262144, - "n": 7168, - "k": 16384, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 341, + "n": 4608, + "k": 7168, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/gate_up_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/gate_up_proj", + "nvidia/Kimi-K2.5-NVFP4/gate_up_proj" + ] }, { "type": "gemm", "args": { - "m": 128, - "n": 24576, - "k": 1536, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 341, + "n": 4608, + "k": 7168, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 32 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 1, + 32 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "amd/DeepSeek-R1-0528-MXFP4-Preview/gate_up_proj", + "amd/DeepSeek-R1-0528-MXFP4-v2/gate_up_proj", + "amd/DeepSeek-R1-0528-MXFP4/gate_up_proj", + "amd/Kimi-K2.5-MXFP4/gate_up_proj" + ] }, { "type": "gemm", "args": { - "m": 128, - "n": 32768, - "k": 512, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 341, + "n": 7168, + "k": 4608, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/down_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/down_proj", + "nvidia/Kimi-K2.5-NVFP4/down_proj" + ] }, { "type": "gemm", "args": { - "m": 128, + "m": 341, "n": 7168, - "k": 16384, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "k": 4608, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 32 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 1, + 32 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "amd/DeepSeek-R1-0528-MXFP4-Preview/down_proj", + "amd/DeepSeek-R1-0528-MXFP4-v2/down_proj", + "amd/DeepSeek-R1-0528-MXFP4/down_proj", + "amd/Kimi-K2.5-MXFP4/down_proj" + ] }, { "type": "gemm", "args": { - "m": 8, - "n": 3072, - "k": 1536, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 341, + "n": 7168, + "k": 16384, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/o_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/o_proj" + ] }, { "type": "gemm", "args": { - "m": 8, - "n": 4096, - "k": 512, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 341, + "n": 7168, + "k": 16384, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 32 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 1, + 32 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "amd/DeepSeek-R1-0528-MXFP4-Preview/o_proj", + "amd/DeepSeek-R1-0528-MXFP4-v2/o_proj", + "amd/DeepSeek-R1-0528-MXFP4/o_proj" + ] }, { "type": "gemm", "args": { - "m": 8, + "m": 341, "n": 7168, - "k": 2048, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "k": 16384, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] }, { "type": "gemm", "args": { - "m": 16384, - "n": 3072, + "m": 341, + "n": 24576, "k": 1536, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/q_b_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/q_b_proj" + ] }, { "type": "gemm", "args": { - "m": 16384, - "n": 4096, - "k": 512, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 341, + "n": 24576, + "k": 1536, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 32 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 1, + 32 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "amd/DeepSeek-R1-0528-MXFP4-Preview/q_b_proj", + "amd/DeepSeek-R1-0528-MXFP4-v2/q_b_proj", + "amd/DeepSeek-R1-0528-MXFP4/q_b_proj" + ] }, { "type": "gemm", "args": { - "m": 16384, - "n": 7168, - "k": 2048, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 341, + "n": 32768, + "k": 512, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/kv_b_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/kv_b_proj" + ] }, { "type": "gemm", "args": { - "m": 32768, - "n": 6144, - "k": 1536, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 341, + "n": 32768, + "k": 512, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 32 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 1, + 32 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "amd/DeepSeek-R1-0528-MXFP4-Preview/kv_b_proj", + "amd/DeepSeek-R1-0528-MXFP4-v2/kv_b_proj", + "amd/DeepSeek-R1-0528-MXFP4/kv_b_proj" + ] }, { "type": "gemm", "args": { - "m": 32768, - "n": 8192, - "k": 512, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 360, + "n": 576, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/Kimi-K2.5-MXFP4/kv_a_proj_with_mqa", + "moonshotai/Kimi-K2.5/kv_a_proj_with_mqa", + "nvidia/DeepSeek-R1-0528-FP4-V2/kv_a_proj_with_mqa", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/kv_a_proj_with_mqa", + "nvidia/Kimi-K2.5-NVFP4/kv_a_proj_with_mqa", + "nvidia/Kimi-K2.6-NVFP4/kv_a_proj_with_mqa" + ] }, { "type": "gemm", "args": { - "m": 32768, - "n": 7168, - "k": 4096, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 360, + "n": 1536, + "k": 1536, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/Kimi-K2.5-MXFP4/q_b_proj", + "moonshotai/Kimi-K2.5/q_b_proj", + "nvidia/DeepSeek-R1-0528-FP4-V2/q_b_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/q_b_proj", + "nvidia/Kimi-K2.5-NVFP4/q_b_proj", + "nvidia/Kimi-K2.6-NVFP4/q_b_proj" + ] }, { "type": "gemm", "args": { - "m": 4, - "n": 6144, - "k": 1536, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 360, + "n": 1536, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/Kimi-K2.5-MXFP4/q_a_proj", + "moonshotai/Kimi-K2.5/q_a_proj", + "nvidia/DeepSeek-R1-0528-FP4-V2/q_a_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/q_a_proj", + "nvidia/Kimi-K2.5-NVFP4/q_a_proj", + "nvidia/Kimi-K2.6-NVFP4/q_a_proj" + ] }, { "type": "gemm", "args": { - "m": 4, - "n": 8192, + "m": 360, + "n": 2048, "k": 512, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/Kimi-K2.5-MXFP4/kv_b_proj", + "moonshotai/Kimi-K2.5/kv_b_proj", + "nvidia/DeepSeek-R1-0528-FP4-V2/kv_b_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/kv_b_proj", + "nvidia/Kimi-K2.5-NVFP4/kv_b_proj", + "nvidia/Kimi-K2.6-NVFP4/kv_b_proj" + ] }, { "type": "gemm", "args": { - "m": 4, - "n": 7168, - "k": 4096, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 360, + "n": 2304, + "k": 7168, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/gate_up_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/gate_up_proj", + "nvidia/Kimi-K2.5-NVFP4/gate_up_proj" + ] }, { "type": "gemm", "args": { - "m": 65536, - "n": 6144, + "m": 360, + "n": 3072, "k": 1536, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/Kimi-K2.5-MXFP4/q_b_proj", + "amd/MiniMax-M2.5-MXFP4/o_proj", + "moonshotai/Kimi-K2.5/q_b_proj", + "nvidia/DeepSeek-R1-0528-FP4-V2/q_b_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/q_b_proj", + "nvidia/Kimi-K2.5-NVFP4/q_b_proj", + "nvidia/Kimi-K2.6-NVFP4/q_b_proj", + "nvidia/MiniMax-M2.5-NVFP4/o_proj" + ] }, { "type": "gemm", "args": { - "m": 65536, - "n": 8192, + "m": 360, + "n": 4096, "k": 512, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "Qwen/Qwen3.5-397B-A17B/o_proj", + "Qwen/Qwen3.5-397B-A17B/shared_expert.down_proj", + "amd/Kimi-K2.5-MXFP4/kv_b_proj", + "amd/Qwen3.5-397B-A17B-MXFP4/o_proj", + "amd/Qwen3.5-397B-A17B-MXFP4/shared_expert.down_proj", + "moonshotai/Kimi-K2.5/kv_b_proj", + "nvidia/DeepSeek-R1-0528-FP4-V2/kv_b_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/kv_b_proj", + "nvidia/Kimi-K2.5-NVFP4/kv_b_proj", + "nvidia/Kimi-K2.6-NVFP4/kv_b_proj", + "nvidia/Qwen3.5-397B-A17B-NVFP4/o_proj", + "nvidia/Qwen3.5-397B-A17B-NVFP4/shared_expert.down_proj" + ] }, { "type": "gemm", "args": { - "m": 65536, - "n": 7168, - "k": 4096, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 360, + "n": 4608, + "k": 7168, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/gate_up_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/gate_up_proj", + "nvidia/Kimi-K2.5-NVFP4/gate_up_proj" + ] }, { "type": "gemm", "args": { - "m": 65536, - "n": 4608, - "k": 7168, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 360, + "n": 7168, + "k": 1024, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/Kimi-K2.5-MXFP4/o_proj", + "moonshotai/Kimi-K2.5/o_proj", + "nvidia/Kimi-K2.5-NVFP4/o_proj", + "nvidia/Kimi-K2.6-NVFP4/o_proj" + ] }, { "type": "gemm", "args": { - "m": 65536, + "m": 360, "n": 7168, - "k": 4608, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "k": 2048, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/o_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/o_proj" + ] }, { "type": "gemm", "args": { - "m": 8, - "n": 6144, - "k": 1536, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 360, + "n": 7168, + "k": 2304, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/down_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/down_proj", + "nvidia/Kimi-K2.5-NVFP4/down_proj" + ] }, { "type": "gemm", "args": { - "m": 8, - "n": 8192, - "k": 512, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 360, + "n": 7168, + "k": 4608, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/down_proj" + ] }, { "type": "gemm", "args": { - "m": 8, + "m": 360, "n": 7168, - "k": 4096, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "k": 16384, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/o_proj", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] }, { "type": "gemm", "args": { - "m": 8, - "n": 4608, - "k": 7168, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 360, + "n": 24576, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/q_b_proj" + ] }, { "type": "gemm", "args": { - "m": 8, - "n": 7168, - "k": 4608, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 360, + "n": 32768, + "k": 512, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/kv_b_proj" + ] }, { "type": "gemm", "args": { - "m": 131072, - "n": 6144, - "k": 1536, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 370, + "n": 576, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/Kimi-K2.5-MXFP4/kv_a_proj_with_mqa", + "moonshotai/Kimi-K2.5/kv_a_proj_with_mqa", + "nvidia/DeepSeek-R1-0528-FP4-V2/kv_a_proj_with_mqa", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/kv_a_proj_with_mqa", + "nvidia/Kimi-K2.5-NVFP4/kv_a_proj_with_mqa", + "nvidia/Kimi-K2.6-NVFP4/kv_a_proj_with_mqa" + ] }, { "type": "gemm", "args": { - "m": 131072, - "n": 8192, - "k": 512, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 370, + "n": 1536, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/Kimi-K2.5-MXFP4/q_a_proj", + "moonshotai/Kimi-K2.5/q_a_proj", + "nvidia/DeepSeek-R1-0528-FP4-V2/q_a_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/q_a_proj", + "nvidia/Kimi-K2.5-NVFP4/q_a_proj", + "nvidia/Kimi-K2.6-NVFP4/q_a_proj" + ] }, { "type": "gemm", "args": { - "m": 131072, - "n": 7168, - "k": 4096, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 370, + "n": 2304, + "k": 7168, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/gate_up_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/gate_up_proj", + "nvidia/Kimi-K2.5-NVFP4/gate_up_proj" + ] }, { "type": "gemm", "args": { - "m": 16, - "n": 6144, + "m": 370, + "n": 3072, "k": 1536, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/Kimi-K2.5-MXFP4/q_b_proj", + "amd/MiniMax-M2.5-MXFP4/o_proj", + "moonshotai/Kimi-K2.5/q_b_proj", + "nvidia/DeepSeek-R1-0528-FP4-V2/q_b_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/q_b_proj", + "nvidia/Kimi-K2.5-NVFP4/q_b_proj", + "nvidia/Kimi-K2.6-NVFP4/q_b_proj", + "nvidia/MiniMax-M2.5-NVFP4/o_proj" + ] }, { "type": "gemm", "args": { - "m": 16, - "n": 8192, + "m": 370, + "n": 4096, "k": 512, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "Qwen/Qwen3.5-397B-A17B/o_proj", + "Qwen/Qwen3.5-397B-A17B/shared_expert.down_proj", + "amd/Kimi-K2.5-MXFP4/kv_b_proj", + "amd/Qwen3.5-397B-A17B-MXFP4/o_proj", + "amd/Qwen3.5-397B-A17B-MXFP4/shared_expert.down_proj", + "moonshotai/Kimi-K2.5/kv_b_proj", + "nvidia/DeepSeek-R1-0528-FP4-V2/kv_b_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/kv_b_proj", + "nvidia/Kimi-K2.5-NVFP4/kv_b_proj", + "nvidia/Kimi-K2.6-NVFP4/kv_b_proj", + "nvidia/Qwen3.5-397B-A17B-NVFP4/o_proj", + "nvidia/Qwen3.5-397B-A17B-NVFP4/shared_expert.down_proj" + ] }, { "type": "gemm", "args": { - "m": 16, - "n": 7168, - "k": 4096, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 262144, - "n": 6144, - "k": 1536, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 262144, - "n": 8192, - "k": 512, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 262144, - "n": 7168, - "k": 4096, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 262144, - "n": 4608, - "k": 7168, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 262144, - "n": 7168, - "k": 4608, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 32, - "n": 6144, - "k": 1536, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 32, - "n": 8192, - "k": 512, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 32, - "n": 7168, - "k": 4096, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 32768, - "n": 16384, - "k": 1536, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 32768, - "n": 30720, - "k": 1024, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 32, - "n": 16384, - "k": 1536, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 32, - "n": 30720, - "k": 1024, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 262144, - "n": 16384, - "k": 1536, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 262144, - "n": 30720, - "k": 1024, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 2097152, - "n": 65536, - "k": 1536, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 2097152, - "n": 1088, - "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 2097152, - "n": 122880, - "k": 1024, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 2097152, - "n": 7168, - "k": 65536, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 4194304, - "n": 65536, - "k": 1536, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 4194304, - "n": 1088, - "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 4194304, - "n": 122880, - "k": 1024, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 4194304, - "n": 7168, - "k": 65536, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 4096, - "n": 16384, - "k": 1536, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 4096, - "n": 30720, - "k": 1024, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 8, - "n": 16384, - "k": 1536, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 8, - "n": 30720, - "k": 1024, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 8192, - "n": 16384, - "k": 1536, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 8192, - "n": 30720, - "k": 1024, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 16, - "n": 16384, - "k": 1536, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 16, - "n": 30720, - "k": 1024, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 16384, - "n": 16384, - "k": 1536, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 16384, - "n": 30720, - "k": 1024, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 64, - "n": 16384, - "k": 1536, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 64, - "n": 30720, - "k": 1024, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 65536, - "n": 16384, - "k": 1536, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 65536, - "n": 30720, - "k": 1024, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 131072, - "n": 16384, - "k": 1536, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 131072, - "n": 30720, - "k": 1024, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 4096, - "n": 1024, - "k": 4096, - "dtype_a": "bf16", - "dtype_b": "bf16", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 4096, - "n": 256, - "k": 4096, - "dtype_a": "bf16", - "dtype_b": "bf16", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 4096, - "n": 4096, - "k": 1024, - "dtype_a": "bf16", - "dtype_b": "bf16", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 4096, - "n": 128, - "k": 4096, - "dtype_a": "bf16", - "dtype_b": "bf16", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 4096, - "n": 4096, - "k": 128, - "dtype_a": "bf16", - "dtype_b": "bf16", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 4, - "n": 1024, - "k": 4096, - "dtype_a": "bf16", - "dtype_b": "bf16", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 4, - "n": 256, - "k": 4096, - "dtype_a": "bf16", - "dtype_b": "bf16", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 4, - "n": 4096, - "k": 1024, - "dtype_a": "bf16", - "dtype_b": "bf16", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 4, - "n": 128, - "k": 4096, - "dtype_a": "bf16", - "dtype_b": "bf16", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 4, - "n": 4096, - "k": 128, - "dtype_a": "bf16", - "dtype_b": "bf16", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 8192, - "n": 1024, - "k": 4096, - "dtype_a": "bf16", - "dtype_b": "bf16", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 8192, - "n": 256, - "k": 4096, - "dtype_a": "bf16", - "dtype_b": "bf16", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 8192, - "n": 4096, - "k": 1024, - "dtype_a": "bf16", - "dtype_b": "bf16", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 8192, - "n": 128, - "k": 4096, - "dtype_a": "bf16", - "dtype_b": "bf16", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 8192, - "n": 4096, - "k": 128, - "dtype_a": "bf16", - "dtype_b": "bf16", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 8, - "n": 1024, - "k": 4096, - "dtype_a": "bf16", - "dtype_b": "bf16", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 8, - "n": 256, - "k": 4096, - "dtype_a": "bf16", - "dtype_b": "bf16", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 8, - "n": 4096, - "k": 1024, - "dtype_a": "bf16", - "dtype_b": "bf16", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 8, - "n": 128, - "k": 4096, - "dtype_a": "bf16", - "dtype_b": "bf16", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 8, - "n": 4096, - "k": 128, - "dtype_a": "bf16", - "dtype_b": "bf16", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 16384, - "n": 1024, - "k": 4096, - "dtype_a": "bf16", - "dtype_b": "bf16", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 16384, - "n": 256, - "k": 4096, - "dtype_a": "bf16", - "dtype_b": "bf16", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 16384, - "n": 4096, - "k": 1024, - "dtype_a": "bf16", - "dtype_b": "bf16", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 16384, - "n": 128, - "k": 4096, - "dtype_a": "bf16", - "dtype_b": "bf16", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 16384, - "n": 4096, - "k": 128, - "dtype_a": "bf16", - "dtype_b": "bf16", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 16, - "n": 1024, - "k": 4096, - "dtype_a": "bf16", - "dtype_b": "bf16", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 16, - "n": 256, - "k": 4096, - "dtype_a": "bf16", - "dtype_b": "bf16", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 16, - "n": 4096, - "k": 1024, - "dtype_a": "bf16", - "dtype_b": "bf16", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 16, - "n": 128, - "k": 4096, - "dtype_a": "bf16", - "dtype_b": "bf16", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 16, - "n": 4096, - "k": 128, - "dtype_a": "bf16", - "dtype_b": "bf16", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 32768, - "n": 1024, - "k": 4096, - "dtype_a": "bf16", - "dtype_b": "bf16", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 32768, - "n": 256, - "k": 4096, - "dtype_a": "bf16", - "dtype_b": "bf16", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 32768, - "n": 4096, - "k": 1024, - "dtype_a": "bf16", - "dtype_b": "bf16", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 32768, - "n": 128, - "k": 4096, - "dtype_a": "bf16", - "dtype_b": "bf16", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 32768, - "n": 4096, - "k": 128, - "dtype_a": "bf16", - "dtype_b": "bf16", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 32, - "n": 1024, - "k": 4096, - "dtype_a": "bf16", - "dtype_b": "bf16", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 32, - "n": 256, - "k": 4096, - "dtype_a": "bf16", - "dtype_b": "bf16", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 32, - "n": 4096, - "k": 1024, - "dtype_a": "bf16", - "dtype_b": "bf16", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 32, - "n": 128, - "k": 4096, - "dtype_a": "bf16", - "dtype_b": "bf16", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 32, - "n": 4096, - "k": 128, - "dtype_a": "bf16", - "dtype_b": "bf16", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 65536, - "n": 1024, - "k": 4096, - "dtype_a": "bf16", - "dtype_b": "bf16", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 65536, - "n": 256, - "k": 4096, - "dtype_a": "bf16", - "dtype_b": "bf16", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 65536, - "n": 4096, - "k": 1024, - "dtype_a": "bf16", - "dtype_b": "bf16", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 65536, - "n": 128, - "k": 4096, - "dtype_a": "bf16", - "dtype_b": "bf16", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 65536, - "n": 4096, - "k": 128, - "dtype_a": "bf16", - "dtype_b": "bf16", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 64, - "n": 1024, - "k": 4096, - "dtype_a": "bf16", - "dtype_b": "bf16", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 64, - "n": 256, - "k": 4096, - "dtype_a": "bf16", - "dtype_b": "bf16", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 64, - "n": 4096, - "k": 1024, - "dtype_a": "bf16", - "dtype_b": "bf16", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 64, - "n": 128, - "k": 4096, - "dtype_a": "bf16", - "dtype_b": "bf16", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 64, - "n": 4096, - "k": 128, - "dtype_a": "bf16", - "dtype_b": "bf16", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 131072, - "n": 1024, - "k": 4096, - "dtype_a": "bf16", - "dtype_b": "bf16", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 131072, - "n": 256, - "k": 4096, - "dtype_a": "bf16", - "dtype_b": "bf16", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 131072, - "n": 4096, - "k": 1024, - "dtype_a": "bf16", - "dtype_b": "bf16", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 131072, - "n": 128, - "k": 4096, - "dtype_a": "bf16", - "dtype_b": "bf16", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 131072, - "n": 4096, - "k": 128, - "dtype_a": "bf16", - "dtype_b": "bf16", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 262144, - "n": 1024, - "k": 4096, - "dtype_a": "bf16", - "dtype_b": "bf16", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 262144, - "n": 256, - "k": 4096, - "dtype_a": "bf16", - "dtype_b": "bf16", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 262144, - "n": 4096, - "k": 1024, - "dtype_a": "bf16", - "dtype_b": "bf16", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 262144, - "n": 128, - "k": 4096, - "dtype_a": "bf16", - "dtype_b": "bf16", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 262144, - "n": 4096, - "k": 128, - "dtype_a": "bf16", - "dtype_b": "bf16", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 524288, - "n": 1024, - "k": 4096, - "dtype_a": "bf16", - "dtype_b": "bf16", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 524288, - "n": 256, - "k": 4096, - "dtype_a": "bf16", - "dtype_b": "bf16", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 524288, - "n": 4096, - "k": 1024, - "dtype_a": "bf16", - "dtype_b": "bf16", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 524288, - "n": 128, - "k": 4096, - "dtype_a": "bf16", - "dtype_b": "bf16", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 524288, - "n": 4096, - "k": 128, - "dtype_a": "bf16", - "dtype_b": "bf16", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 128, - "n": 1024, - "k": 4096, - "dtype_a": "bf16", - "dtype_b": "bf16", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 128, - "n": 256, - "k": 4096, - "dtype_a": "bf16", - "dtype_b": "bf16", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 128, - "n": 4096, - "k": 1024, - "dtype_a": "bf16", - "dtype_b": "bf16", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 128, - "n": 128, - "k": 4096, - "dtype_a": "bf16", - "dtype_b": "bf16", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 128, - "n": 4096, - "k": 128, - "dtype_a": "bf16", - "dtype_b": "bf16", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 4096, - "n": 1024, - "k": 4096, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 4096, - "n": 256, - "k": 4096, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 4096, - "n": 4096, - "k": 1024, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 4096, - "n": 128, - "k": 4096, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 4096, - "n": 4096, - "k": 128, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 4, - "n": 1024, - "k": 4096, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 4, - "n": 256, - "k": 4096, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 4, - "n": 4096, - "k": 1024, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 4, - "n": 128, - "k": 4096, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 4, - "n": 4096, - "k": 128, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 4096, - "n": 2048, - "k": 4096, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 4096, - "n": 4096, - "k": 2048, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 4096, - "n": 4096, - "k": 256, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 4, - "n": 2048, - "k": 4096, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 4, - "n": 4096, - "k": 2048, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 4, - "n": 4096, - "k": 256, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 8192, - "n": 2048, - "k": 4096, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 8192, - "n": 256, - "k": 4096, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 8192, - "n": 4096, - "k": 2048, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 8192, - "n": 4096, - "k": 256, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 8, - "n": 2048, - "k": 4096, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 8, - "n": 256, - "k": 4096, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 8, - "n": 4096, - "k": 2048, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 8, - "n": 4096, - "k": 256, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 16384, - "n": 2048, - "k": 4096, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 16384, - "n": 256, - "k": 4096, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 16384, - "n": 4096, - "k": 2048, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 16384, - "n": 4096, - "k": 256, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 16, - "n": 2048, - "k": 4096, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 16, - "n": 256, - "k": 4096, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 16, - "n": 4096, - "k": 2048, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 16, - "n": 4096, - "k": 256, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 32768, - "n": 2048, - "k": 4096, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 32768, - "n": 256, - "k": 4096, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 32768, - "n": 4096, - "k": 2048, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 32768, - "n": 4096, - "k": 256, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 32, - "n": 2048, - "k": 4096, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 32, - "n": 256, - "k": 4096, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 32, - "n": 4096, - "k": 2048, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 32, - "n": 4096, - "k": 256, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 65536, - "n": 2048, - "k": 4096, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 65536, - "n": 256, - "k": 4096, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 65536, - "n": 4096, - "k": 2048, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 65536, - "n": 4096, - "k": 256, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 64, - "n": 2048, - "k": 4096, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 64, - "n": 256, - "k": 4096, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 64, - "n": 4096, - "k": 2048, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 64, - "n": 4096, - "k": 256, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 131072, - "n": 2048, - "k": 4096, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 131072, - "n": 256, - "k": 4096, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 131072, - "n": 4096, - "k": 2048, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 131072, - "n": 4096, - "k": 256, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 128, - "n": 2048, - "k": 4096, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 128, - "n": 256, - "k": 4096, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 128, - "n": 4096, - "k": 2048, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 128, - "n": 4096, - "k": 256, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 262144, - "n": 2048, - "k": 4096, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 262144, - "n": 256, - "k": 4096, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 262144, - "n": 4096, - "k": 2048, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 262144, - "n": 4096, - "k": 256, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 256, - "n": 2048, - "k": 4096, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 256, - "n": 256, - "k": 4096, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 256, - "n": 4096, - "k": 2048, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 256, - "n": 4096, - "k": 256, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 32768, - "n": 1024, - "k": 4096, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 32768, - "n": 4096, - "k": 1024, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 32768, - "n": 128, - "k": 4096, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 32768, - "n": 4096, - "k": 128, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 524288, - "n": 2048, - "k": 4096, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 524288, - "n": 256, - "k": 4096, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 524288, - "n": 4096, - "k": 2048, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 524288, - "n": 4096, - "k": 256, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 1048576, - "n": 2048, - "k": 4096, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 1048576, - "n": 256, - "k": 4096, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 1048576, - "n": 4096, - "k": 2048, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 1048576, - "n": 4096, - "k": 256, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 2097152, - "n": 2048, - "k": 4096, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 2097152, - "n": 256, - "k": 4096, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 2097152, - "n": 4096, - "k": 2048, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 2097152, - "n": 4096, - "k": 256, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 4096, - "n": 2048, - "k": 4096, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 4096, - "n": 256, - "k": 4096, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 4096, - "n": 4096, - "k": 2048, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 4096, - "n": 4096, - "k": 256, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 4, - "n": 2048, - "k": 4096, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 4, - "n": 256, - "k": 4096, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 4, - "n": 4096, - "k": 2048, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 4, - "n": 4096, - "k": 256, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 4096, - "n": 4096, - "k": 4096, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 4096, - "n": 512, - "k": 4096, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 4, - "n": 4096, - "k": 4096, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 4, - "n": 512, - "k": 4096, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 8192, - "n": 4096, - "k": 4096, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 8192, - "n": 256, - "k": 4096, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 8192, - "n": 512, - "k": 4096, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 8, - "n": 4096, - "k": 4096, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 8, - "n": 256, - "k": 4096, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 8, - "n": 512, - "k": 4096, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 16384, - "n": 4096, - "k": 4096, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 16384, - "n": 256, - "k": 4096, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 16384, - "n": 512, - "k": 4096, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 16, - "n": 4096, - "k": 4096, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 16, - "n": 256, - "k": 4096, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 16, - "n": 512, - "k": 4096, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 32768, - "n": 4096, - "k": 4096, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 32768, - "n": 256, - "k": 4096, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 32768, - "n": 512, - "k": 4096, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 32, - "n": 4096, - "k": 4096, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 32, - "n": 256, - "k": 4096, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 32, - "n": 512, - "k": 4096, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 65536, - "n": 4096, - "k": 4096, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 65536, - "n": 256, - "k": 4096, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 65536, - "n": 512, - "k": 4096, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 64, - "n": 4096, - "k": 4096, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 64, - "n": 256, - "k": 4096, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 64, - "n": 512, - "k": 4096, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 131072, - "n": 4096, - "k": 4096, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 131072, - "n": 256, - "k": 4096, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 131072, - "n": 512, - "k": 4096, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 128, - "n": 4096, - "k": 4096, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 128, - "n": 256, - "k": 4096, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 128, - "n": 512, - "k": 4096, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 32768, - "n": 2048, - "k": 4096, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 32768, - "n": 4096, - "k": 2048, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 32768, - "n": 4096, - "k": 256, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 262144, - "n": 4096, - "k": 4096, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 262144, - "n": 256, - "k": 4096, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 262144, - "n": 512, - "k": 4096, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 524288, - "n": 4096, - "k": 4096, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 524288, - "n": 256, - "k": 4096, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 524288, - "n": 512, - "k": 4096, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 1048576, - "n": 4096, - "k": 4096, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 1048576, - "n": 256, - "k": 4096, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 1048576, - "n": 512, - "k": 4096, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 8, - "n": 2048, - "k": 4096, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 8, - "n": 4096, - "k": 2048, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 8, - "n": 4096, - "k": 256, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 4096, - "n": 2048, - "k": 6144, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 4096, - "n": 2048, - "k": 2048, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 4096, - "n": 576, - "k": 6144, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 4096, - "n": 3584, - "k": 512, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 4096, - "n": 6144, - "k": 2048, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 4096, - "n": 1536, - "k": 6144, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 4096, - "n": 6144, - "k": 1536, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 4, - "n": 2048, - "k": 6144, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 4, - "n": 2048, - "k": 2048, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 4, - "n": 576, - "k": 6144, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 4, - "n": 3584, - "k": 512, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 4, - "n": 6144, - "k": 2048, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 4, - "n": 1536, - "k": 6144, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 8192, - "n": 2048, - "k": 6144, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 8192, - "n": 2048, - "k": 2048, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 8192, - "n": 576, - "k": 6144, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 8192, - "n": 3584, - "k": 512, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 8192, - "n": 6144, - "k": 2048, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 8192, - "n": 1536, - "k": 6144, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 8192, - "n": 6144, - "k": 1536, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 8, - "n": 2048, - "k": 6144, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 8, - "n": 2048, - "k": 2048, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 8, - "n": 576, - "k": 6144, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 8, - "n": 3584, - "k": 512, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 8, - "n": 6144, - "k": 2048, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 8, - "n": 1536, - "k": 6144, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 16384, - "n": 2048, - "k": 6144, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 16384, - "n": 2048, - "k": 2048, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 16384, - "n": 576, - "k": 6144, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 16384, - "n": 3584, - "k": 512, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 16384, - "n": 6144, - "k": 2048, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 16384, - "n": 1536, - "k": 6144, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 16384, - "n": 6144, - "k": 1536, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 16, - "n": 2048, - "k": 6144, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 16, - "n": 2048, - "k": 2048, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 16, - "n": 576, - "k": 6144, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 16, - "n": 3584, - "k": 512, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 16, - "n": 6144, - "k": 2048, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 16, - "n": 1536, - "k": 6144, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 32768, - "n": 2048, - "k": 6144, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 32768, - "n": 2048, - "k": 2048, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 32768, - "n": 576, - "k": 6144, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 32768, - "n": 3584, - "k": 512, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 32768, - "n": 6144, - "k": 2048, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 32768, - "n": 1536, - "k": 6144, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 32, - "n": 2048, - "k": 6144, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 32, - "n": 2048, - "k": 2048, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 32, - "n": 576, - "k": 6144, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 32, - "n": 3584, - "k": 512, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 32, - "n": 6144, - "k": 2048, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 32, - "n": 1536, - "k": 6144, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 65536, - "n": 2048, - "k": 6144, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 65536, - "n": 2048, - "k": 2048, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 65536, - "n": 576, - "k": 6144, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 65536, - "n": 3584, - "k": 512, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 65536, - "n": 6144, - "k": 2048, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 65536, - "n": 1536, - "k": 6144, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 64, - "n": 2048, - "k": 6144, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 64, - "n": 2048, - "k": 2048, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 64, - "n": 576, - "k": 6144, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 64, - "n": 3584, - "k": 512, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 64, - "n": 6144, - "k": 2048, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 64, - "n": 1536, - "k": 6144, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 64, - "n": 6144, - "k": 1536, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 131072, - "n": 2048, - "k": 6144, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 131072, - "n": 2048, - "k": 2048, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 131072, - "n": 576, - "k": 6144, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 131072, - "n": 3584, - "k": 512, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 131072, - "n": 6144, - "k": 2048, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 131072, - "n": 1536, - "k": 6144, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 128, - "n": 2048, - "k": 6144, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 128, - "n": 2048, - "k": 2048, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 128, - "n": 576, - "k": 6144, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 128, - "n": 3584, - "k": 512, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 128, - "n": 6144, - "k": 2048, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 128, - "n": 1536, - "k": 6144, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 128, - "n": 6144, - "k": 1536, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 262144, - "n": 2048, - "k": 6144, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 262144, - "n": 2048, - "k": 2048, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 262144, - "n": 576, - "k": 6144, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 262144, - "n": 3584, - "k": 512, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 262144, - "n": 6144, - "k": 2048, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 262144, - "n": 1536, - "k": 6144, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 256, - "n": 2048, - "k": 6144, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 256, - "n": 2048, - "k": 2048, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 256, - "n": 576, - "k": 6144, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 256, - "n": 3584, - "k": 512, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 256, - "n": 6144, - "k": 2048, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 256, - "n": 1536, - "k": 6144, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 256, - "n": 6144, - "k": 1536, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 524288, - "n": 2048, - "k": 6144, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 524288, - "n": 2048, - "k": 2048, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 524288, - "n": 576, - "k": 6144, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 524288, - "n": 3584, - "k": 512, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 524288, - "n": 6144, - "k": 2048, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 524288, - "n": 1536, - "k": 6144, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 524288, - "n": 6144, - "k": 1536, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 1048576, - "n": 2048, - "k": 6144, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 1048576, - "n": 2048, - "k": 2048, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 1048576, - "n": 576, - "k": 6144, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 1048576, - "n": 3584, - "k": 512, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 1048576, - "n": 6144, - "k": 2048, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 1048576, - "n": 1536, - "k": 6144, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 1048576, - "n": 6144, - "k": 1536, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 2097152, - "n": 2048, - "k": 6144, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 2097152, - "n": 2048, - "k": 2048, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 2097152, - "n": 576, - "k": 6144, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 2097152, - "n": 3584, - "k": 512, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 2097152, - "n": 6144, - "k": 2048, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 2097152, - "n": 1536, - "k": 6144, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 2097152, - "n": 6144, - "k": 1536, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 512, - "n": 2048, - "k": 6144, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 512, - "n": 2048, - "k": 2048, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 512, - "n": 576, - "k": 6144, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 512, - "n": 3584, - "k": 512, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 512, - "n": 6144, - "k": 2048, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 512, - "n": 1536, - "k": 6144, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 512, - "n": 6144, - "k": 1536, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 4096, - "n": 2048, - "k": 6144, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 4096, - "n": 2048, - "k": 2048, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 4096, - "n": 576, - "k": 6144, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 4096, - "n": 3584, - "k": 512, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 4096, - "n": 6144, - "k": 2048, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 4096, - "n": 1536, - "k": 6144, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 4, - "n": 2048, - "k": 6144, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 4, - "n": 2048, - "k": 2048, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 4, - "n": 576, - "k": 6144, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 4, - "n": 3584, - "k": 512, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 4, - "n": 6144, - "k": 2048, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 4, - "n": 1536, - "k": 6144, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 4096, - "n": 7168, - "k": 512, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 4096, - "n": 6144, - "k": 4096, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 4096, - "n": 3072, - "k": 6144, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 4096, - "n": 6144, - "k": 3072, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 4, - "n": 7168, - "k": 512, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 4, - "n": 6144, - "k": 4096, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 4, - "n": 3072, - "k": 6144, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 4, - "n": 6144, - "k": 3072, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 8192, - "n": 2048, - "k": 6144, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 8192, - "n": 4096, - "k": 2048, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 8192, - "n": 576, - "k": 6144, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 8192, - "n": 7168, - "k": 512, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 8192, - "n": 6144, - "k": 4096, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 8192, - "n": 3072, - "k": 6144, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 8192, - "n": 6144, - "k": 3072, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 8, - "n": 2048, - "k": 6144, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 8, - "n": 576, - "k": 6144, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 8, - "n": 7168, - "k": 512, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 8, - "n": 6144, - "k": 4096, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 8, - "n": 3072, - "k": 6144, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 8, - "n": 6144, - "k": 3072, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 16384, - "n": 2048, - "k": 6144, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 16384, - "n": 4096, - "k": 2048, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 16384, - "n": 576, - "k": 6144, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 16384, - "n": 7168, - "k": 512, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 16384, - "n": 6144, - "k": 4096, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 16384, - "n": 3072, - "k": 6144, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 16384, - "n": 6144, - "k": 3072, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 16, - "n": 2048, - "k": 6144, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 16, - "n": 4096, - "k": 2048, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 16, - "n": 576, - "k": 6144, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 16, - "n": 7168, - "k": 512, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 16, - "n": 6144, - "k": 4096, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 16, - "n": 3072, - "k": 6144, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 16, - "n": 6144, - "k": 3072, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 32768, - "n": 2048, - "k": 6144, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 32768, - "n": 576, - "k": 6144, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 32768, - "n": 7168, - "k": 512, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 32768, - "n": 6144, - "k": 4096, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 32768, - "n": 3072, - "k": 6144, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 32768, - "n": 6144, - "k": 3072, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 32, - "n": 2048, - "k": 6144, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 32, - "n": 4096, - "k": 2048, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 32, - "n": 576, - "k": 6144, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 32, - "n": 7168, - "k": 512, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 32, - "n": 6144, - "k": 4096, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 32, - "n": 3072, - "k": 6144, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 32, - "n": 6144, - "k": 3072, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 65536, - "n": 2048, - "k": 6144, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 65536, - "n": 4096, - "k": 2048, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 65536, - "n": 576, - "k": 6144, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 65536, - "n": 7168, - "k": 512, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 65536, - "n": 6144, - "k": 4096, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 65536, - "n": 3072, - "k": 6144, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 65536, - "n": 6144, - "k": 3072, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 64, - "n": 2048, - "k": 6144, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 64, - "n": 4096, - "k": 2048, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 64, - "n": 576, - "k": 6144, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 64, - "n": 7168, - "k": 512, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 64, - "n": 6144, - "k": 4096, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 64, - "n": 3072, - "k": 6144, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 64, - "n": 6144, - "k": 3072, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 131072, - "n": 2048, - "k": 6144, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 131072, - "n": 4096, - "k": 2048, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 131072, - "n": 576, - "k": 6144, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 131072, - "n": 7168, - "k": 512, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 131072, - "n": 6144, - "k": 4096, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 131072, - "n": 3072, - "k": 6144, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 131072, - "n": 6144, - "k": 3072, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 128, - "n": 2048, - "k": 6144, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 128, - "n": 4096, - "k": 2048, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 128, - "n": 576, - "k": 6144, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 128, - "n": 7168, - "k": 512, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 128, - "n": 6144, - "k": 4096, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 128, - "n": 3072, - "k": 6144, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 128, - "n": 6144, - "k": 3072, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 262144, - "n": 2048, - "k": 6144, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 262144, - "n": 4096, - "k": 2048, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 262144, - "n": 576, - "k": 6144, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 262144, - "n": 7168, - "k": 512, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 262144, - "n": 6144, - "k": 4096, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 262144, - "n": 3072, - "k": 6144, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 262144, - "n": 6144, - "k": 3072, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 256, - "n": 2048, - "k": 6144, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 256, - "n": 4096, - "k": 2048, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 256, - "n": 576, - "k": 6144, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 256, - "n": 7168, - "k": 512, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 256, - "n": 6144, - "k": 4096, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 256, - "n": 3072, - "k": 6144, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 256, - "n": 6144, - "k": 3072, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 32768, - "n": 2048, - "k": 2048, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 32768, - "n": 3584, - "k": 512, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 32768, - "n": 6144, - "k": 2048, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 32768, - "n": 1536, - "k": 6144, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 524288, - "n": 2048, - "k": 6144, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 524288, - "n": 4096, - "k": 2048, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 524288, - "n": 576, - "k": 6144, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 524288, - "n": 7168, - "k": 512, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 524288, - "n": 6144, - "k": 4096, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 524288, - "n": 3072, - "k": 6144, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 524288, - "n": 6144, - "k": 3072, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 1048576, - "n": 2048, - "k": 6144, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 1048576, - "n": 4096, - "k": 2048, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 1048576, - "n": 576, - "k": 6144, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 1048576, - "n": 7168, - "k": 512, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 1048576, - "n": 6144, - "k": 4096, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 1048576, - "n": 3072, - "k": 6144, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 1048576, - "n": 6144, - "k": 3072, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 2097152, - "n": 2048, - "k": 6144, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 2097152, - "n": 4096, - "k": 2048, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 2097152, - "n": 576, - "k": 6144, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 2097152, - "n": 7168, - "k": 512, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 2097152, - "n": 6144, - "k": 4096, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 2097152, - "n": 3072, - "k": 6144, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 2097152, - "n": 6144, - "k": 3072, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 8, - "n": 2048, - "k": 2048, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 8, - "n": 3584, - "k": 512, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 8, - "n": 6144, - "k": 2048, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 8, - "n": 1536, - "k": 6144, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 512, - "n": 2048, - "k": 6144, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 512, - "n": 4096, - "k": 2048, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 512, - "n": 576, - "k": 6144, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 512, - "n": 7168, - "k": 512, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 512, - "n": 6144, - "k": 4096, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 512, - "n": 3072, - "k": 6144, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 512, - "n": 6144, - "k": 3072, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 8, - "n": 1024, - "k": 4096, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 8, - "n": 4096, - "k": 1024, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 8, - "n": 128, - "k": 4096, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 8, - "n": 4096, - "k": 128, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 512, - "n": 2048, - "k": 4096, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 512, - "n": 256, - "k": 4096, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 512, - "n": 4096, - "k": 2048, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 512, - "n": 4096, - "k": 256, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 8192, - "n": 2048, - "k": 4096, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 8192, - "n": 4096, - "k": 256, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 16384, - "n": 2048, - "k": 4096, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 16384, - "n": 4096, - "k": 256, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 16, - "n": 2048, - "k": 4096, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 16, - "n": 4096, - "k": 256, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 32, - "n": 2048, - "k": 4096, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 32, - "n": 4096, - "k": 256, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 65536, - "n": 2048, - "k": 4096, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 65536, - "n": 4096, - "k": 256, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 64, - "n": 2048, - "k": 4096, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 64, - "n": 4096, - "k": 256, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 131072, - "n": 2048, - "k": 4096, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 131072, - "n": 4096, - "k": 256, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 128, - "n": 2048, - "k": 4096, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 128, - "n": 4096, - "k": 256, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 262144, - "n": 2048, - "k": 4096, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 262144, - "n": 4096, - "k": 256, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 524288, - "n": 2048, - "k": 4096, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 524288, - "n": 4096, - "k": 256, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 1048576, - "n": 2048, - "k": 4096, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 1048576, - "n": 4096, - "k": 256, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 256, - "n": 2048, - "k": 4096, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 256, - "n": 256, - "k": 4096, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 256, - "n": 4096, - "k": 256, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 256, - "n": 4096, - "k": 4096, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 256, - "n": 512, - "k": 4096, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 4096, - "n": 2048, - "k": 4096, - "dtype_a": "bf16", - "dtype_b": "bf16", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 4096, - "n": 4096, - "k": 2048, - "dtype_a": "bf16", - "dtype_b": "bf16", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 4096, - "n": 4096, - "k": 256, - "dtype_a": "bf16", - "dtype_b": "bf16", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 4, - "n": 2048, - "k": 4096, - "dtype_a": "bf16", - "dtype_b": "bf16", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 4, - "n": 4096, - "k": 2048, - "dtype_a": "bf16", - "dtype_b": "bf16", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 4, - "n": 4096, - "k": 256, - "dtype_a": "bf16", - "dtype_b": "bf16", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 8192, - "n": 2048, - "k": 4096, - "dtype_a": "bf16", - "dtype_b": "bf16", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 8192, - "n": 4096, - "k": 2048, - "dtype_a": "bf16", - "dtype_b": "bf16", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 8192, - "n": 4096, - "k": 256, - "dtype_a": "bf16", - "dtype_b": "bf16", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 8, - "n": 2048, - "k": 4096, - "dtype_a": "bf16", - "dtype_b": "bf16", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 8, - "n": 4096, - "k": 2048, - "dtype_a": "bf16", - "dtype_b": "bf16", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 8, - "n": 4096, - "k": 256, - "dtype_a": "bf16", - "dtype_b": "bf16", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 16384, - "n": 2048, - "k": 4096, - "dtype_a": "bf16", - "dtype_b": "bf16", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 16384, - "n": 4096, - "k": 2048, - "dtype_a": "bf16", - "dtype_b": "bf16", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 16384, - "n": 4096, - "k": 256, - "dtype_a": "bf16", - "dtype_b": "bf16", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 16, - "n": 2048, - "k": 4096, - "dtype_a": "bf16", - "dtype_b": "bf16", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 16, - "n": 4096, - "k": 2048, - "dtype_a": "bf16", - "dtype_b": "bf16", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 16, - "n": 4096, - "k": 256, - "dtype_a": "bf16", - "dtype_b": "bf16", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 32768, - "n": 2048, - "k": 4096, - "dtype_a": "bf16", - "dtype_b": "bf16", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 32768, - "n": 4096, - "k": 2048, - "dtype_a": "bf16", - "dtype_b": "bf16", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 32768, - "n": 4096, - "k": 256, - "dtype_a": "bf16", - "dtype_b": "bf16", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 32, - "n": 2048, - "k": 4096, - "dtype_a": "bf16", - "dtype_b": "bf16", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 32, - "n": 4096, - "k": 2048, - "dtype_a": "bf16", - "dtype_b": "bf16", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 32, - "n": 4096, - "k": 256, - "dtype_a": "bf16", - "dtype_b": "bf16", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 65536, - "n": 2048, - "k": 4096, - "dtype_a": "bf16", - "dtype_b": "bf16", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 65536, - "n": 4096, - "k": 2048, - "dtype_a": "bf16", - "dtype_b": "bf16", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 65536, - "n": 4096, - "k": 256, - "dtype_a": "bf16", - "dtype_b": "bf16", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 64, - "n": 2048, - "k": 4096, - "dtype_a": "bf16", - "dtype_b": "bf16", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 64, - "n": 4096, - "k": 2048, - "dtype_a": "bf16", - "dtype_b": "bf16", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 64, - "n": 4096, - "k": 256, - "dtype_a": "bf16", - "dtype_b": "bf16", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 131072, - "n": 2048, - "k": 4096, - "dtype_a": "bf16", - "dtype_b": "bf16", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 131072, - "n": 4096, - "k": 2048, - "dtype_a": "bf16", - "dtype_b": "bf16", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 131072, - "n": 4096, - "k": 256, - "dtype_a": "bf16", - "dtype_b": "bf16", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 262144, - "n": 2048, - "k": 4096, - "dtype_a": "bf16", - "dtype_b": "bf16", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 262144, - "n": 4096, - "k": 2048, - "dtype_a": "bf16", - "dtype_b": "bf16", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 262144, - "n": 4096, - "k": 256, - "dtype_a": "bf16", - "dtype_b": "bf16", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 524288, - "n": 2048, - "k": 4096, - "dtype_a": "bf16", - "dtype_b": "bf16", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 524288, - "n": 4096, - "k": 2048, - "dtype_a": "bf16", - "dtype_b": "bf16", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 524288, - "n": 4096, - "k": 256, - "dtype_a": "bf16", - "dtype_b": "bf16", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 128, - "n": 2048, - "k": 4096, - "dtype_a": "bf16", - "dtype_b": "bf16", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 128, - "n": 4096, - "k": 2048, - "dtype_a": "bf16", - "dtype_b": "bf16", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 128, - "n": 4096, - "k": 256, - "dtype_a": "bf16", - "dtype_b": "bf16", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 4096, - "n": 1536, - "k": 7168, - "dtype_a": "int4", - "dtype_b": "int4", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 4096, - "n": 1536, - "k": 1536, - "dtype_a": "int4", - "dtype_b": "int4", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 4096, - "n": 576, - "k": 7168, - "dtype_a": "int4", - "dtype_b": "int4", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 4096, - "n": 2048, - "k": 512, - "dtype_a": "int4", - "dtype_b": "int4", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 4096, - "n": 7168, - "k": 1024, - "dtype_a": "int4", - "dtype_b": "int4", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 4096, - "n": 2304, - "k": 7168, - "dtype_a": "int4", - "dtype_b": "int4", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 4096, - "n": 7168, - "k": 2304, - "dtype_a": "int4", - "dtype_b": "int4", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 4, - "n": 1536, - "k": 7168, - "dtype_a": "int4", - "dtype_b": "int4", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 4, - "n": 1536, - "k": 1536, - "dtype_a": "int4", - "dtype_b": "int4", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 4, - "n": 576, - "k": 7168, - "dtype_a": "int4", - "dtype_b": "int4", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 4, - "n": 2048, - "k": 512, - "dtype_a": "int4", - "dtype_b": "int4", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 4, - "n": 7168, - "k": 1024, - "dtype_a": "int4", - "dtype_b": "int4", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 4, - "n": 2304, - "k": 7168, - "dtype_a": "int4", - "dtype_b": "int4", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 4, - "n": 7168, - "k": 2304, - "dtype_a": "int4", - "dtype_b": "int4", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 8192, - "n": 1536, - "k": 7168, - "dtype_a": "int4", - "dtype_b": "int4", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 8192, - "n": 1536, - "k": 1536, - "dtype_a": "int4", - "dtype_b": "int4", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 8192, - "n": 576, - "k": 7168, - "dtype_a": "int4", - "dtype_b": "int4", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 8192, - "n": 2048, - "k": 512, - "dtype_a": "int4", - "dtype_b": "int4", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 8192, - "n": 7168, - "k": 1024, - "dtype_a": "int4", - "dtype_b": "int4", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 8192, - "n": 2304, - "k": 7168, - "dtype_a": "int4", - "dtype_b": "int4", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 8192, - "n": 7168, - "k": 2304, - "dtype_a": "int4", - "dtype_b": "int4", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 8, - "n": 1536, - "k": 7168, - "dtype_a": "int4", - "dtype_b": "int4", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 8, - "n": 1536, - "k": 1536, - "dtype_a": "int4", - "dtype_b": "int4", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 8, - "n": 576, - "k": 7168, - "dtype_a": "int4", - "dtype_b": "int4", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 8, - "n": 2048, - "k": 512, - "dtype_a": "int4", - "dtype_b": "int4", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 8, - "n": 7168, - "k": 1024, - "dtype_a": "int4", - "dtype_b": "int4", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 8, - "n": 2304, - "k": 7168, - "dtype_a": "int4", - "dtype_b": "int4", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 8, - "n": 7168, - "k": 2304, - "dtype_a": "int4", - "dtype_b": "int4", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 16384, - "n": 1536, - "k": 7168, - "dtype_a": "int4", - "dtype_b": "int4", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 16384, - "n": 1536, - "k": 1536, - "dtype_a": "int4", - "dtype_b": "int4", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 16384, - "n": 576, - "k": 7168, - "dtype_a": "int4", - "dtype_b": "int4", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 16384, - "n": 2048, - "k": 512, - "dtype_a": "int4", - "dtype_b": "int4", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 16384, - "n": 7168, - "k": 1024, - "dtype_a": "int4", - "dtype_b": "int4", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 16384, - "n": 2304, - "k": 7168, - "dtype_a": "int4", - "dtype_b": "int4", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 16384, - "n": 7168, - "k": 2304, - "dtype_a": "int4", - "dtype_b": "int4", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 16, - "n": 1536, - "k": 7168, - "dtype_a": "int4", - "dtype_b": "int4", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 16, - "n": 1536, - "k": 1536, - "dtype_a": "int4", - "dtype_b": "int4", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 16, - "n": 576, - "k": 7168, - "dtype_a": "int4", - "dtype_b": "int4", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 16, - "n": 2048, - "k": 512, - "dtype_a": "int4", - "dtype_b": "int4", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 16, - "n": 7168, - "k": 1024, - "dtype_a": "int4", - "dtype_b": "int4", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 16, - "n": 2304, - "k": 7168, - "dtype_a": "int4", - "dtype_b": "int4", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 16, - "n": 7168, - "k": 2304, - "dtype_a": "int4", - "dtype_b": "int4", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 32768, - "n": 1536, - "k": 7168, - "dtype_a": "int4", - "dtype_b": "int4", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 32768, - "n": 1536, - "k": 1536, - "dtype_a": "int4", - "dtype_b": "int4", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 32768, - "n": 576, - "k": 7168, - "dtype_a": "int4", - "dtype_b": "int4", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 32768, - "n": 2048, - "k": 512, - "dtype_a": "int4", - "dtype_b": "int4", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 32768, - "n": 7168, - "k": 1024, - "dtype_a": "int4", - "dtype_b": "int4", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 32768, - "n": 2304, - "k": 7168, - "dtype_a": "int4", - "dtype_b": "int4", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 32768, - "n": 7168, - "k": 2304, - "dtype_a": "int4", - "dtype_b": "int4", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 32, - "n": 1536, - "k": 7168, - "dtype_a": "int4", - "dtype_b": "int4", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 32, - "n": 1536, - "k": 1536, - "dtype_a": "int4", - "dtype_b": "int4", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 32, - "n": 576, - "k": 7168, - "dtype_a": "int4", - "dtype_b": "int4", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 32, - "n": 2048, - "k": 512, - "dtype_a": "int4", - "dtype_b": "int4", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 32, - "n": 7168, - "k": 1024, - "dtype_a": "int4", - "dtype_b": "int4", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 32, - "n": 2304, - "k": 7168, - "dtype_a": "int4", - "dtype_b": "int4", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 32, - "n": 7168, - "k": 2304, - "dtype_a": "int4", - "dtype_b": "int4", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 65536, - "n": 1536, - "k": 7168, - "dtype_a": "int4", - "dtype_b": "int4", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 65536, - "n": 1536, - "k": 1536, - "dtype_a": "int4", - "dtype_b": "int4", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 65536, - "n": 576, - "k": 7168, - "dtype_a": "int4", - "dtype_b": "int4", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 65536, - "n": 2048, - "k": 512, - "dtype_a": "int4", - "dtype_b": "int4", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 65536, - "n": 7168, - "k": 1024, - "dtype_a": "int4", - "dtype_b": "int4", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 65536, - "n": 2304, - "k": 7168, - "dtype_a": "int4", - "dtype_b": "int4", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 65536, - "n": 7168, - "k": 2304, - "dtype_a": "int4", - "dtype_b": "int4", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 64, - "n": 1536, - "k": 7168, - "dtype_a": "int4", - "dtype_b": "int4", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 64, - "n": 1536, - "k": 1536, - "dtype_a": "int4", - "dtype_b": "int4", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 64, - "n": 576, - "k": 7168, - "dtype_a": "int4", - "dtype_b": "int4", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 64, - "n": 2048, - "k": 512, - "dtype_a": "int4", - "dtype_b": "int4", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 64, - "n": 7168, - "k": 1024, - "dtype_a": "int4", - "dtype_b": "int4", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 64, - "n": 2304, - "k": 7168, - "dtype_a": "int4", - "dtype_b": "int4", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 64, - "n": 7168, - "k": 2304, - "dtype_a": "int4", - "dtype_b": "int4", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 131072, - "n": 1536, - "k": 7168, - "dtype_a": "int4", - "dtype_b": "int4", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 131072, - "n": 1536, - "k": 1536, - "dtype_a": "int4", - "dtype_b": "int4", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 131072, - "n": 576, - "k": 7168, - "dtype_a": "int4", - "dtype_b": "int4", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 131072, - "n": 2048, - "k": 512, - "dtype_a": "int4", - "dtype_b": "int4", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 131072, - "n": 7168, - "k": 1024, - "dtype_a": "int4", - "dtype_b": "int4", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 131072, - "n": 2304, - "k": 7168, - "dtype_a": "int4", - "dtype_b": "int4", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 131072, - "n": 7168, - "k": 2304, - "dtype_a": "int4", - "dtype_b": "int4", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 262144, - "n": 1536, - "k": 7168, - "dtype_a": "int4", - "dtype_b": "int4", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 262144, - "n": 1536, - "k": 1536, - "dtype_a": "int4", - "dtype_b": "int4", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 262144, - "n": 576, - "k": 7168, - "dtype_a": "int4", - "dtype_b": "int4", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 262144, - "n": 2048, - "k": 512, - "dtype_a": "int4", - "dtype_b": "int4", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 262144, - "n": 7168, - "k": 1024, - "dtype_a": "int4", - "dtype_b": "int4", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 262144, - "n": 2304, - "k": 7168, - "dtype_a": "int4", - "dtype_b": "int4", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 262144, - "n": 7168, - "k": 2304, - "dtype_a": "int4", - "dtype_b": "int4", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 524288, - "n": 1536, - "k": 7168, - "dtype_a": "int4", - "dtype_b": "int4", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 524288, - "n": 1536, - "k": 1536, - "dtype_a": "int4", - "dtype_b": "int4", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 524288, - "n": 576, - "k": 7168, - "dtype_a": "int4", - "dtype_b": "int4", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 524288, - "n": 2048, - "k": 512, - "dtype_a": "int4", - "dtype_b": "int4", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 524288, - "n": 7168, - "k": 1024, - "dtype_a": "int4", - "dtype_b": "int4", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 524288, - "n": 2304, - "k": 7168, - "dtype_a": "int4", - "dtype_b": "int4", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 524288, - "n": 7168, - "k": 2304, - "dtype_a": "int4", - "dtype_b": "int4", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 4096, - "n": 3072, - "k": 1536, - "dtype_a": "int4", - "dtype_b": "int4", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 4096, - "n": 4096, - "k": 512, - "dtype_a": "int4", - "dtype_b": "int4", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 4096, - "n": 7168, - "k": 2048, - "dtype_a": "int4", - "dtype_b": "int4", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 4096, - "n": 4608, - "k": 7168, - "dtype_a": "int4", - "dtype_b": "int4", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 4096, - "n": 7168, - "k": 4608, - "dtype_a": "int4", - "dtype_b": "int4", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 4, - "n": 3072, - "k": 1536, - "dtype_a": "int4", - "dtype_b": "int4", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 4, - "n": 4096, - "k": 512, - "dtype_a": "int4", - "dtype_b": "int4", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 4, - "n": 7168, - "k": 2048, - "dtype_a": "int4", - "dtype_b": "int4", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 4, - "n": 4608, - "k": 7168, - "dtype_a": "int4", - "dtype_b": "int4", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 4, - "n": 7168, - "k": 4608, - "dtype_a": "int4", - "dtype_b": "int4", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 8192, - "n": 3072, - "k": 1536, - "dtype_a": "int4", - "dtype_b": "int4", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 8192, - "n": 4096, - "k": 512, - "dtype_a": "int4", - "dtype_b": "int4", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 8192, - "n": 7168, - "k": 2048, - "dtype_a": "int4", - "dtype_b": "int4", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 8192, - "n": 4608, - "k": 7168, - "dtype_a": "int4", - "dtype_b": "int4", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 8192, - "n": 7168, - "k": 4608, - "dtype_a": "int4", - "dtype_b": "int4", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 8, - "n": 3072, - "k": 1536, - "dtype_a": "int4", - "dtype_b": "int4", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 8, - "n": 4096, - "k": 512, - "dtype_a": "int4", - "dtype_b": "int4", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 8, - "n": 7168, - "k": 2048, - "dtype_a": "int4", - "dtype_b": "int4", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 8, - "n": 4608, - "k": 7168, - "dtype_a": "int4", - "dtype_b": "int4", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 8, - "n": 7168, - "k": 4608, - "dtype_a": "int4", - "dtype_b": "int4", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 16384, - "n": 3072, - "k": 1536, - "dtype_a": "int4", - "dtype_b": "int4", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 16384, - "n": 4096, - "k": 512, - "dtype_a": "int4", - "dtype_b": "int4", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 16384, - "n": 7168, - "k": 2048, - "dtype_a": "int4", - "dtype_b": "int4", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 16384, - "n": 4608, - "k": 7168, - "dtype_a": "int4", - "dtype_b": "int4", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 16384, - "n": 7168, - "k": 4608, - "dtype_a": "int4", - "dtype_b": "int4", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 16, - "n": 3072, - "k": 1536, - "dtype_a": "int4", - "dtype_b": "int4", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 16, - "n": 4096, - "k": 512, - "dtype_a": "int4", - "dtype_b": "int4", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 16, - "n": 7168, - "k": 2048, - "dtype_a": "int4", - "dtype_b": "int4", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 16, - "n": 4608, - "k": 7168, - "dtype_a": "int4", - "dtype_b": "int4", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 16, - "n": 7168, - "k": 4608, - "dtype_a": "int4", - "dtype_b": "int4", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 32768, - "n": 3072, - "k": 1536, - "dtype_a": "int4", - "dtype_b": "int4", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 32768, - "n": 4096, - "k": 512, - "dtype_a": "int4", - "dtype_b": "int4", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 32768, - "n": 7168, - "k": 2048, - "dtype_a": "int4", - "dtype_b": "int4", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 32768, - "n": 4608, - "k": 7168, - "dtype_a": "int4", - "dtype_b": "int4", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 32768, - "n": 7168, - "k": 4608, - "dtype_a": "int4", - "dtype_b": "int4", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 32, - "n": 3072, - "k": 1536, - "dtype_a": "int4", - "dtype_b": "int4", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 32, - "n": 4096, - "k": 512, - "dtype_a": "int4", - "dtype_b": "int4", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 32, - "n": 7168, - "k": 2048, - "dtype_a": "int4", - "dtype_b": "int4", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 32, - "n": 4608, - "k": 7168, - "dtype_a": "int4", - "dtype_b": "int4", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 32, - "n": 7168, - "k": 4608, - "dtype_a": "int4", - "dtype_b": "int4", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 65536, - "n": 3072, - "k": 1536, - "dtype_a": "int4", - "dtype_b": "int4", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 65536, - "n": 4096, - "k": 512, - "dtype_a": "int4", - "dtype_b": "int4", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 65536, - "n": 7168, - "k": 2048, - "dtype_a": "int4", - "dtype_b": "int4", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 65536, - "n": 4608, - "k": 7168, - "dtype_a": "int4", - "dtype_b": "int4", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 65536, - "n": 7168, - "k": 4608, - "dtype_a": "int4", - "dtype_b": "int4", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 64, - "n": 3072, - "k": 1536, - "dtype_a": "int4", - "dtype_b": "int4", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 64, - "n": 4096, - "k": 512, - "dtype_a": "int4", - "dtype_b": "int4", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 64, - "n": 7168, - "k": 2048, - "dtype_a": "int4", - "dtype_b": "int4", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 64, - "n": 4608, - "k": 7168, - "dtype_a": "int4", - "dtype_b": "int4", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 64, - "n": 7168, - "k": 4608, - "dtype_a": "int4", - "dtype_b": "int4", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 131072, - "n": 3072, - "k": 1536, - "dtype_a": "int4", - "dtype_b": "int4", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 131072, - "n": 4096, - "k": 512, - "dtype_a": "int4", - "dtype_b": "int4", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 131072, - "n": 7168, - "k": 2048, - "dtype_a": "int4", - "dtype_b": "int4", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 131072, - "n": 4608, - "k": 7168, - "dtype_a": "int4", - "dtype_b": "int4", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 131072, - "n": 7168, - "k": 4608, - "dtype_a": "int4", - "dtype_b": "int4", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 262144, - "n": 3072, - "k": 1536, - "dtype_a": "int4", - "dtype_b": "int4", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 262144, - "n": 4096, - "k": 512, - "dtype_a": "int4", - "dtype_b": "int4", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 262144, - "n": 7168, - "k": 2048, - "dtype_a": "int4", - "dtype_b": "int4", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 262144, - "n": 4608, - "k": 7168, - "dtype_a": "int4", - "dtype_b": "int4", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 262144, - "n": 7168, - "k": 4608, - "dtype_a": "int4", - "dtype_b": "int4", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 524288, - "n": 3072, - "k": 1536, - "dtype_a": "int4", - "dtype_b": "int4", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 524288, - "n": 4096, - "k": 512, - "dtype_a": "int4", - "dtype_b": "int4", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 524288, - "n": 7168, - "k": 2048, - "dtype_a": "int4", - "dtype_b": "int4", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 524288, - "n": 4608, - "k": 7168, - "dtype_a": "int4", - "dtype_b": "int4", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 524288, - "n": 7168, - "k": 4608, - "dtype_a": "int4", - "dtype_b": "int4", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 4096, - "n": 1536, - "k": 1536, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 4096, - "n": 2048, - "k": 512, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 4096, - "n": 7168, - "k": 1024, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 4, - "n": 1536, - "k": 1536, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 4, - "n": 2048, - "k": 512, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 4, - "n": 7168, - "k": 1024, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 32768, - "n": 1536, - "k": 1536, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 32768, - "n": 2048, - "k": 512, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 32768, - "n": 7168, - "k": 1024, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 1024, - "n": 3072, - "k": 1536, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 1024, - "n": 4096, - "k": 512, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 1024, - "n": 7168, - "k": 2048, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 4194304, - "n": 3072, - "k": 1536, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 4194304, - "n": 4096, - "k": 512, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 4194304, - "n": 7168, - "k": 2048, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 4194304, - "n": 2304, - "k": 7168, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 4194304, - "n": 7168, - "k": 2304, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 8192, - "n": 8192, - "k": 512, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 8192, - "n": 7168, - "k": 4096, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 8192, - "n": 4608, - "k": 7168, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 8192, - "n": 7168, - "k": 4608, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 16384, - "n": 8192, - "k": 512, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 16384, - "n": 7168, - "k": 4096, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 1024, - "n": 8192, - "k": 1536, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 1024, - "n": 1088, - "k": 7168, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 1024, - "n": 15360, - "k": 1024, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 1024, - "n": 7168, - "k": 8192, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 1, - "n": 8192, - "k": 1536, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 1, - "n": 1088, - "k": 7168, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 1, - "n": 15360, - "k": 1024, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 1, - "n": 7168, - "k": 8192, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 2048, - "n": 8192, - "k": 1536, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 2048, - "n": 1088, - "k": 7168, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 2048, - "n": 15360, - "k": 1024, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 2048, - "n": 7168, - "k": 8192, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 2, - "n": 8192, - "k": 1536, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 2, - "n": 1088, - "k": 7168, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 2, - "n": 15360, - "k": 1024, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 2, - "n": 7168, - "k": 8192, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 4096, - "n": 8192, - "k": 1536, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 4096, - "n": 1088, - "k": 7168, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 4096, - "n": 15360, - "k": 1024, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 4096, - "n": 7168, - "k": 8192, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 4, - "n": 8192, - "k": 1536, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 4, - "n": 1088, - "k": 7168, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 4, - "n": 15360, - "k": 1024, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 4, - "n": 7168, - "k": 8192, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 8192, - "n": 8192, - "k": 1536, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 8192, - "n": 1088, - "k": 7168, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 8192, - "n": 15360, - "k": 1024, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 8192, - "n": 7168, - "k": 8192, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 8, - "n": 8192, - "k": 1536, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 8, - "n": 1088, - "k": 7168, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 8, - "n": 15360, - "k": 1024, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 8, - "n": 7168, - "k": 8192, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 16384, - "n": 8192, - "k": 1536, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 16384, - "n": 1088, - "k": 7168, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 16384, - "n": 15360, - "k": 1024, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 16384, - "n": 7168, - "k": 8192, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 16, - "n": 8192, - "k": 1536, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 16, - "n": 1088, - "k": 7168, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 16, - "n": 15360, - "k": 1024, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 16, - "n": 7168, - "k": 8192, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 32768, - "n": 8192, - "k": 1536, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 32768, - "n": 1088, - "k": 7168, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 32768, - "n": 15360, - "k": 1024, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 32768, - "n": 7168, - "k": 8192, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 32, - "n": 8192, - "k": 1536, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 32, - "n": 1088, - "k": 7168, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 32, - "n": 15360, - "k": 1024, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 32, - "n": 7168, - "k": 8192, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 65536, - "n": 8192, - "k": 1536, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 65536, - "n": 1088, - "k": 7168, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 65536, - "n": 15360, - "k": 1024, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 65536, - "n": 7168, - "k": 8192, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 64, - "n": 8192, - "k": 1536, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 64, - "n": 1088, - "k": 7168, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 64, - "n": 15360, - "k": 1024, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 64, - "n": 7168, - "k": 8192, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 131072, - "n": 8192, - "k": 1536, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 131072, - "n": 1088, - "k": 7168, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 131072, - "n": 15360, - "k": 1024, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 131072, - "n": 7168, - "k": 8192, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 128, - "n": 8192, - "k": 1536, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 128, - "n": 1088, - "k": 7168, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 128, - "n": 15360, - "k": 1024, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 128, - "n": 7168, - "k": 8192, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 262144, - "n": 8192, - "k": 1536, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 262144, - "n": 1088, - "k": 7168, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 262144, - "n": 15360, - "k": 1024, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 262144, - "n": 7168, - "k": 8192, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 256, - "n": 8192, - "k": 1536, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 256, - "n": 1088, - "k": 7168, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 256, - "n": 15360, - "k": 1024, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 256, - "n": 7168, - "k": 8192, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 128, - "n": 65536, - "k": 1536, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 128, - "n": 122880, - "k": 1024, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 128, - "n": 7168, - "k": 65536, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 0, - "n": 65536, - "k": 1536, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 0, - "n": 1088, - "k": 7168, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 0, - "n": 122880, - "k": 1024, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 0, - "n": 7168, - "k": 65536, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 256, - "n": 65536, - "k": 1536, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 256, - "n": 122880, - "k": 1024, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 256, - "n": 7168, - "k": 65536, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 512, - "n": 65536, - "k": 1536, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 512, - "n": 1088, - "k": 7168, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 512, - "n": 122880, - "k": 1024, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 512, - "n": 7168, - "k": 65536, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 1024, - "n": 65536, - "k": 1536, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 1024, - "n": 122880, - "k": 1024, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 1024, - "n": 7168, - "k": 65536, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 1, - "n": 65536, - "k": 1536, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 1, - "n": 122880, - "k": 1024, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 1, - "n": 7168, - "k": 65536, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 2048, - "n": 65536, - "k": 1536, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 2048, - "n": 122880, - "k": 1024, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 2048, - "n": 7168, - "k": 65536, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 2, - "n": 65536, - "k": 1536, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 2, - "n": 122880, - "k": 1024, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 2, - "n": 7168, - "k": 65536, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 4096, - "n": 65536, - "k": 1536, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 4096, - "n": 122880, - "k": 1024, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 4096, - "n": 7168, - "k": 65536, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 4, - "n": 65536, - "k": 1536, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 4, - "n": 122880, - "k": 1024, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 4, - "n": 7168, - "k": 65536, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 8192, - "n": 65536, - "k": 1536, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 8192, - "n": 122880, - "k": 1024, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 8192, - "n": 7168, - "k": 65536, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 8, - "n": 65536, - "k": 1536, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 8, - "n": 122880, - "k": 1024, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 8, - "n": 7168, - "k": 65536, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 16384, - "n": 65536, - "k": 1536, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 16384, - "n": 122880, - "k": 1024, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 16384, - "n": 7168, - "k": 65536, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 16, - "n": 65536, - "k": 1536, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 16, - "n": 122880, - "k": 1024, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 16, - "n": 7168, - "k": 65536, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 32768, - "n": 65536, - "k": 1536, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 32768, - "n": 122880, - "k": 1024, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 32768, - "n": 7168, - "k": 65536, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 32, - "n": 65536, - "k": 1536, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 32, - "n": 122880, - "k": 1024, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 32, - "n": 7168, - "k": 65536, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 524288, - "n": 8192, - "k": 1536, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 524288, - "n": 1088, - "k": 7168, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 524288, - "n": 15360, - "k": 1024, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 524288, - "n": 7168, - "k": 8192, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 1048576, - "n": 8192, - "k": 1536, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 1048576, - "n": 1088, - "k": 7168, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 1048576, - "n": 15360, - "k": 1024, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 1048576, - "n": 7168, - "k": 8192, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 2097152, - "n": 8192, - "k": 1536, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 2097152, - "n": 1088, - "k": 7168, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 2097152, - "n": 15360, - "k": 1024, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 2097152, - "n": 7168, - "k": 8192, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 65536, - "n": 65536, - "k": 1536, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 65536, - "n": 122880, - "k": 1024, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 65536, - "n": 7168, - "k": 65536, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 131072, - "n": 65536, - "k": 1536, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 131072, - "n": 122880, - "k": 1024, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 131072, - "n": 7168, - "k": 65536, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 262144, - "n": 65536, - "k": 1536, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 262144, - "n": 122880, - "k": 1024, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 262144, - "n": 7168, - "k": 65536, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 512, - "n": 8192, - "k": 1536, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 512, - "n": 15360, - "k": 1024, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 512, - "n": 7168, - "k": 8192, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 64, - "n": 65536, - "k": 1536, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 64, - "n": 122880, - "k": 1024, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 64, - "n": 7168, - "k": 65536, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 1024, - "n": 16384, - "k": 1536, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 1024, - "n": 30720, - "k": 1024, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 1, - "n": 16384, - "k": 1536, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 1, - "n": 30720, - "k": 1024, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 2048, - "n": 16384, - "k": 1536, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 2048, - "n": 30720, - "k": 1024, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 2, - "n": 16384, - "k": 1536, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 2, - "n": 30720, - "k": 1024, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 4, - "n": 16384, - "k": 1536, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 4, - "n": 30720, - "k": 1024, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 128, - "n": 16384, - "k": 1536, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 128, - "n": 30720, - "k": 1024, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 524288, - "n": 16384, - "k": 1536, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 524288, - "n": 30720, - "k": 1024, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 256, - "n": 16384, - "k": 1536, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 256, - "n": 30720, - "k": 1024, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 512, - "n": 16384, - "k": 1536, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 512, - "n": 30720, - "k": 1024, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 524288, - "n": 8192, - "k": 1536, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 524288, - "n": 15360, - "k": 1024, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 524288, - "n": 7168, - "k": 8192, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 1048576, - "n": 16384, - "k": 1536, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 1048576, - "n": 30720, - "k": 1024, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 2097152, - "n": 16384, - "k": 1536, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 2097152, - "n": 30720, - "k": 1024, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 8192, - "n": 1024, - "k": 4096, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 8192, - "n": 4096, - "k": 1024, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 8192, - "n": 128, - "k": 4096, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 8192, - "n": 4096, - "k": 128, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 16384, - "n": 1024, - "k": 4096, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 16384, - "n": 4096, - "k": 1024, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 16384, - "n": 128, - "k": 4096, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 16384, - "n": 4096, - "k": 128, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 16, - "n": 1024, - "k": 4096, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 16, - "n": 4096, - "k": 1024, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 16, - "n": 128, - "k": 4096, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 16, - "n": 4096, - "k": 128, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 32, - "n": 1024, - "k": 4096, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 32, - "n": 4096, - "k": 1024, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 32, - "n": 128, - "k": 4096, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 32, - "n": 4096, - "k": 128, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 65536, - "n": 1024, - "k": 4096, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 65536, - "n": 4096, - "k": 1024, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 65536, - "n": 128, - "k": 4096, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 65536, - "n": 4096, - "k": 128, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 64, - "n": 1024, - "k": 4096, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 64, - "n": 4096, - "k": 1024, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 64, - "n": 128, - "k": 4096, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 64, - "n": 4096, - "k": 128, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 131072, - "n": 1024, - "k": 4096, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 131072, - "n": 4096, - "k": 1024, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 131072, - "n": 128, - "k": 4096, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 131072, - "n": 4096, - "k": 128, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 262144, - "n": 1024, - "k": 4096, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 262144, - "n": 4096, - "k": 1024, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 262144, - "n": 128, - "k": 4096, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 262144, - "n": 4096, - "k": 128, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 524288, - "n": 1024, - "k": 4096, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 524288, - "n": 4096, - "k": 1024, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 524288, - "n": 128, - "k": 4096, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 524288, - "n": 4096, - "k": 128, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 128, - "n": 1024, - "k": 4096, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 128, - "n": 4096, - "k": 1024, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 128, - "n": 128, - "k": 4096, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 128, - "n": 4096, - "k": 128, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 256, - "n": 1024, - "k": 4096, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 256, - "n": 4096, - "k": 1024, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 256, - "n": 128, - "k": 4096, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 256, - "n": 4096, - "k": 128, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 1048576, - "n": 1024, - "k": 4096, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 1048576, - "n": 4096, - "k": 1024, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 1048576, - "n": 128, - "k": 4096, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 1048576, - "n": 4096, - "k": 128, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 2, - "n": 3072, - "k": 1536, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 2, - "n": 4096, - "k": 512, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 2, - "n": 7168, - "k": 2048, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 1, - "n": 24576, - "k": 1536, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 1, - "n": 32768, - "k": 512, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 1, - "n": 7168, - "k": 16384, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 384, - "n": 1152, - "k": 7168, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 384, - "n": 7168, - "k": 1152, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 0, - "n": 1152, - "k": 7168, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 0, - "n": 7168, - "k": 1152, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 6144, - "n": 1536, - "k": 7168, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 6144, - "n": 1536, - "k": 1536, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 6144, - "n": 576, - "k": 7168, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 6144, - "n": 2048, - "k": 512, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 6144, - "n": 7168, - "k": 1024, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 6144, - "n": 1152, - "k": 7168, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 6144, - "n": 7168, - "k": 1152, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 12, - "n": 1536, - "k": 1536, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 12, - "n": 2048, - "k": 512, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 12, - "n": 7168, - "k": 1024, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 12, - "n": 1152, - "k": 7168, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 12, - "n": 7168, - "k": 1152, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 576, - "n": 1536, - "k": 7168, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 576, - "n": 24576, - "k": 1536, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 576, - "n": 576, - "k": 7168, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 576, - "n": 32768, - "k": 512, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 576, - "n": 7168, - "k": 16384, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 576, - "n": 1152, - "k": 7168, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 576, - "n": 7168, - "k": 1152, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 1, - "n": 1152, - "k": 7168, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 1, - "n": 7168, - "k": 1152, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 9216, - "n": 1536, - "k": 7168, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 9216, - "n": 1536, - "k": 1536, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 9216, - "n": 576, - "k": 7168, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 9216, - "n": 2048, - "k": 512, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 9216, - "n": 7168, - "k": 1024, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 9216, - "n": 1152, - "k": 7168, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 9216, - "n": 7168, - "k": 1152, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 18, - "n": 1536, - "k": 1536, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 18, - "n": 2048, - "k": 512, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 18, - "n": 7168, - "k": 1024, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 18, - "n": 1152, - "k": 7168, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 18, - "n": 7168, - "k": 1152, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 1920, - "n": 1536, - "k": 7168, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 1920, - "n": 24576, - "k": 1536, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 1920, - "n": 576, - "k": 7168, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 1920, - "n": 32768, - "k": 512, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 1920, - "n": 7168, - "k": 16384, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 1920, - "n": 1152, - "k": 7168, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 1920, - "n": 7168, - "k": 1152, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 3, - "n": 1152, - "k": 7168, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 3, - "n": 7168, - "k": 1152, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 30720, - "n": 1536, - "k": 7168, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 30720, - "n": 1536, - "k": 1536, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 30720, - "n": 576, - "k": 7168, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 30720, - "n": 2048, - "k": 512, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 30720, - "n": 7168, - "k": 1024, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 30720, - "n": 1152, - "k": 7168, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 30720, - "n": 7168, - "k": 1152, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 60, - "n": 1536, - "k": 7168, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 60, - "n": 1536, - "k": 1536, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 60, - "n": 576, - "k": 7168, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 60, - "n": 2048, - "k": 512, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 60, - "n": 7168, - "k": 1024, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 60, - "n": 1152, - "k": 7168, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 60, - "n": 7168, - "k": 1152, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 3840, - "n": 1536, - "k": 7168, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 3840, - "n": 24576, - "k": 1536, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 3840, - "n": 576, - "k": 7168, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 3840, - "n": 32768, - "k": 512, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 3840, - "n": 7168, - "k": 16384, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 3840, - "n": 1152, - "k": 7168, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 3840, - "n": 7168, - "k": 1152, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 7, - "n": 1536, - "k": 7168, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 7, - "n": 24576, - "k": 1536, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 7, - "n": 576, - "k": 7168, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 7, - "n": 32768, - "k": 512, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 7, - "n": 7168, - "k": 16384, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 7, - "n": 1152, - "k": 7168, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 7, - "n": 7168, - "k": 1152, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 61440, - "n": 1536, - "k": 7168, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 61440, - "n": 1536, - "k": 1536, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 61440, - "n": 576, - "k": 7168, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 61440, - "n": 2048, - "k": 512, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 61440, - "n": 7168, - "k": 1024, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 61440, - "n": 1152, - "k": 7168, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 61440, - "n": 7168, - "k": 1152, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 120, - "n": 1536, - "k": 7168, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 120, - "n": 1536, - "k": 1536, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 120, - "n": 576, - "k": 7168, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 120, - "n": 2048, - "k": 512, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 120, - "n": 7168, - "k": 1024, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 120, - "n": 1152, - "k": 7168, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 120, - "n": 7168, - "k": 1152, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 7488, - "n": 1536, - "k": 7168, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 7488, - "n": 24576, - "k": 1536, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 7488, - "n": 576, - "k": 7168, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 7488, - "n": 32768, - "k": 512, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 7488, - "n": 7168, - "k": 16384, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 7488, - "n": 1152, - "k": 7168, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 7488, - "n": 7168, - "k": 1152, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 14, - "n": 1536, - "k": 7168, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 14, - "n": 24576, - "k": 1536, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 14, - "n": 576, - "k": 7168, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 14, - "n": 32768, - "k": 512, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 14, - "n": 7168, - "k": 16384, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 14, - "n": 1152, - "k": 7168, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 14, - "n": 7168, - "k": 1152, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 119808, - "n": 1536, - "k": 7168, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 119808, - "n": 1536, - "k": 1536, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 119808, - "n": 576, - "k": 7168, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 119808, - "n": 2048, - "k": 512, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 119808, - "n": 7168, - "k": 1024, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 119808, - "n": 1152, - "k": 7168, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 119808, - "n": 7168, - "k": 1152, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 234, - "n": 1536, - "k": 7168, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 234, - "n": 1536, - "k": 1536, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 234, - "n": 576, - "k": 7168, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 234, - "n": 2048, - "k": 512, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 234, - "n": 7168, - "k": 1024, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 234, - "n": 1152, - "k": 7168, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 234, - "n": 7168, - "k": 1152, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 14784, - "n": 1536, - "k": 7168, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 14784, - "n": 24576, - "k": 1536, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 14784, - "n": 576, - "k": 7168, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 14784, - "n": 32768, - "k": 512, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 14784, - "n": 7168, - "k": 16384, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 14784, - "n": 1152, - "k": 7168, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 14784, - "n": 7168, - "k": 1152, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 28, - "n": 1536, - "k": 7168, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 28, - "n": 24576, - "k": 1536, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 28, - "n": 576, - "k": 7168, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 28, - "n": 32768, - "k": 512, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 28, - "n": 7168, - "k": 16384, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 28, - "n": 1152, - "k": 7168, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 28, - "n": 7168, - "k": 1152, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 29568, - "n": 1536, - "k": 7168, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 29568, - "n": 24576, - "k": 1536, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 29568, - "n": 576, - "k": 7168, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 29568, - "n": 32768, - "k": 512, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 29568, - "n": 7168, - "k": 16384, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 29568, - "n": 1152, - "k": 7168, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 29568, - "n": 7168, - "k": 1152, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 57, - "n": 1536, - "k": 7168, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 57, - "n": 24576, - "k": 1536, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 57, - "n": 576, - "k": 7168, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 57, - "n": 32768, - "k": 512, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 57, - "n": 7168, - "k": 16384, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 57, - "n": 1152, - "k": 7168, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 57, - "n": 7168, - "k": 1152, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 473088, - "n": 1536, - "k": 7168, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 473088, - "n": 1536, - "k": 1536, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 473088, - "n": 576, - "k": 7168, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 473088, - "n": 2048, - "k": 512, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 473088, - "n": 7168, - "k": 1024, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 473088, - "n": 1152, - "k": 7168, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 473088, - "n": 7168, - "k": 1152, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 924, - "n": 1536, - "k": 7168, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 924, - "n": 1536, - "k": 1536, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 924, - "n": 576, - "k": 7168, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 924, - "n": 2048, - "k": 512, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 924, - "n": 7168, - "k": 1024, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 924, - "n": 1152, - "k": 7168, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 924, - "n": 7168, - "k": 1152, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 39360, - "n": 1536, - "k": 7168, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 39360, - "n": 24576, - "k": 1536, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 39360, - "n": 576, - "k": 7168, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 39360, - "n": 32768, - "k": 512, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 39360, - "n": 7168, - "k": 16384, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 39360, - "n": 1152, - "k": 7168, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 39360, - "n": 7168, - "k": 1152, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 76, - "n": 1536, - "k": 7168, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 76, - "n": 24576, - "k": 1536, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 76, - "n": 576, - "k": 7168, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 76, - "n": 32768, - "k": 512, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 76, - "n": 7168, - "k": 16384, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 76, - "n": 1152, - "k": 7168, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 76, - "n": 7168, - "k": 1152, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 78656, - "n": 1536, - "k": 7168, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 78656, - "n": 24576, - "k": 1536, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 78656, - "n": 576, - "k": 7168, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 78656, - "n": 32768, - "k": 512, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 78656, - "n": 7168, - "k": 16384, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 78656, - "n": 1152, - "k": 7168, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 78656, - "n": 7168, - "k": 1152, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 153, - "n": 1536, - "k": 7168, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 153, - "n": 24576, - "k": 1536, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 153, - "n": 576, - "k": 7168, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 153, - "n": 32768, - "k": 512, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 153, - "n": 7168, - "k": 16384, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 153, - "n": 1152, - "k": 7168, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 153, - "n": 7168, - "k": 1152, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 6, - "n": 1536, - "k": 7168, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 6, - "n": 1536, - "k": 1536, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 6, - "n": 576, - "k": 7168, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 6, - "n": 2048, - "k": 512, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 6, - "n": 7168, - "k": 1024, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 6, - "n": 1152, - "k": 7168, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 6, - "n": 7168, - "k": 1152, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 9, - "n": 1536, - "k": 7168, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 9, - "n": 1536, - "k": 1536, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 9, - "n": 576, - "k": 7168, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 9, - "n": 2048, - "k": 512, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 9, - "n": 7168, - "k": 1024, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 9, - "n": 1152, - "k": 7168, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 9, - "n": 7168, - "k": 1152, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 30, - "n": 1536, - "k": 7168, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 30, - "n": 1536, - "k": 1536, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 30, - "n": 576, - "k": 7168, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 30, - "n": 2048, - "k": 512, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 30, - "n": 7168, - "k": 1024, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 30, - "n": 1152, - "k": 7168, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 30, - "n": 7168, - "k": 1152, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 59136, - "n": 1536, - "k": 7168, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 59136, - "n": 24576, - "k": 1536, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 59136, - "n": 576, - "k": 7168, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 59136, - "n": 32768, - "k": 512, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 59136, - "n": 7168, - "k": 16384, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 59136, - "n": 1152, - "k": 7168, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 59136, - "n": 7168, - "k": 1152, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 118080, - "n": 1536, - "k": 7168, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 118080, - "n": 24576, - "k": 1536, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 118080, - "n": 576, - "k": 7168, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 118080, - "n": 32768, - "k": 512, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 118080, - "n": 7168, - "k": 16384, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 118080, - "n": 1152, - "k": 7168, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 118080, - "n": 7168, - "k": 1152, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 115, - "n": 1536, - "k": 7168, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 115, - "n": 24576, - "k": 1536, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 115, - "n": 576, - "k": 7168, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 115, - "n": 32768, - "k": 512, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 115, - "n": 7168, - "k": 16384, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 115, - "n": 1152, - "k": 7168, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 115, - "n": 7168, - "k": 1152, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 314624, - "n": 1536, - "k": 7168, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 314624, - "n": 24576, - "k": 1536, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 314624, - "n": 576, - "k": 7168, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 314624, - "n": 32768, - "k": 512, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 314624, - "n": 7168, - "k": 16384, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 314624, - "n": 1152, - "k": 7168, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 314624, - "n": 7168, - "k": 1152, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 307, - "n": 1536, - "k": 7168, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 307, - "n": 24576, - "k": 1536, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 307, - "n": 576, - "k": 7168, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 307, - "n": 32768, - "k": 512, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 307, - "n": 7168, - "k": 16384, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 307, - "n": 1152, - "k": 7168, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 307, - "n": 7168, - "k": 1152, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 3072, - "n": 1152, - "k": 7168, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 3072, - "n": 7168, - "k": 1152, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 49152, - "n": 1536, - "k": 1536, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 49152, - "n": 2048, - "k": 512, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 49152, - "n": 7168, - "k": 1024, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 49152, - "n": 1152, - "k": 7168, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 49152, - "n": 7168, - "k": 1152, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 4608, - "n": 1536, - "k": 7168, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 4608, - "n": 24576, - "k": 1536, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 4608, - "n": 576, - "k": 7168, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 4608, - "n": 32768, - "k": 512, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 4608, - "n": 7168, - "k": 16384, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 4608, - "n": 1152, - "k": 7168, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 4608, - "n": 7168, - "k": 1152, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 73728, - "n": 1536, - "k": 1536, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 73728, - "n": 2048, - "k": 512, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 73728, - "n": 7168, - "k": 1024, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 73728, - "n": 1152, - "k": 7168, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 73728, - "n": 7168, - "k": 1152, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 15360, - "n": 1536, - "k": 7168, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 15360, - "n": 24576, - "k": 1536, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 15360, - "n": 576, - "k": 7168, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 15360, - "n": 32768, - "k": 512, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 15360, - "n": 7168, - "k": 16384, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 15360, - "n": 1152, - "k": 7168, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 15360, - "n": 7168, - "k": 1152, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 245760, - "n": 1536, - "k": 1536, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 245760, - "n": 2048, - "k": 512, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 245760, - "n": 7168, - "k": 1024, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 245760, - "n": 1152, - "k": 7168, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 245760, - "n": 7168, - "k": 1152, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 39424, - "n": 1536, - "k": 7168, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 39424, - "n": 24576, - "k": 1536, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 39424, - "n": 576, - "k": 7168, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 39424, - "n": 32768, - "k": 512, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 39424, - "n": 7168, - "k": 16384, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 39424, - "n": 1152, - "k": 7168, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 39424, - "n": 7168, - "k": 1152, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 9, - "n": 24576, - "k": 1536, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 9, - "n": 32768, - "k": 512, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 9, - "n": 7168, - "k": 16384, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 78848, - "n": 1536, - "k": 7168, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 78848, - "n": 24576, - "k": 1536, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 78848, - "n": 576, - "k": 7168, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 78848, - "n": 32768, - "k": 512, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 78848, - "n": 7168, - "k": 16384, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 78848, - "n": 1152, - "k": 7168, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 78848, - "n": 7168, - "k": 1152, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 19, - "n": 1536, - "k": 7168, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 19, - "n": 24576, - "k": 1536, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 19, - "n": 576, - "k": 7168, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 19, - "n": 32768, - "k": 512, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 19, - "n": 7168, - "k": 16384, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 19, - "n": 1152, - "k": 7168, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 19, - "n": 7168, - "k": 1152, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 1261568, - "n": 1536, - "k": 7168, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 1261568, - "n": 1536, - "k": 1536, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 1261568, - "n": 576, - "k": 7168, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 1261568, - "n": 2048, - "k": 512, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 1261568, - "n": 7168, - "k": 1024, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 1261568, - "n": 1152, - "k": 7168, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 1261568, - "n": 7168, - "k": 1152, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 154, - "n": 1536, - "k": 7168, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 154, - "n": 1536, - "k": 1536, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 154, - "n": 576, - "k": 7168, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 154, - "n": 2048, - "k": 512, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 154, - "n": 7168, - "k": 1024, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 154, - "n": 1152, - "k": 7168, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 154, - "n": 7168, - "k": 1152, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 157696, - "n": 1536, - "k": 7168, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 157696, - "n": 24576, - "k": 1536, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 157696, - "n": 576, - "k": 7168, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 157696, - "n": 32768, - "k": 512, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 157696, - "n": 7168, - "k": 16384, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 157696, - "n": 1152, - "k": 7168, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 157696, - "n": 7168, - "k": 1152, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 262144, - "n": 4096, - "k": 2880, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 262144, - "n": 512, - "k": 2880, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 262144, - "n": 2880, - "k": 4096, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 256, - "n": 4096, - "k": 2880, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 256, - "n": 512, - "k": 2880, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 256, - "n": 2880, - "k": 4096, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 131072, - "n": 4096, - "k": 2880, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 131072, - "n": 512, - "k": 2880, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 131072, - "n": 2880, - "k": 4096, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 128, - "n": 4096, - "k": 2880, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 128, - "n": 512, - "k": 2880, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 128, - "n": 2880, - "k": 4096, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 4096, - "n": 2048, - "k": 2880, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 4096, - "n": 256, - "k": 2880, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 4096, - "n": 2880, - "k": 2048, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 4, - "n": 2048, - "k": 2880, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 4, - "n": 256, - "k": 2880, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 4, - "n": 2880, - "k": 2048, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 8192, - "n": 2048, - "k": 2880, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 8192, - "n": 256, - "k": 2880, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 8192, - "n": 2880, - "k": 2048, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 8, - "n": 2048, - "k": 2880, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 8, - "n": 256, - "k": 2880, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 8, - "n": 2880, - "k": 2048, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 16384, - "n": 2048, - "k": 2880, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 16384, - "n": 256, - "k": 2880, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 16384, - "n": 2880, - "k": 2048, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 16, - "n": 2048, - "k": 2880, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 16, - "n": 256, - "k": 2880, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 16, - "n": 2880, - "k": 2048, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 32768, - "n": 2048, - "k": 2880, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 32768, - "n": 256, - "k": 2880, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 32768, - "n": 2880, - "k": 2048, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 32, - "n": 2048, - "k": 2880, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 32, - "n": 256, - "k": 2880, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 32, - "n": 2880, - "k": 2048, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 65536, - "n": 2048, - "k": 2880, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 65536, - "n": 256, - "k": 2880, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 65536, - "n": 2880, - "k": 2048, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 64, - "n": 2048, - "k": 2880, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 64, - "n": 256, - "k": 2880, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 64, - "n": 2880, - "k": 2048, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 131072, - "n": 2048, - "k": 2880, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 131072, - "n": 256, - "k": 2880, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 131072, - "n": 2880, - "k": 2048, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 128, - "n": 2048, - "k": 2880, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 128, - "n": 256, - "k": 2880, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 128, - "n": 2880, - "k": 2048, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 262144, - "n": 2048, - "k": 2880, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 262144, - "n": 256, - "k": 2880, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 262144, - "n": 2880, - "k": 2048, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 256, - "n": 2048, - "k": 2880, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 256, - "n": 256, - "k": 2880, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 256, - "n": 2880, - "k": 2048, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 16384, - "n": 4096, - "k": 2880, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 16384, - "n": 512, - "k": 2880, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 16384, - "n": 2880, - "k": 4096, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 16, - "n": 4096, - "k": 2880, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 16, - "n": 512, - "k": 2880, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 16, - "n": 2880, - "k": 4096, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 4096, - "n": 1024, - "k": 2880, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 4096, - "n": 128, - "k": 2880, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 4096, - "n": 2880, - "k": 1024, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 4, - "n": 1024, - "k": 2880, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 4, - "n": 128, - "k": 2880, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 4, - "n": 2880, - "k": 1024, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 4096, - "n": 512, - "k": 2880, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 4096, - "n": 64, - "k": 2880, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 4096, - "n": 2880, - "k": 512, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 4, - "n": 512, - "k": 2880, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 4, - "n": 64, - "k": 2880, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 4, - "n": 2880, - "k": 512, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 32768, - "n": 4096, - "k": 2880, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 32768, - "n": 512, - "k": 2880, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 32768, - "n": 2880, - "k": 4096, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 4, - "n": 4096, - "k": 2880, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 4, - "n": 2880, - "k": 4096, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 65536, - "n": 4096, - "k": 2880, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 65536, - "n": 512, - "k": 2880, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 65536, - "n": 2880, - "k": 4096, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 8, - "n": 4096, - "k": 2880, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 8, - "n": 512, - "k": 2880, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 8, - "n": 2880, - "k": 4096, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 32, - "n": 4096, - "k": 2880, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 32, - "n": 512, - "k": 2880, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 32, - "n": 2880, - "k": 4096, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 524288, - "n": 4096, - "k": 2880, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 524288, - "n": 512, - "k": 2880, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 524288, - "n": 2880, - "k": 4096, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 64, - "n": 4096, - "k": 2880, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 64, - "n": 512, - "k": 2880, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 64, - "n": 2880, - "k": 4096, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 1048576, - "n": 4096, - "k": 2880, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 1048576, - "n": 512, - "k": 2880, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 1048576, - "n": 2880, - "k": 4096, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 2097152, - "n": 4096, - "k": 2880, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 2097152, - "n": 512, - "k": 2880, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 2097152, - "n": 2880, - "k": 4096, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 524288, - "n": 2048, - "k": 2880, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 524288, - "n": 256, - "k": 2880, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 524288, - "n": 2880, - "k": 2048, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 1048576, - "n": 2048, - "k": 2880, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 1048576, - "n": 256, - "k": 2880, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 1048576, - "n": 2880, - "k": 2048, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 2097152, - "n": 2048, - "k": 2880, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 2097152, - "n": 256, - "k": 2880, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 2097152, - "n": 2880, - "k": 2048, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 32768, - "n": 1024, - "k": 2880, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 32768, - "n": 128, - "k": 2880, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 32768, - "n": 2880, - "k": 1024, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 65536, - "n": 1024, - "k": 2880, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 65536, - "n": 128, - "k": 2880, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 65536, - "n": 2880, - "k": 1024, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 8, - "n": 1024, - "k": 2880, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 8, - "n": 128, - "k": 2880, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 8, - "n": 2880, - "k": 1024, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 131072, - "n": 1024, - "k": 2880, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 131072, - "n": 128, - "k": 2880, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 131072, - "n": 2880, - "k": 1024, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 16, - "n": 1024, - "k": 2880, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 16, - "n": 128, - "k": 2880, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 16, - "n": 2880, - "k": 1024, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 262144, - "n": 1024, - "k": 2880, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 262144, - "n": 128, - "k": 2880, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 262144, - "n": 2880, - "k": 1024, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 32, - "n": 1024, - "k": 2880, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 32, - "n": 128, - "k": 2880, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 32, - "n": 2880, - "k": 1024, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 32768, - "n": 64, - "k": 2880, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 32768, - "n": 2880, - "k": 512, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 4096, - "n": 4096, - "k": 2880, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 4096, - "n": 2880, - "k": 4096, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 8192, - "n": 4096, - "k": 2880, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 8192, - "n": 512, - "k": 2880, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 8192, - "n": 2880, - "k": 4096, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 8192, - "n": 1024, - "k": 2880, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 8192, - "n": 128, - "k": 2880, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 8192, - "n": 2880, - "k": 1024, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 16384, - "n": 1024, - "k": 2880, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 16384, - "n": 128, - "k": 2880, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 16384, - "n": 2880, - "k": 1024, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 64, - "n": 1024, - "k": 2880, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 64, - "n": 128, - "k": 2880, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 64, - "n": 2880, - "k": 1024, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 8192, - "n": 64, - "k": 2880, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 8192, - "n": 2880, - "k": 512, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 8, - "n": 64, - "k": 2880, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 8, - "n": 2880, - "k": 512, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 524288, - "n": 1024, - "k": 2880, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 524288, - "n": 128, - "k": 2880, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 524288, - "n": 2880, - "k": 1024, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 524288, - "n": 3072, - "k": 3072, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 524288, - "n": 512, - "k": 3072, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 524288, - "n": 768, - "k": 3072, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 524288, - "n": 3072, - "k": 768, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 512, - "n": 3072, - "k": 3072, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 512, - "n": 512, - "k": 3072, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 512, - "n": 768, - "k": 3072, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 512, - "n": 3072, - "k": 768, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 4096, - "n": 1536, - "k": 3072, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 4096, - "n": 256, - "k": 3072, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 4096, - "n": 384, - "k": 3072, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 4096, - "n": 3072, - "k": 384, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 4, - "n": 1536, - "k": 3072, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 4, - "n": 256, - "k": 3072, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 4, - "n": 384, - "k": 3072, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 4, - "n": 3072, - "k": 384, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 8192, - "n": 1536, - "k": 3072, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 8192, - "n": 256, - "k": 3072, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 8192, - "n": 384, - "k": 3072, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 8192, - "n": 3072, - "k": 384, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 8, - "n": 1536, - "k": 3072, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 8, - "n": 256, - "k": 3072, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 8, - "n": 384, - "k": 3072, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 8, - "n": 3072, - "k": 384, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 16384, - "n": 1536, - "k": 3072, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 16384, - "n": 256, - "k": 3072, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 16384, - "n": 384, - "k": 3072, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 16384, - "n": 3072, - "k": 384, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 16, - "n": 1536, - "k": 3072, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 16, - "n": 256, - "k": 3072, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 16, - "n": 384, - "k": 3072, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 16, - "n": 3072, - "k": 384, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 32768, - "n": 1536, - "k": 3072, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 32768, - "n": 256, - "k": 3072, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 32768, - "n": 384, - "k": 3072, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 32768, - "n": 3072, - "k": 384, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 32, - "n": 1536, - "k": 3072, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 32, - "n": 256, - "k": 3072, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 32, - "n": 384, - "k": 3072, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 32, - "n": 3072, - "k": 384, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 65536, - "n": 1536, - "k": 3072, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 65536, - "n": 256, - "k": 3072, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 65536, - "n": 384, - "k": 3072, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 65536, - "n": 3072, - "k": 384, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 64, - "n": 1536, - "k": 3072, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 64, - "n": 256, - "k": 3072, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 64, - "n": 384, - "k": 3072, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 64, - "n": 3072, - "k": 384, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 131072, - "n": 1536, - "k": 3072, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 131072, - "n": 256, - "k": 3072, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 131072, - "n": 384, - "k": 3072, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 131072, - "n": 3072, - "k": 384, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 128, - "n": 1536, - "k": 3072, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 128, - "n": 256, - "k": 3072, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 128, - "n": 384, - "k": 3072, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 128, - "n": 3072, - "k": 384, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 262144, - "n": 1536, - "k": 3072, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 262144, - "n": 256, - "k": 3072, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 262144, - "n": 384, - "k": 3072, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 262144, - "n": 3072, - "k": 384, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 256, - "n": 1536, - "k": 3072, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 256, - "n": 256, - "k": 3072, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 256, - "n": 384, - "k": 3072, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 256, - "n": 3072, - "k": 384, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 524288, - "n": 1536, - "k": 3072, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 524288, - "n": 256, - "k": 3072, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 524288, - "n": 384, - "k": 3072, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 524288, - "n": 3072, - "k": 384, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 512, - "n": 1536, - "k": 3072, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 512, - "n": 256, - "k": 3072, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 512, - "n": 384, - "k": 3072, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 512, - "n": 3072, - "k": 384, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 32768, - "n": 3072, - "k": 3072, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 32768, - "n": 512, - "k": 3072, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 32768, - "n": 768, - "k": 3072, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 32768, - "n": 3072, - "k": 768, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 4, - "n": 3072, - "k": 3072, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 4, - "n": 512, - "k": 3072, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 4, - "n": 768, - "k": 3072, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 4, - "n": 3072, - "k": 768, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 65536, - "n": 3072, - "k": 3072, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 65536, - "n": 512, - "k": 3072, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 65536, - "n": 768, - "k": 3072, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 65536, - "n": 3072, - "k": 768, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 8, - "n": 3072, - "k": 3072, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 8, - "n": 512, - "k": 3072, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 8, - "n": 768, - "k": 3072, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 8, - "n": 3072, - "k": 768, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 131072, - "n": 3072, - "k": 3072, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 131072, - "n": 512, - "k": 3072, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 131072, - "n": 768, - "k": 3072, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 131072, - "n": 3072, - "k": 768, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 16, - "n": 3072, - "k": 3072, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 16, - "n": 512, - "k": 3072, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 16, - "n": 768, - "k": 3072, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 16, - "n": 3072, - "k": 768, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 262144, - "n": 3072, - "k": 3072, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 262144, - "n": 512, - "k": 3072, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 262144, - "n": 768, - "k": 3072, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 262144, - "n": 3072, - "k": 768, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 32, - "n": 3072, - "k": 3072, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 32, - "n": 512, - "k": 3072, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 32, - "n": 768, - "k": 3072, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 32, - "n": 3072, - "k": 768, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 64, - "n": 3072, - "k": 3072, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 64, - "n": 512, - "k": 3072, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 64, - "n": 768, - "k": 3072, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 64, - "n": 3072, - "k": 768, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 1048576, - "n": 3072, - "k": 3072, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 1048576, - "n": 512, - "k": 3072, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 1048576, - "n": 768, - "k": 3072, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 1048576, - "n": 3072, - "k": 768, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 128, - "n": 3072, - "k": 3072, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 128, - "n": 512, - "k": 3072, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 128, - "n": 768, - "k": 3072, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 128, - "n": 3072, - "k": 768, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 2097152, - "n": 3072, - "k": 3072, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 2097152, - "n": 512, - "k": 3072, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 2097152, - "n": 768, - "k": 3072, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 2097152, - "n": 3072, - "k": 768, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 256, - "n": 3072, - "k": 3072, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 256, - "n": 512, - "k": 3072, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 256, - "n": 768, - "k": 3072, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 256, - "n": 3072, - "k": 768, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 4194304, - "n": 3072, - "k": 3072, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 4194304, - "n": 512, - "k": 3072, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 4194304, - "n": 768, - "k": 3072, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 4194304, - "n": 3072, - "k": 768, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 1048576, - "n": 1536, - "k": 3072, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 1048576, - "n": 256, - "k": 3072, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 1048576, - "n": 384, - "k": 3072, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 1048576, - "n": 3072, - "k": 384, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 2097152, - "n": 1536, - "k": 3072, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 2097152, - "n": 256, - "k": 3072, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 2097152, - "n": 384, - "k": 3072, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 2097152, - "n": 3072, - "k": 384, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 4194304, - "n": 1536, - "k": 3072, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 4194304, - "n": 256, - "k": 3072, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 4194304, - "n": 384, - "k": 3072, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 4194304, - "n": 3072, - "k": 384, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 1024, - "n": 3072, - "k": 3072, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 1024, - "n": 512, - "k": 3072, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 1024, - "n": 768, - "k": 3072, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 1024, - "n": 3072, - "k": 768, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 524288, - "n": 6144, - "k": 3072, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 524288, - "n": 1024, - "k": 3072, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 524288, - "n": 3072, - "k": 6144, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 512, - "n": 6144, - "k": 3072, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 512, - "n": 1024, - "k": 3072, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 512, - "n": 3072, - "k": 6144, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 32768, - "n": 6144, - "k": 3072, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 32768, - "n": 1024, - "k": 3072, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 32768, - "n": 3072, - "k": 6144, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 4, - "n": 6144, - "k": 3072, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 4, - "n": 1024, - "k": 3072, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 4, - "n": 3072, - "k": 6144, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 65536, - "n": 6144, - "k": 3072, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 65536, - "n": 1024, - "k": 3072, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 65536, - "n": 3072, - "k": 6144, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 8, - "n": 6144, - "k": 3072, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 8, - "n": 1024, - "k": 3072, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 8, - "n": 3072, - "k": 6144, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 131072, - "n": 6144, - "k": 3072, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 131072, - "n": 1024, - "k": 3072, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 131072, - "n": 3072, - "k": 6144, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 16, - "n": 6144, - "k": 3072, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 16, - "n": 1024, - "k": 3072, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 16, - "n": 3072, - "k": 6144, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 4096, - "n": 1024, - "k": 3072, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 4096, - "n": 1536, - "k": 3072, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 4, - "n": 1024, - "k": 3072, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 4, - "n": 1536, - "k": 3072, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 8192, - "n": 1024, - "k": 3072, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 8192, - "n": 1536, - "k": 3072, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 8, - "n": 1024, - "k": 3072, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 8, - "n": 1536, - "k": 3072, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 16384, - "n": 1024, - "k": 3072, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 16384, - "n": 1536, - "k": 3072, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 16, - "n": 1024, - "k": 3072, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 16, - "n": 1536, - "k": 3072, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 16384, - "n": 3072, - "k": 3072, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 16384, - "n": 512, - "k": 3072, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 16384, - "n": 768, - "k": 3072, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 16384, - "n": 3072, - "k": 768, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 16, - "n": 3072, - "k": 3072, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 16, - "n": 512, - "k": 3072, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 16, - "n": 768, - "k": 3072, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 16, - "n": 3072, - "k": 768, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 131072, - "n": 3072, - "k": 3072, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 131072, - "n": 512, - "k": 3072, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 131072, - "n": 768, - "k": 3072, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 131072, - "n": 3072, - "k": 768, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 128, - "n": 3072, - "k": 3072, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 128, - "n": 512, - "k": 3072, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 128, - "n": 768, - "k": 3072, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 128, - "n": 3072, - "k": 768, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 131072, - "n": 1024, - "k": 3072, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 128, - "n": 1024, - "k": 3072, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 262144, - "n": 1024, - "k": 3072, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 262144, - "n": 768, - "k": 3072, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 262144, - "n": 3072, - "k": 768, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 256, - "n": 1024, - "k": 3072, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 256, - "n": 768, - "k": 3072, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 256, - "n": 3072, - "k": 768, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 524288, - "n": 1024, - "k": 3072, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 524288, - "n": 768, - "k": 3072, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 524288, - "n": 3072, - "k": 768, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 512, - "n": 1024, - "k": 3072, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 512, - "n": 768, - "k": 3072, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 512, - "n": 3072, - "k": 768, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 4096, - "n": 256, - "k": 3072, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 4096, - "n": 384, - "k": 3072, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 4096, - "n": 3072, - "k": 384, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 4, - "n": 256, - "k": 3072, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 4, - "n": 384, - "k": 3072, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 4, - "n": 3072, - "k": 384, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 8192, - "n": 256, - "k": 3072, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 8192, - "n": 384, - "k": 3072, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 8192, - "n": 3072, - "k": 384, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 8, - "n": 256, - "k": 3072, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 8, - "n": 384, - "k": 3072, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 8, - "n": 3072, - "k": 384, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 16384, - "n": 256, - "k": 3072, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 16384, - "n": 384, - "k": 3072, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 16384, - "n": 3072, - "k": 384, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 16, - "n": 256, - "k": 3072, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 16, - "n": 384, - "k": 3072, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 16, - "n": 3072, - "k": 384, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 65536, - "n": 1536, - "k": 3072, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 65536, - "n": 256, - "k": 3072, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 65536, - "n": 384, - "k": 3072, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 65536, - "n": 3072, - "k": 384, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 64, - "n": 1536, - "k": 3072, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 64, - "n": 256, - "k": 3072, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 64, - "n": 384, - "k": 3072, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 64, - "n": 3072, - "k": 384, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 131072, - "n": 1536, - "k": 3072, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 131072, - "n": 256, - "k": 3072, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 131072, - "n": 384, - "k": 3072, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 131072, - "n": 3072, - "k": 384, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 128, - "n": 1536, - "k": 3072, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 128, - "n": 256, - "k": 3072, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 128, - "n": 384, - "k": 3072, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 128, - "n": 3072, - "k": 384, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 4096, - "n": 768, - "k": 3072, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 4096, - "n": 128, - "k": 3072, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 4096, - "n": 3072, - "k": 768, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 4096, - "n": 192, - "k": 3072, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 4096, - "n": 3072, - "k": 192, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 4, - "n": 768, - "k": 3072, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 4, - "n": 128, - "k": 3072, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 4, - "n": 3072, - "k": 768, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 4, - "n": 192, - "k": 3072, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 4, - "n": 3072, - "k": 192, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 8192, - "n": 768, - "k": 3072, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 8192, - "n": 128, - "k": 3072, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 8192, - "n": 3072, - "k": 768, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 8192, - "n": 192, - "k": 3072, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 8192, - "n": 3072, - "k": 192, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 8, - "n": 768, - "k": 3072, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 8, - "n": 128, - "k": 3072, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 8, - "n": 3072, - "k": 768, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 8, - "n": 192, - "k": 3072, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 8, - "n": 3072, - "k": 192, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 32768, - "n": 1024, - "k": 3072, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 32768, - "n": 1536, - "k": 3072, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 65536, - "n": 1024, - "k": 3072, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 262144, - "n": 1536, - "k": 3072, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 32, - "n": 1024, - "k": 3072, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 32, - "n": 1536, - "k": 3072, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 2097152, - "n": 1024, - "k": 3072, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 2097152, - "n": 1536, - "k": 3072, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 256, - "n": 1536, - "k": 3072, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 1048576, - "n": 3072, - "k": 3072, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 1048576, - "n": 512, - "k": 3072, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 1048576, - "n": 768, - "k": 3072, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 1048576, - "n": 3072, - "k": 768, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 2097152, - "n": 3072, - "k": 3072, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 2097152, - "n": 512, - "k": 3072, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 2097152, - "n": 768, - "k": 3072, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 2097152, - "n": 3072, - "k": 768, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 256, - "n": 3072, - "k": 3072, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 256, - "n": 512, - "k": 3072, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 4194304, - "n": 3072, - "k": 3072, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 4194304, - "n": 512, - "k": 3072, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 4194304, - "n": 768, - "k": 3072, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 370, + "n": 7168, + "k": 2048, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/o_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/o_proj" + ] }, { "type": "gemm", "args": { - "m": 4194304, - "n": 3072, - "k": 768, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 370, + "n": 7168, + "k": 2304, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/down_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/down_proj", + "nvidia/Kimi-K2.5-NVFP4/down_proj" + ] }, { "type": "gemm", "args": { - "m": 512, - "n": 3072, - "k": 3072, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 384, + "n": 256, + "k": 2880, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/gpt-oss-120b-w-mxfp4-a-fp8/k_proj", + "amd/gpt-oss-120b-w-mxfp4-a-fp8/q_proj", + "amd/gpt-oss-120b-w-mxfp4-a-fp8/v_proj", + "openai/gpt-oss-120b/k_proj", + "openai/gpt-oss-120b/q_proj", + "openai/gpt-oss-120b/v_proj" + ] }, { "type": "gemm", "args": { - "m": 512, + "m": 384, "n": 512, - "k": 3072, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "k": 2880, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/gpt-oss-120b-w-mxfp4-a-fp8/k_proj", + "amd/gpt-oss-120b-w-mxfp4-a-fp8/q_proj", + "amd/gpt-oss-120b-w-mxfp4-a-fp8/v_proj", + "openai/gpt-oss-120b/k_proj", + "openai/gpt-oss-120b/q_proj", + "openai/gpt-oss-120b/v_proj" + ] }, { "type": "gemm", "args": { - "m": 32768, - "n": 256, - "k": 3072, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 384, + "n": 576, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/kv_a_proj_with_mqa" + ] }, { "type": "gemm", "args": { - "m": 32768, - "n": 384, - "k": 3072, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 384, + "n": 1152, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/gate_up_proj" + ] }, { "type": "gemm", "args": { - "m": 32768, - "n": 3072, - "k": 384, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 384, + "n": 1536, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/gate_up_proj", + "deepseek-ai/DeepSeek-R1-0528/q_a_proj", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w3" + ] }, { "type": "gemm", "args": { - "m": 32768, - "n": 768, - "k": 3072, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 384, + "n": 2048, + "k": 2880, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/gpt-oss-120b-w-mxfp4-a-fp8/q_proj", + "openai/gpt-oss-120b/q_proj" + ] }, { "type": "gemm", "args": { - "m": 32768, - "n": 128, - "k": 3072, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 384, + "n": 2304, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/gate_up_proj" + ] }, { "type": "gemm", "args": { - "m": 32768, - "n": 3072, - "k": 768, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 384, + "n": 2880, + "k": 2048, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/gpt-oss-120b-w-mxfp4-a-fp8/o_proj", + "openai/gpt-oss-120b/o_proj" + ] }, { "type": "gemm", "args": { - "m": 32768, - "n": 192, - "k": 3072, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 384, + "n": 2880, + "k": 4096, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/gpt-oss-120b-w-mxfp4-a-fp8/o_proj", + "openai/gpt-oss-120b/o_proj" + ] }, { "type": "gemm", "args": { - "m": 32768, + "m": 384, "n": 3072, - "k": 192, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 1048576, - "n": 1024, - "k": 3072, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 1024, - "n": 6144, - "k": 3072, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 1024, - "n": 1024, - "k": 3072, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "MiniMaxAI/MiniMax-M2.5/experts.down_proj", + "MiniMaxAI/MiniMax-M2.5/o_proj", + "deepseek-ai/DeepSeek-R1-0528/q_b_proj" + ] }, { "type": "gemm", "args": { - "m": 1024, - "n": 3072, - "k": 6144, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 384, + "n": 4096, + "k": 512, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "Qwen/Qwen3.5-397B-A17B-FP8/o_proj", + "Qwen/Qwen3.5-397B-A17B-FP8/shared_expert.down_proj", + "deepseek-ai/DeepSeek-R1-0528/kv_b_proj" + ] }, { "type": "gemm", "args": { - "m": 1024, - "n": 768, - "k": 3072, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 384, + "n": 4096, + "k": 2880, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/gpt-oss-120b-w-mxfp4-a-fp8/q_proj", + "openai/gpt-oss-120b/q_proj" + ] }, { "type": "gemm", "args": { - "m": 1024, - "n": 3072, - "k": 768, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 384, + "n": 7168, + "k": 1152, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/down_proj" + ] }, { "type": "gemm", "args": { - "m": 524288, - "n": 1536, - "k": 3072, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 384, + "n": 7168, + "k": 2048, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/o_proj", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] }, { "type": "gemm", "args": { - "m": 64, - "n": 1024, - "k": 3072, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 384, + "n": 7168, + "k": 2304, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/down_proj" + ] }, { "type": "gemm", "args": { - "m": 1048576, - "n": 1536, - "k": 3072, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 384, + "n": 7168, + "k": 16384, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/o_proj", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] }, { "type": "gemm", "args": { - "m": 16384, - "n": 64, - "k": 2880, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 384, + "n": 24576, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/q_b_proj" + ] }, { "type": "gemm", "args": { - "m": 16384, - "n": 2880, + "m": 384, + "n": 32768, "k": 512, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 16, - "n": 64, - "k": 2880, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/kv_b_proj" + ] }, { "type": "gemm", "args": { - "m": 16, - "n": 2880, - "k": 512, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 395, + "n": 576, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/Kimi-K2.5-MXFP4/kv_a_proj_with_mqa", + "moonshotai/Kimi-K2.5/kv_a_proj_with_mqa", + "nvidia/DeepSeek-R1-0528-FP4-V2/kv_a_proj_with_mqa", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/kv_a_proj_with_mqa", + "nvidia/Kimi-K2.5-NVFP4/kv_a_proj_with_mqa", + "nvidia/Kimi-K2.6-NVFP4/kv_a_proj_with_mqa" + ] }, { "type": "gemm", "args": { - "m": 32, - "n": 64, - "k": 2880, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 395, + "n": 1536, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/Kimi-K2.5-MXFP4/q_a_proj", + "moonshotai/Kimi-K2.5/q_a_proj", + "nvidia/DeepSeek-R1-0528-FP4-V2/q_a_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/q_a_proj", + "nvidia/Kimi-K2.5-NVFP4/q_a_proj", + "nvidia/Kimi-K2.6-NVFP4/q_a_proj" + ] }, { "type": "gemm", "args": { - "m": 32, - "n": 2880, - "k": 512, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 395, + "n": 2304, + "k": 7168, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/gate_up_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/gate_up_proj", + "nvidia/Kimi-K2.5-NVFP4/gate_up_proj" + ] }, { "type": "gemm", "args": { - "m": 65536, - "n": 64, - "k": 2880, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 395, + "n": 3072, + "k": 1536, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/Kimi-K2.5-MXFP4/q_b_proj", + "amd/MiniMax-M2.5-MXFP4/o_proj", + "moonshotai/Kimi-K2.5/q_b_proj", + "nvidia/DeepSeek-R1-0528-FP4-V2/q_b_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/q_b_proj", + "nvidia/Kimi-K2.5-NVFP4/q_b_proj", + "nvidia/Kimi-K2.6-NVFP4/q_b_proj", + "nvidia/MiniMax-M2.5-NVFP4/o_proj" + ] }, { "type": "gemm", "args": { - "m": 65536, - "n": 2880, + "m": 395, + "n": 4096, "k": 512, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 64, - "n": 64, - "k": 2880, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "Qwen/Qwen3.5-397B-A17B/o_proj", + "Qwen/Qwen3.5-397B-A17B/shared_expert.down_proj", + "amd/Kimi-K2.5-MXFP4/kv_b_proj", + "amd/Qwen3.5-397B-A17B-MXFP4/o_proj", + "amd/Qwen3.5-397B-A17B-MXFP4/shared_expert.down_proj", + "moonshotai/Kimi-K2.5/kv_b_proj", + "nvidia/DeepSeek-R1-0528-FP4-V2/kv_b_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/kv_b_proj", + "nvidia/Kimi-K2.5-NVFP4/kv_b_proj", + "nvidia/Kimi-K2.6-NVFP4/kv_b_proj", + "nvidia/Qwen3.5-397B-A17B-NVFP4/o_proj", + "nvidia/Qwen3.5-397B-A17B-NVFP4/shared_expert.down_proj" + ] }, { "type": "gemm", "args": { - "m": 64, - "n": 2880, - "k": 512, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 395, + "n": 7168, + "k": 2048, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/o_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/o_proj" + ] }, { "type": "gemm", "args": { - "m": 131072, - "n": 64, - "k": 2880, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 395, + "n": 7168, + "k": 2304, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/down_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/down_proj", + "nvidia/Kimi-K2.5-NVFP4/down_proj" + ] }, { "type": "gemm", "args": { - "m": 131072, - "n": 2880, - "k": 512, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 400, + "n": 576, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/Kimi-K2.5-MXFP4/kv_a_proj_with_mqa", + "moonshotai/Kimi-K2.5/kv_a_proj_with_mqa", + "nvidia/DeepSeek-R1-0528-FP4-V2/kv_a_proj_with_mqa", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/kv_a_proj_with_mqa", + "nvidia/Kimi-K2.5-NVFP4/kv_a_proj_with_mqa", + "nvidia/Kimi-K2.6-NVFP4/kv_a_proj_with_mqa" + ] }, { "type": "gemm", "args": { - "m": 1024, + "m": 400, "n": 1536, - "k": 1536, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 1024, - "n": 2048, - "k": 512, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 1024, - "n": 7168, - "k": 1024, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/Kimi-K2.5-MXFP4/q_a_proj", + "moonshotai/Kimi-K2.5/q_a_proj", + "nvidia/DeepSeek-R1-0528-FP4-V2/q_a_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/q_a_proj", + "nvidia/Kimi-K2.5-NVFP4/q_a_proj", + "nvidia/Kimi-K2.6-NVFP4/q_a_proj" + ] }, { "type": "gemm", "args": { - "m": 1024, - "n": 1152, + "m": 400, + "n": 2304, "k": 7168, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/gate_up_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/gate_up_proj", + "nvidia/Kimi-K2.5-NVFP4/gate_up_proj" + ] }, { "type": "gemm", "args": { - "m": 1024, - "n": 7168, - "k": 1152, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 400, + "n": 2304, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/gate_up_proj" + ] }, { "type": "gemm", "args": { - "m": 2, - "n": 1536, + "m": 400, + "n": 3072, "k": 1536, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/Kimi-K2.5-MXFP4/q_b_proj", + "amd/MiniMax-M2.5-MXFP4/o_proj", + "moonshotai/Kimi-K2.5/q_b_proj", + "nvidia/DeepSeek-R1-0528-FP4-V2/q_b_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/q_b_proj", + "nvidia/Kimi-K2.5-NVFP4/q_b_proj", + "nvidia/Kimi-K2.6-NVFP4/q_b_proj", + "nvidia/MiniMax-M2.5-NVFP4/o_proj" + ] }, { "type": "gemm", "args": { - "m": 2, - "n": 2048, + "m": 400, + "n": 4096, "k": 512, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "Qwen/Qwen3.5-397B-A17B/o_proj", + "Qwen/Qwen3.5-397B-A17B/shared_expert.down_proj", + "amd/Kimi-K2.5-MXFP4/kv_b_proj", + "amd/Qwen3.5-397B-A17B-MXFP4/o_proj", + "amd/Qwen3.5-397B-A17B-MXFP4/shared_expert.down_proj", + "moonshotai/Kimi-K2.5/kv_b_proj", + "nvidia/DeepSeek-R1-0528-FP4-V2/kv_b_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/kv_b_proj", + "nvidia/Kimi-K2.5-NVFP4/kv_b_proj", + "nvidia/Kimi-K2.6-NVFP4/kv_b_proj", + "nvidia/Qwen3.5-397B-A17B-NVFP4/o_proj", + "nvidia/Qwen3.5-397B-A17B-NVFP4/shared_expert.down_proj" + ] }, { "type": "gemm", "args": { - "m": 2, + "m": 400, "n": 7168, - "k": 1024, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "k": 2048, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/o_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/o_proj" + ] }, { "type": "gemm", "args": { - "m": 2, - "n": 1152, - "k": 7168, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 400, + "n": 7168, + "k": 2304, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/down_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/down_proj", + "nvidia/Kimi-K2.5-NVFP4/down_proj" + ] }, { "type": "gemm", "args": { - "m": 2, + "m": 400, "n": 7168, - "k": 1152, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "k": 16384, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/o_proj", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] }, { "type": "gemm", "args": { - "m": 2048, - "n": 1536, + "m": 400, + "n": 24576, "k": 1536, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/q_b_proj" + ] }, { "type": "gemm", "args": { - "m": 2048, - "n": 2048, + "m": 400, + "n": 32768, "k": 512, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/kv_b_proj" + ] }, { "type": "gemm", "args": { - "m": 2048, - "n": 7168, - "k": 1024, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 401, + "n": 576, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/Kimi-K2.5-MXFP4/kv_a_proj_with_mqa", + "moonshotai/Kimi-K2.5/kv_a_proj_with_mqa", + "nvidia/DeepSeek-R1-0528-FP4-V2/kv_a_proj_with_mqa", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/kv_a_proj_with_mqa", + "nvidia/Kimi-K2.5-NVFP4/kv_a_proj_with_mqa", + "nvidia/Kimi-K2.6-NVFP4/kv_a_proj_with_mqa" + ] }, { "type": "gemm", "args": { - "m": 2048, - "n": 1152, + "m": 401, + "n": 576, "k": 7168, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 32 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 1, + 32 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "amd/DeepSeek-R1-0528-MXFP4-Preview/kv_a_proj_with_mqa", + "amd/DeepSeek-R1-0528-MXFP4-v2/kv_a_proj_with_mqa", + "amd/DeepSeek-R1-0528-MXFP4/kv_a_proj_with_mqa" + ] }, { "type": "gemm", "args": { - "m": 2048, - "n": 7168, - "k": 1152, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 401, + "n": 1536, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/Kimi-K2.5-MXFP4/q_a_proj", + "moonshotai/Kimi-K2.5/q_a_proj", + "nvidia/DeepSeek-R1-0528-FP4-V2/q_a_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/q_a_proj", + "nvidia/Kimi-K2.5-NVFP4/q_a_proj", + "nvidia/Kimi-K2.6-NVFP4/q_a_proj" + ] }, { "type": "gemm", "args": { - "m": 4, + "m": 401, "n": 1536, - "k": 1536, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "k": 7168, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 32 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 1, + 32 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "amd/DeepSeek-R1-0528-MXFP4-Preview/q_a_proj", + "amd/DeepSeek-R1-0528-MXFP4-v2/q_a_proj", + "amd/DeepSeek-R1-0528-MXFP4/q_a_proj" + ] }, { "type": "gemm", "args": { - "m": 4, - "n": 2048, - "k": 512, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 401, + "n": 1536, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w3" + ] }, { "type": "gemm", "args": { - "m": 4, - "n": 7168, - "k": 1024, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 401, + "n": 4608, + "k": 7168, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/gate_up_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/gate_up_proj", + "nvidia/Kimi-K2.5-NVFP4/gate_up_proj" + ] }, { "type": "gemm", "args": { - "m": 4, - "n": 1152, + "m": 401, + "n": 4608, "k": 7168, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 32 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 1, + 32 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "amd/DeepSeek-R1-0528-MXFP4-Preview/gate_up_proj", + "amd/DeepSeek-R1-0528-MXFP4-v2/gate_up_proj", + "amd/DeepSeek-R1-0528-MXFP4/gate_up_proj", + "amd/Kimi-K2.5-MXFP4/gate_up_proj" + ] }, { "type": "gemm", "args": { - "m": 4, + "m": 401, "n": 7168, - "k": 1152, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "k": 4608, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/down_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/down_proj", + "nvidia/Kimi-K2.5-NVFP4/down_proj" + ] }, { "type": "gemm", "args": { - "m": 4096, - "n": 1536, - "k": 1536, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 401, + "n": 7168, + "k": 4608, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 32 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 1, + 32 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "amd/DeepSeek-R1-0528-MXFP4-Preview/down_proj", + "amd/DeepSeek-R1-0528-MXFP4-v2/down_proj", + "amd/DeepSeek-R1-0528-MXFP4/down_proj", + "amd/Kimi-K2.5-MXFP4/down_proj" + ] }, { "type": "gemm", "args": { - "m": 4096, - "n": 2048, - "k": 512, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 401, + "n": 7168, + "k": 16384, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/o_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/o_proj" + ] }, { "type": "gemm", "args": { - "m": 4096, + "m": 401, "n": 7168, - "k": 1024, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "k": 16384, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 32 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 1, + 32 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "amd/DeepSeek-R1-0528-MXFP4-Preview/o_proj", + "amd/DeepSeek-R1-0528-MXFP4-v2/o_proj", + "amd/DeepSeek-R1-0528-MXFP4/o_proj" + ] }, { "type": "gemm", "args": { - "m": 4096, - "n": 1152, - "k": 7168, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 401, + "n": 7168, + "k": 16384, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] }, { "type": "gemm", "args": { - "m": 4096, - "n": 7168, - "k": 1152, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 401, + "n": 24576, + "k": 1536, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/q_b_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/q_b_proj" + ] }, { "type": "gemm", "args": { - "m": 8, - "n": 1536, + "m": 401, + "n": 24576, "k": 1536, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 32 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 1, + 32 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "amd/DeepSeek-R1-0528-MXFP4-Preview/q_b_proj", + "amd/DeepSeek-R1-0528-MXFP4-v2/q_b_proj", + "amd/DeepSeek-R1-0528-MXFP4/q_b_proj" + ] }, { "type": "gemm", "args": { - "m": 8, - "n": 2048, + "m": 401, + "n": 32768, "k": 512, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/kv_b_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/kv_b_proj" + ] }, { "type": "gemm", "args": { - "m": 8, - "n": 7168, - "k": 1024, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 401, + "n": 32768, + "k": 512, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 32 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 1, + 32 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "amd/DeepSeek-R1-0528-MXFP4-Preview/kv_b_proj", + "amd/DeepSeek-R1-0528-MXFP4-v2/kv_b_proj", + "amd/DeepSeek-R1-0528-MXFP4/kv_b_proj" + ] }, { "type": "gemm", "args": { - "m": 8, - "n": 1152, + "m": 422, + "n": 576, "k": 7168, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 8, - "n": 7168, - "k": 1152, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/Kimi-K2.5-MXFP4/kv_a_proj_with_mqa", + "moonshotai/Kimi-K2.5/kv_a_proj_with_mqa", + "nvidia/DeepSeek-R1-0528-FP4-V2/kv_a_proj_with_mqa", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/kv_a_proj_with_mqa", + "nvidia/Kimi-K2.5-NVFP4/kv_a_proj_with_mqa", + "nvidia/Kimi-K2.6-NVFP4/kv_a_proj_with_mqa" + ] }, { "type": "gemm", "args": { - "m": 8192, + "m": 422, "n": 1536, - "k": 1536, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/Kimi-K2.5-MXFP4/q_a_proj", + "moonshotai/Kimi-K2.5/q_a_proj", + "nvidia/DeepSeek-R1-0528-FP4-V2/q_a_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/q_a_proj", + "nvidia/Kimi-K2.5-NVFP4/q_a_proj", + "nvidia/Kimi-K2.6-NVFP4/q_a_proj" + ] }, { "type": "gemm", "args": { - "m": 8192, - "n": 2048, - "k": 512, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 422, + "n": 2304, + "k": 7168, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/gate_up_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/gate_up_proj", + "nvidia/Kimi-K2.5-NVFP4/gate_up_proj" + ] }, { "type": "gemm", "args": { - "m": 8192, + "m": 422, "n": 7168, - "k": 1024, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 8192, - "n": 1152, - "k": 7168, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "k": 2304, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/down_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/down_proj", + "nvidia/Kimi-K2.5-NVFP4/down_proj" + ] }, { "type": "gemm", "args": { - "m": 8192, + "m": 422, "n": 7168, - "k": 1152, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "k": 16384, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/o_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/o_proj" + ] }, { "type": "gemm", "args": { - "m": 16, - "n": 1536, + "m": 422, + "n": 24576, "k": 1536, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/q_b_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/q_b_proj" + ] }, { "type": "gemm", "args": { - "m": 16, - "n": 2048, + "m": 422, + "n": 32768, "k": 512, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/kv_b_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/kv_b_proj" + ] }, { "type": "gemm", "args": { - "m": 16, - "n": 7168, - "k": 1024, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 437, + "n": 576, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/Kimi-K2.5-MXFP4/kv_a_proj_with_mqa", + "moonshotai/Kimi-K2.5/kv_a_proj_with_mqa", + "nvidia/DeepSeek-R1-0528-FP4-V2/kv_a_proj_with_mqa", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/kv_a_proj_with_mqa", + "nvidia/Kimi-K2.5-NVFP4/kv_a_proj_with_mqa", + "nvidia/Kimi-K2.6-NVFP4/kv_a_proj_with_mqa" + ] }, { "type": "gemm", "args": { - "m": 16, - "n": 1152, + "m": 437, + "n": 576, "k": 7168, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 32 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 1, + 32 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "amd/DeepSeek-R1-0528-MXFP4-Preview/kv_a_proj_with_mqa", + "amd/DeepSeek-R1-0528-MXFP4-v2/kv_a_proj_with_mqa", + "amd/DeepSeek-R1-0528-MXFP4/kv_a_proj_with_mqa" + ] }, { "type": "gemm", "args": { - "m": 16, - "n": 7168, - "k": 1152, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 437, + "n": 1536, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/Kimi-K2.5-MXFP4/q_a_proj", + "moonshotai/Kimi-K2.5/q_a_proj", + "nvidia/DeepSeek-R1-0528-FP4-V2/q_a_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/q_a_proj", + "nvidia/Kimi-K2.5-NVFP4/q_a_proj", + "nvidia/Kimi-K2.6-NVFP4/q_a_proj" + ] }, { "type": "gemm", "args": { - "m": 16384, + "m": 437, "n": 1536, - "k": 1536, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "k": 7168, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 32 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 1, + 32 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "amd/DeepSeek-R1-0528-MXFP4-Preview/q_a_proj", + "amd/DeepSeek-R1-0528-MXFP4-v2/q_a_proj", + "amd/DeepSeek-R1-0528-MXFP4/q_a_proj" + ] }, { "type": "gemm", "args": { - "m": 16384, - "n": 2048, - "k": 512, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 437, + "n": 1536, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w3" + ] }, { "type": "gemm", "args": { - "m": 16384, - "n": 7168, - "k": 1024, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 437, + "n": 4608, + "k": 7168, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/gate_up_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/gate_up_proj", + "nvidia/Kimi-K2.5-NVFP4/gate_up_proj" + ] }, { "type": "gemm", "args": { - "m": 16384, - "n": 1152, + "m": 437, + "n": 4608, "k": 7168, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 32 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 1, + 32 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "amd/DeepSeek-R1-0528-MXFP4-Preview/gate_up_proj", + "amd/DeepSeek-R1-0528-MXFP4-v2/gate_up_proj", + "amd/DeepSeek-R1-0528-MXFP4/gate_up_proj", + "amd/Kimi-K2.5-MXFP4/gate_up_proj" + ] }, { "type": "gemm", "args": { - "m": 16384, + "m": 437, "n": 7168, - "k": 1152, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "k": 4608, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/down_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/down_proj", + "nvidia/Kimi-K2.5-NVFP4/down_proj" + ] }, { "type": "gemm", "args": { - "m": 32, - "n": 1536, - "k": 1536, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 437, + "n": 7168, + "k": 4608, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 32 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 1, + 32 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "amd/DeepSeek-R1-0528-MXFP4-Preview/down_proj", + "amd/DeepSeek-R1-0528-MXFP4-v2/down_proj", + "amd/DeepSeek-R1-0528-MXFP4/down_proj", + "amd/Kimi-K2.5-MXFP4/down_proj" + ] }, { "type": "gemm", "args": { - "m": 32, - "n": 2048, - "k": 512, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 437, + "n": 7168, + "k": 16384, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/o_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/o_proj" + ] }, { "type": "gemm", "args": { - "m": 32, + "m": 437, "n": 7168, - "k": 1024, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "k": 16384, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 32 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 1, + 32 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "amd/DeepSeek-R1-0528-MXFP4-Preview/o_proj", + "amd/DeepSeek-R1-0528-MXFP4-v2/o_proj", + "amd/DeepSeek-R1-0528-MXFP4/o_proj" + ] }, { "type": "gemm", "args": { - "m": 32, - "n": 1152, - "k": 7168, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 437, + "n": 7168, + "k": 16384, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] }, { "type": "gemm", "args": { - "m": 32, - "n": 7168, - "k": 1152, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 437, + "n": 24576, + "k": 1536, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/q_b_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/q_b_proj" + ] }, { "type": "gemm", "args": { - "m": 32768, - "n": 1536, + "m": 437, + "n": 24576, "k": 1536, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 32 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 1, + 32 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "amd/DeepSeek-R1-0528-MXFP4-Preview/q_b_proj", + "amd/DeepSeek-R1-0528-MXFP4-v2/q_b_proj", + "amd/DeepSeek-R1-0528-MXFP4/q_b_proj" + ] }, { "type": "gemm", "args": { - "m": 32768, - "n": 2048, + "m": 437, + "n": 32768, "k": 512, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/kv_b_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/kv_b_proj" + ] }, { "type": "gemm", "args": { - "m": 32768, - "n": 7168, - "k": 1024, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 437, + "n": 32768, + "k": 512, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 32 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 1, + 32 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "amd/DeepSeek-R1-0528-MXFP4-Preview/kv_b_proj", + "amd/DeepSeek-R1-0528-MXFP4-v2/kv_b_proj", + "amd/DeepSeek-R1-0528-MXFP4/kv_b_proj" + ] }, { "type": "gemm", "args": { - "m": 32768, - "n": 1152, + "m": 450, + "n": 576, "k": 7168, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 32768, - "n": 7168, - "k": 1152, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/Kimi-K2.5-MXFP4/kv_a_proj_with_mqa", + "moonshotai/Kimi-K2.5/kv_a_proj_with_mqa", + "nvidia/DeepSeek-R1-0528-FP4-V2/kv_a_proj_with_mqa", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/kv_a_proj_with_mqa", + "nvidia/Kimi-K2.5-NVFP4/kv_a_proj_with_mqa", + "nvidia/Kimi-K2.6-NVFP4/kv_a_proj_with_mqa" + ] }, { "type": "gemm", "args": { - "m": 64, + "m": 450, "n": 1536, - "k": 1536, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 64, - "n": 2048, - "k": 512, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 64, - "n": 7168, - "k": 1024, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 64, - "n": 1152, "k": 7168, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/Kimi-K2.5-MXFP4/q_a_proj", + "moonshotai/Kimi-K2.5/q_a_proj", + "nvidia/DeepSeek-R1-0528-FP4-V2/q_a_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/q_a_proj", + "nvidia/Kimi-K2.5-NVFP4/q_a_proj", + "nvidia/Kimi-K2.6-NVFP4/q_a_proj" + ] }, { "type": "gemm", "args": { - "m": 64, - "n": 7168, - "k": 1152, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 450, + "n": 2304, + "k": 7168, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/gate_up_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/gate_up_proj", + "nvidia/Kimi-K2.5-NVFP4/gate_up_proj" + ] }, { "type": "gemm", "args": { - "m": 65536, - "n": 1536, + "m": 450, + "n": 3072, "k": 1536, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/Kimi-K2.5-MXFP4/q_b_proj", + "amd/MiniMax-M2.5-MXFP4/o_proj", + "moonshotai/Kimi-K2.5/q_b_proj", + "nvidia/DeepSeek-R1-0528-FP4-V2/q_b_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/q_b_proj", + "nvidia/Kimi-K2.5-NVFP4/q_b_proj", + "nvidia/Kimi-K2.6-NVFP4/q_b_proj", + "nvidia/MiniMax-M2.5-NVFP4/o_proj" + ] }, { "type": "gemm", "args": { - "m": 65536, - "n": 2048, + "m": 450, + "n": 4096, "k": 512, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "Qwen/Qwen3.5-397B-A17B/o_proj", + "Qwen/Qwen3.5-397B-A17B/shared_expert.down_proj", + "amd/Kimi-K2.5-MXFP4/kv_b_proj", + "amd/Qwen3.5-397B-A17B-MXFP4/o_proj", + "amd/Qwen3.5-397B-A17B-MXFP4/shared_expert.down_proj", + "moonshotai/Kimi-K2.5/kv_b_proj", + "nvidia/DeepSeek-R1-0528-FP4-V2/kv_b_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/kv_b_proj", + "nvidia/Kimi-K2.5-NVFP4/kv_b_proj", + "nvidia/Kimi-K2.6-NVFP4/kv_b_proj", + "nvidia/Qwen3.5-397B-A17B-NVFP4/o_proj", + "nvidia/Qwen3.5-397B-A17B-NVFP4/shared_expert.down_proj" + ] }, { "type": "gemm", "args": { - "m": 65536, + "m": 450, "n": 7168, - "k": 1024, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "k": 2048, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/o_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/o_proj" + ] }, { "type": "gemm", "args": { - "m": 65536, - "n": 1152, - "k": 7168, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 450, + "n": 7168, + "k": 2304, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/down_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/down_proj", + "nvidia/Kimi-K2.5-NVFP4/down_proj" + ] }, { "type": "gemm", "args": { - "m": 65536, - "n": 7168, - "k": 1152, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 512, + "n": 128, + "k": 2880, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/gpt-oss-120b-w-mxfp4-a-fp8/k_proj", + "amd/gpt-oss-120b-w-mxfp4-a-fp8/v_proj", + "openai/gpt-oss-120b/k_proj", + "openai/gpt-oss-120b/v_proj" + ] }, { "type": "gemm", "args": { - "m": 128, - "n": 1536, - "k": 1536, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 512, + "n": 128, + "k": 3072, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "MiniMaxAI/MiniMax-M2.5/k_proj", + "MiniMaxAI/MiniMax-M2.5/v_proj" + ] }, { "type": "gemm", "args": { - "m": 128, - "n": 2048, - "k": 512, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 512, + "n": 128, + "k": 4096, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "Qwen/Qwen3.5-397B-A17B-FP8/shared_expert.gate_up_proj" + ] }, { "type": "gemm", "args": { - "m": 128, - "n": 7168, - "k": 1024, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 512, + "n": 192, + "k": 3072, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "MiniMaxAI/MiniMax-M2.5/experts.gate_up_proj" + ] }, { "type": "gemm", "args": { - "m": 128, - "n": 1152, - "k": 7168, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 512, + "n": 256, + "k": 3072, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "MiniMaxAI/MiniMax-M2.5/k_proj", + "MiniMaxAI/MiniMax-M2.5/v_proj" + ] }, { "type": "gemm", "args": { - "m": 128, - "n": 7168, - "k": 1152, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 512, + "n": 256, + "k": 4096, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "Qwen/Qwen3.5-397B-A17B/k_proj", + "Qwen/Qwen3.5-397B-A17B/shared_expert.gate_up_proj", + "Qwen/Qwen3.5-397B-A17B/v_proj", + "amd/Qwen3.5-397B-A17B-MXFP4/k_proj", + "amd/Qwen3.5-397B-A17B-MXFP4/shared_expert.gate_up_proj", + "amd/Qwen3.5-397B-A17B-MXFP4/v_proj", + "nvidia/Qwen3.5-397B-A17B-NVFP4/k_proj", + "nvidia/Qwen3.5-397B-A17B-NVFP4/shared_expert.gate_up_proj", + "nvidia/Qwen3.5-397B-A17B-NVFP4/v_proj" + ] }, { "type": "gemm", "args": { - "m": 131072, - "n": 1536, - "k": 1536, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 512, + "n": 256, + "k": 4096, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "Qwen/Qwen3.5-397B-A17B-FP8/k_proj", + "Qwen/Qwen3.5-397B-A17B-FP8/shared_expert.gate_up_proj", + "Qwen/Qwen3.5-397B-A17B-FP8/v_proj" + ] }, { "type": "gemm", "args": { - "m": 131072, - "n": 2048, - "k": 512, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 512, + "n": 384, + "k": 3072, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "MiniMaxAI/MiniMax-M2.5/experts.gate_up_proj", + "MiniMaxAI/MiniMax-M2.5/q_proj" + ] }, { "type": "gemm", "args": { - "m": 131072, - "n": 7168, - "k": 1024, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 512, + "n": 512, + "k": 2880, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/gpt-oss-120b-w-mxfp4-a-fp8/k_proj", + "amd/gpt-oss-120b-w-mxfp4-a-fp8/q_proj", + "amd/gpt-oss-120b-w-mxfp4-a-fp8/v_proj", + "openai/gpt-oss-120b/k_proj", + "openai/gpt-oss-120b/q_proj", + "openai/gpt-oss-120b/v_proj" + ] }, { "type": "gemm", "args": { - "m": 131072, - "n": 1152, - "k": 7168, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 512, + "n": 512, + "k": 3072, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/MiniMax-M2.5-MXFP4/k_proj", + "amd/MiniMax-M2.5-MXFP4/v_proj", + "nvidia/MiniMax-M2.5-NVFP4/k_proj", + "nvidia/MiniMax-M2.5-NVFP4/v_proj" + ] }, { "type": "gemm", "args": { - "m": 131072, - "n": 7168, - "k": 1152, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 512, + "n": 512, + "k": 3072, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "MiniMaxAI/MiniMax-M2.5/k_proj", + "MiniMaxAI/MiniMax-M2.5/v_proj" + ] }, { "type": "gemm", "args": { - "m": 256, - "n": 1536, - "k": 1536, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 512, + "n": 512, + "k": 4096, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "Qwen/Qwen3.5-397B-A17B-FP8/k_proj", + "Qwen/Qwen3.5-397B-A17B-FP8/q_proj", + "Qwen/Qwen3.5-397B-A17B-FP8/shared_expert.gate_up_proj", + "Qwen/Qwen3.5-397B-A17B-FP8/v_proj", + "deepseek-ai/DeepSeek-V4.1-Flash/attn.wo_a" + ] }, { "type": "gemm", "args": { - "m": 256, - "n": 2048, - "k": 512, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 512, + "n": 576, + "k": 6144, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/GLM-5.1-MXFP4/kv_a_proj_with_mqa", + "amd/GLM-5.2-MXFP4/kv_a_proj_with_mqa", + "nvidia/GLM-5-NVFP4/kv_a_proj_with_mqa", + "nvidia/GLM-5.1-NVFP4/kv_a_proj_with_mqa", + "nvidia/GLM-5.2-NVFP4/kv_a_proj_with_mqa" + ] }, { "type": "gemm", "args": { - "m": 256, - "n": 7168, - "k": 1024, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 512, + "n": 576, + "k": 6144, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "zai-org/GLM-5-FP8/kv_a_proj_with_mqa", + "zai-org/GLM-5.1-FP8/kv_a_proj_with_mqa", + "zai-org/GLM-5.2-FP8/kv_a_proj_with_mqa" + ] }, { "type": "gemm", "args": { - "m": 256, - "n": 1152, + "m": 512, + "n": 576, "k": 7168, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/Kimi-K2.5-MXFP4/kv_a_proj_with_mqa", + "moonshotai/Kimi-K2.5/kv_a_proj_with_mqa", + "nvidia/DeepSeek-R1-0528-FP4-V2/kv_a_proj_with_mqa", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/kv_a_proj_with_mqa", + "nvidia/Kimi-K2.5-NVFP4/kv_a_proj_with_mqa", + "nvidia/Kimi-K2.6-NVFP4/kv_a_proj_with_mqa" + ] }, { "type": "gemm", "args": { - "m": 256, - "n": 7168, - "k": 1152, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 512, + "n": 576, + "k": 7168, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 32 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 1, + 32 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "amd/DeepSeek-R1-0528-MXFP4-Preview/kv_a_proj_with_mqa", + "amd/DeepSeek-R1-0528-MXFP4-v2/kv_a_proj_with_mqa", + "amd/DeepSeek-R1-0528-MXFP4/kv_a_proj_with_mqa" + ] }, { "type": "gemm", "args": { "m": 512, - "n": 1152, + "n": 576, "k": 7168, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/kv_a_proj_with_mqa" + ] }, { "type": "gemm", "args": { "m": 512, - "n": 7168, - "k": 1152, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "n": 768, + "k": 3072, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "MiniMaxAI/MiniMax-M2.5/experts.gate_up_proj", + "MiniMaxAI/MiniMax-M2.5/q_proj" + ] }, { "type": "gemm", "args": { - "m": 262144, - "n": 1536, - "k": 1536, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 512, + "n": 1024, + "k": 2880, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/gpt-oss-120b-w-mxfp4-a-fp8/q_proj", + "openai/gpt-oss-120b/q_proj" + ] }, { "type": "gemm", "args": { - "m": 262144, - "n": 2048, - "k": 512, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 512, + "n": 1024, + "k": 3072, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "MiniMaxAI/MiniMax-M2.5/k_proj", + "MiniMaxAI/MiniMax-M2.5/qkv_proj", + "MiniMaxAI/MiniMax-M2.5/v_proj" + ] }, { "type": "gemm", "args": { - "m": 262144, - "n": 7168, - "k": 1024, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 512, + "n": 1024, + "k": 4096, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "Qwen/Qwen3.5-397B-A17B/q_proj", + "Qwen/Qwen3.5-397B-A17B/shared_expert.gate_up_proj", + "amd/Qwen3.5-397B-A17B-MXFP4/q_proj", + "amd/Qwen3.5-397B-A17B-MXFP4/shared_expert.gate_up_proj", + "nvidia/Qwen3.5-397B-A17B-NVFP4/q_proj", + "nvidia/Qwen3.5-397B-A17B-NVFP4/shared_expert.gate_up_proj" + ] }, { "type": "gemm", "args": { - "m": 262144, - "n": 1152, - "k": 7168, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 512, + "n": 1024, + "k": 4096, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "Qwen/Qwen3.5-397B-A17B-FP8/q_proj", + "Qwen/Qwen3.5-397B-A17B-FP8/shared_expert.gate_up_proj", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_a", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_a", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_a" + ] }, { "type": "gemm", "args": { - "m": 262144, - "n": 7168, - "k": 1152, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 512, + "n": 1088, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] }, { "type": "gemm", "args": { - "m": 524288, - "n": 1536, - "k": 1536, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 512, + "n": 1152, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/gate_up_proj" + ] }, { "type": "gemm", "args": { - "m": 524288, - "n": 2048, - "k": 512, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 512, + "n": 1536, + "k": 3072, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/MiniMax-M2.5-NVFP4/experts.gate_up_proj" + ] }, { "type": "gemm", "args": { - "m": 524288, - "n": 7168, - "k": 1024, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 512, + "n": 1536, + "k": 3072, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 32 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 1, + 32 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "amd/MiniMax-M2.5-MXFP4/experts.gate_up_proj" + ] }, { "type": "gemm", "args": { - "m": 524288, - "n": 1152, - "k": 7168, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 512, + "n": 1536, + "k": 3072, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "MiniMaxAI/MiniMax-M2.5/experts.gate_up_proj", + "MiniMaxAI/MiniMax-M2.5/q_proj" + ] }, { "type": "gemm", "args": { - "m": 524288, - "n": 7168, - "k": 1152, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 512, + "n": 1536, + "k": 6144, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "zai-org/GLM-5-FP8/gate_up_proj", + "zai-org/GLM-5.1-FP8/gate_up_proj", + "zai-org/GLM-5.2-FP8/gate_up_proj" + ] }, { "type": "gemm", "args": { - "m": 1048576, + "m": 512, "n": 1536, - "k": 1536, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/Kimi-K2.5-MXFP4/q_a_proj", + "moonshotai/Kimi-K2.5/q_a_proj", + "nvidia/DeepSeek-R1-0528-FP4-V2/q_a_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/q_a_proj", + "nvidia/Kimi-K2.5-NVFP4/q_a_proj", + "nvidia/Kimi-K2.6-NVFP4/q_a_proj" + ] }, { "type": "gemm", "args": { - "m": 1048576, - "n": 2048, - "k": 512, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 512, + "n": 1536, + "k": 7168, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 32 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 1, + 32 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "amd/DeepSeek-R1-0528-MXFP4-Preview/q_a_proj", + "amd/DeepSeek-R1-0528-MXFP4-v2/q_a_proj", + "amd/DeepSeek-R1-0528-MXFP4/q_a_proj" + ] }, { "type": "gemm", "args": { - "m": 1048576, - "n": 7168, - "k": 1024, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 512, + "n": 1536, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/gate_up_proj", + "deepseek-ai/DeepSeek-R1-0528/q_a_proj", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w3" + ] }, { "type": "gemm", "args": { - "m": 1048576, - "n": 1152, + "m": 512, + "n": 1536, "k": 7168, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w3" + ] }, { "type": "gemm", "args": { - "m": 1048576, - "n": 7168, - "k": 1152, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 512, + "n": 2048, + "k": 2048, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "zai-org/GLM-5-FP8/q_b_proj", + "zai-org/GLM-5.1-FP8/q_b_proj", + "zai-org/GLM-5.2-FP8/q_b_proj" + ] }, { "type": "gemm", "args": { - "m": 262144, - "n": 64, - "k": 2880, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 512, + "n": 2048, + "k": 4096, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "Qwen/Qwen3.5-397B-A17B-FP8/q_proj", + "Qwen/Qwen3.5-397B-A17B-FP8/shared_expert.gate_up_proj" + ] }, { "type": "gemm", "args": { - "m": 262144, - "n": 2880, - "k": 512, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 512, + "n": 2048, + "k": 6144, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/GLM-5.1-MXFP4/q_a_proj", + "amd/GLM-5.2-MXFP4/q_a_proj", + "amd/MiniMax-M3-MXFP4/q_proj", + "nvidia/GLM-5-NVFP4/q_a_proj", + "nvidia/GLM-5.1-NVFP4/q_a_proj", + "nvidia/GLM-5.2-NVFP4/q_a_proj" + ] }, { "type": "gemm", "args": { - "m": 4096, - "n": 768, - "k": 3072, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 512, + "n": 2048, + "k": 6144, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "zai-org/GLM-5-FP8/q_a_proj", + "zai-org/GLM-5.1-FP8/q_a_proj", + "zai-org/GLM-5.2-FP8/q_a_proj" + ] }, { "type": "gemm", "args": { - "m": 4096, - "n": 128, - "k": 3072, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 512, + "n": 2304, + "k": 7168, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/gate_up_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/gate_up_proj", + "nvidia/Kimi-K2.5-NVFP4/gate_up_proj" + ] }, { "type": "gemm", "args": { - "m": 4096, - "n": 3072, - "k": 768, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 512, + "n": 2304, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/gate_up_proj" + ] }, { "type": "gemm", "args": { - "m": 4096, - "n": 192, - "k": 3072, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 512, + "n": 2880, + "k": 1024, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/gpt-oss-120b-w-mxfp4-a-fp8/o_proj", + "openai/gpt-oss-120b/o_proj" + ] }, { "type": "gemm", "args": { - "m": 4096, - "n": 3072, - "k": 192, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 512, + "n": 2880, + "k": 4096, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/gpt-oss-120b-w-mxfp4-a-fp8/o_proj", + "openai/gpt-oss-120b/o_proj" + ] }, { "type": "gemm", "args": { - "m": 4, - "n": 128, - "k": 3072, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 512, + "n": 3072, + "k": 192, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "MiniMaxAI/MiniMax-M2.5/experts.down_proj" + ] }, { "type": "gemm", "args": { - "m": 4, - "n": 192, - "k": 3072, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 512, + "n": 3072, + "k": 384, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "MiniMaxAI/MiniMax-M2.5/experts.down_proj", + "MiniMaxAI/MiniMax-M2.5/o_proj" + ] }, { "type": "gemm", "args": { - "m": 4, + "m": 512, "n": 3072, - "k": 192, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "k": 768, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/MiniMax-M2.5-NVFP4/experts.down_proj" + ] }, { "type": "gemm", "args": { - "m": 8192, - "n": 768, - "k": 3072, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 512, + "n": 3072, + "k": 768, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 32 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 1, + 32 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "amd/MiniMax-M2.5-MXFP4/experts.down_proj" + ] }, { "type": "gemm", "args": { - "m": 8192, - "n": 128, - "k": 3072, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 512, + "n": 3072, + "k": 768, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "MiniMaxAI/MiniMax-M2.5/experts.down_proj", + "MiniMaxAI/MiniMax-M2.5/o_proj" + ] }, { "type": "gemm", "args": { - "m": 8192, + "m": 512, "n": 3072, - "k": 768, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "k": 1536, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/Kimi-K2.5-MXFP4/q_b_proj", + "amd/MiniMax-M2.5-MXFP4/o_proj", + "moonshotai/Kimi-K2.5/q_b_proj", + "nvidia/DeepSeek-R1-0528-FP4-V2/q_b_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/q_b_proj", + "nvidia/Kimi-K2.5-NVFP4/q_b_proj", + "nvidia/Kimi-K2.6-NVFP4/q_b_proj", + "nvidia/MiniMax-M2.5-NVFP4/o_proj" + ] }, { "type": "gemm", "args": { - "m": 8192, - "n": 192, - "k": 3072, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 512, + "n": 3072, + "k": 1536, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 32 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 1, + 32 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "amd/DeepSeek-R1-0528-MXFP4-Preview/q_b_proj", + "amd/DeepSeek-R1-0528-MXFP4-v2/q_b_proj", + "amd/DeepSeek-R1-0528-MXFP4/q_b_proj", + "amd/MiniMax-M2.5-MXFP4/experts.down_proj" + ] }, { "type": "gemm", "args": { - "m": 8192, + "m": 512, "n": 3072, - "k": 192, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "MiniMaxAI/MiniMax-M2.5/experts.down_proj", + "MiniMaxAI/MiniMax-M2.5/o_proj", + "deepseek-ai/DeepSeek-R1-0528/q_b_proj" + ] }, { "type": "gemm", "args": { - "m": 8, - "n": 128, + "m": 512, + "n": 3072, "k": 3072, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/MiniMax-M2.5-MXFP4/o_proj", + "amd/MiniMax-M2.5-MXFP4/q_proj", + "nvidia/MiniMax-M2.5-NVFP4/o_proj", + "nvidia/MiniMax-M2.5-NVFP4/q_proj" + ] }, { "type": "gemm", "args": { - "m": 8, - "n": 192, + "m": 512, + "n": 3072, "k": 3072, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 32 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 1, + 32 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "amd/MiniMax-M2.5-MXFP4/experts.gate_up_proj" + ] }, { "type": "gemm", "args": { - "m": 8, + "m": 512, "n": 3072, - "k": 192, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "k": 3072, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "MiniMaxAI/MiniMax-M2.5/experts.gate_up_proj", + "MiniMaxAI/MiniMax-M2.5/o_proj", + "MiniMaxAI/MiniMax-M2.5/q_proj" + ] }, { "type": "gemm", "args": { - "m": 16384, - "n": 768, - "k": 3072, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 512, + "n": 3072, + "k": 6144, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/GLM-5.1-MXFP4/gate_up_proj", + "amd/GLM-5.2-MXFP4/gate_up_proj", + "amd/MiniMax-M2.5-MXFP4/o_proj", + "amd/MiniMax-M3-MXFP4/gate_up_proj", + "nvidia/GLM-5.1-NVFP4/gate_up_proj", + "nvidia/GLM-5.2-NVFP4/gate_up_proj", + "nvidia/MiniMax-M2.5-NVFP4/o_proj" + ] }, { "type": "gemm", "args": { - "m": 16384, - "n": 128, - "k": 3072, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 512, + "n": 3072, + "k": 6144, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "MiniMaxAI/MiniMax-M2.5/o_proj", + "zai-org/GLM-5-FP8/gate_up_proj", + "zai-org/GLM-5.1-FP8/gate_up_proj", + "zai-org/GLM-5.2-FP8/gate_up_proj" + ] }, { "type": "gemm", "args": { - "m": 16384, - "n": 3072, - "k": 768, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 512, + "n": 3584, + "k": 512, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "zai-org/GLM-5-FP8/kv_b_proj", + "zai-org/GLM-5.1-FP8/kv_b_proj", + "zai-org/GLM-5.2-FP8/kv_b_proj" + ] }, { "type": "gemm", "args": { - "m": 16384, - "n": 192, - "k": 3072, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 512, + "n": 4096, + "k": 128, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "Qwen/Qwen3.5-397B-A17B/shared_expert.down_proj", + "amd/Qwen3.5-397B-A17B-MXFP4/shared_expert.down_proj", + "nvidia/Qwen3.5-397B-A17B-NVFP4/shared_expert.down_proj" + ] }, { "type": "gemm", "args": { - "m": 16384, - "n": 3072, - "k": 192, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 512, + "n": 4096, + "k": 128, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "Qwen/Qwen3.5-397B-A17B-FP8/shared_expert.down_proj" + ] }, { "type": "gemm", "args": { - "m": 16, - "n": 128, - "k": 3072, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 512, + "n": 4096, + "k": 256, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "Qwen/Qwen3.5-397B-A17B-FP8/shared_expert.down_proj" + ] }, { "type": "gemm", "args": { - "m": 16, - "n": 192, - "k": 3072, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 512, + "n": 4096, + "k": 512, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "Qwen/Qwen3.5-397B-A17B/o_proj", + "Qwen/Qwen3.5-397B-A17B/shared_expert.down_proj", + "amd/Kimi-K2.5-MXFP4/kv_b_proj", + "amd/Qwen3.5-397B-A17B-MXFP4/o_proj", + "amd/Qwen3.5-397B-A17B-MXFP4/shared_expert.down_proj", + "moonshotai/Kimi-K2.5/kv_b_proj", + "nvidia/DeepSeek-R1-0528-FP4-V2/kv_b_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/kv_b_proj", + "nvidia/Kimi-K2.5-NVFP4/kv_b_proj", + "nvidia/Kimi-K2.6-NVFP4/kv_b_proj", + "nvidia/Qwen3.5-397B-A17B-NVFP4/o_proj", + "nvidia/Qwen3.5-397B-A17B-NVFP4/shared_expert.down_proj" + ] }, { "type": "gemm", "args": { - "m": 16, - "n": 3072, - "k": 192, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 512, + "n": 4096, + "k": 512, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 32 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 1, + 32 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "amd/DeepSeek-R1-0528-MXFP4-Preview/kv_b_proj", + "amd/DeepSeek-R1-0528-MXFP4-v2/kv_b_proj", + "amd/DeepSeek-R1-0528-MXFP4/kv_b_proj", + "amd/Qwen3.5-397B-A17B-MXFP4-AttnFP8-V2/shared_expert.down_proj" + ] }, { "type": "gemm", "args": { - "m": 32768, - "n": 128, - "k": 3072, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 512, + "n": 4096, + "k": 512, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "Qwen/Qwen3.5-397B-A17B-FP8/o_proj", + "Qwen/Qwen3.5-397B-A17B-FP8/shared_expert.down_proj", + "deepseek-ai/DeepSeek-R1-0528/kv_b_proj" + ] }, { "type": "gemm", "args": { - "m": 32768, - "n": 192, - "k": 3072, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 512, + "n": 4096, + "k": 1024, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "Qwen/Qwen3.5-397B-A17B/o_proj", + "Qwen/Qwen3.5-397B-A17B/shared_expert.down_proj", + "amd/Qwen3.5-397B-A17B-MXFP4/o_proj", + "amd/Qwen3.5-397B-A17B-MXFP4/shared_expert.down_proj", + "nvidia/Qwen3.5-397B-A17B-NVFP4/o_proj", + "nvidia/Qwen3.5-397B-A17B-NVFP4/shared_expert.down_proj" + ] }, { "type": "gemm", "args": { - "m": 32768, - "n": 3072, - "k": 192, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 512, + "n": 4096, + "k": 1024, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "Qwen/Qwen3.5-397B-A17B-FP8/o_proj", + "Qwen/Qwen3.5-397B-A17B-FP8/shared_expert.down_proj" + ] }, { "type": "gemm", "args": { - "m": 32, - "n": 128, - "k": 3072, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 512, + "n": 4096, + "k": 2048, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "Qwen/Qwen3.5-397B-A17B/o_proj", + "amd/GLM-5.1-MXFP4/q_b_proj", + "amd/GLM-5.2-MXFP4/q_b_proj", + "amd/Qwen3.5-397B-A17B-MXFP4/o_proj", + "nvidia/GLM-5-NVFP4/q_b_proj", + "nvidia/GLM-5.1-NVFP4/q_b_proj", + "nvidia/GLM-5.2-NVFP4/q_b_proj", + "nvidia/Qwen3.5-397B-A17B-NVFP4/o_proj" + ] }, { "type": "gemm", "args": { - "m": 32, - "n": 192, - "k": 3072, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 512, + "n": 4096, + "k": 2048, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "Qwen/Qwen3.5-397B-A17B-FP8/o_proj", + "zai-org/GLM-5-FP8/q_b_proj", + "zai-org/GLM-5.1-FP8/q_b_proj", + "zai-org/GLM-5.2-FP8/q_b_proj" + ] }, { "type": "gemm", "args": { - "m": 32, - "n": 3072, - "k": 192, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 512, + "n": 4096, + "k": 2880, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/gpt-oss-120b-w-mxfp4-a-fp8/q_proj", + "openai/gpt-oss-120b/q_proj" + ] }, { "type": "gemm", "args": { - "m": 65536, - "n": 128, - "k": 3072, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 512, + "n": 4096, + "k": 4096, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "Qwen/Qwen3.5-397B-A17B-FP8/o_proj", + "Qwen/Qwen3.5-397B-A17B-FP8/q_proj" + ] }, { "type": "gemm", "args": { - "m": 65536, - "n": 192, - "k": 3072, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 512, + "n": 4608, + "k": 7168, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/gate_up_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/gate_up_proj", + "nvidia/Kimi-K2.5-NVFP4/gate_up_proj" + ] }, { "type": "gemm", "args": { - "m": 65536, - "n": 3072, - "k": 192, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 512, + "n": 4608, + "k": 7168, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 32 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 1, + 32 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "amd/DeepSeek-R1-0528-MXFP4-Preview/gate_up_proj", + "amd/DeepSeek-R1-0528-MXFP4-v2/gate_up_proj", + "amd/DeepSeek-R1-0528-MXFP4/gate_up_proj", + "amd/Kimi-K2.5-MXFP4/gate_up_proj" + ] }, { "type": "gemm", "args": { - "m": 64, - "n": 128, - "k": 3072, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 512, + "n": 4608, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/gate_up_proj" + ] }, { "type": "gemm", "args": { - "m": 64, - "n": 192, - "k": 3072, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 512, + "n": 6144, + "k": 1536, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 32 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 1, + 32 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "amd/DeepSeek-R1-0528-MXFP4-Preview/q_b_proj", + "amd/DeepSeek-R1-0528-MXFP4-v2/q_b_proj", + "amd/DeepSeek-R1-0528-MXFP4/q_b_proj" + ] }, { "type": "gemm", "args": { - "m": 64, - "n": 3072, - "k": 192, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 512, + "n": 6144, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/q_b_proj", + "zai-org/GLM-5-FP8/down_proj", + "zai-org/GLM-5.1-FP8/down_proj", + "zai-org/GLM-5.2-FP8/down_proj" + ] }, { "type": "gemm", "args": { - "m": 131072, - "n": 128, - "k": 3072, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 512, + "n": 6144, + "k": 2048, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "zai-org/GLM-5-FP8/o_proj", + "zai-org/GLM-5.1-FP8/o_proj", + "zai-org/GLM-5.2-FP8/o_proj" + ] }, { "type": "gemm", "args": { - "m": 131072, - "n": 192, + "m": 512, + "n": 6144, "k": 3072, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/GLM-5-NVFP4/down_proj" + ] }, { "type": "gemm", "args": { - "m": 131072, - "n": 3072, - "k": 192, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 512, + "n": 6144, + "k": 4096, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/GLM-5.1-MXFP4/o_proj", + "amd/GLM-5.2-MXFP4/o_proj", + "amd/MiniMax-M3-MXFP4/o_proj", + "nvidia/GLM-5-NVFP4/o_proj", + "nvidia/GLM-5.1-NVFP4/o_proj", + "nvidia/GLM-5.2-NVFP4/o_proj" + ] }, { "type": "gemm", "args": { - "m": 128, - "n": 128, - "k": 3072, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 512, + "n": 7168, + "k": 512, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/GLM-5.1-MXFP4/kv_b_proj", + "amd/GLM-5.2-MXFP4/kv_b_proj", + "amd/Kimi-K2.5-MXFP4/o_proj", + "moonshotai/Kimi-K2.5/o_proj", + "nvidia/GLM-5-NVFP4/kv_b_proj", + "nvidia/GLM-5.1-NVFP4/kv_b_proj", + "nvidia/GLM-5.2-NVFP4/kv_b_proj", + "nvidia/Kimi-K2.5-NVFP4/o_proj", + "nvidia/Kimi-K2.6-NVFP4/o_proj" + ] }, { "type": "gemm", "args": { - "m": 128, - "n": 192, - "k": 3072, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 512, + "n": 7168, + "k": 1152, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/down_proj" + ] }, { "type": "gemm", "args": { - "m": 128, - "n": 3072, - "k": 192, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 512, + "n": 7168, + "k": 2048, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/o_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/o_proj" + ] }, { "type": "gemm", "args": { - "m": 262144, - "n": 128, - "k": 3072, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 512, + "n": 7168, + "k": 2048, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 32 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 1, + 32 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "amd/DeepSeek-R1-0528-MXFP4-Preview/o_proj", + "amd/DeepSeek-R1-0528-MXFP4-v2/o_proj", + "amd/DeepSeek-R1-0528-MXFP4/o_proj" + ] }, { "type": "gemm", "args": { - "m": 262144, - "n": 192, - "k": 3072, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 512, + "n": 7168, + "k": 2048, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/o_proj", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] }, { "type": "gemm", "args": { - "m": 262144, - "n": 3072, - "k": 192, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 512, + "n": 7168, + "k": 2304, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/down_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/down_proj", + "nvidia/Kimi-K2.5-NVFP4/down_proj" + ] }, { "type": "gemm", "args": { - "m": 524288, - "n": 128, - "k": 3072, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 512, + "n": 7168, + "k": 2304, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 32 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 1, + 32 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "amd/DeepSeek-R1-0528-MXFP4-Preview/down_proj", + "amd/DeepSeek-R1-0528-MXFP4-v2/down_proj", + "amd/DeepSeek-R1-0528-MXFP4/down_proj", + "amd/Kimi-K2.5-MXFP4/down_proj" + ] }, { "type": "gemm", "args": { - "m": 524288, - "n": 192, - "k": 3072, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 512, + "n": 7168, + "k": 2304, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/down_proj" + ] }, { "type": "gemm", "args": { - "m": 524288, - "n": 3072, - "k": 192, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 512, + "n": 7168, + "k": 4096, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 32 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 1, + 32 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "amd/DeepSeek-R1-0528-MXFP4-Preview/o_proj", + "amd/DeepSeek-R1-0528-MXFP4-v2/o_proj", + "amd/DeepSeek-R1-0528-MXFP4/o_proj" + ] }, { "type": "gemm", "args": { - "m": 1048576, - "n": 128, - "k": 3072, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 512, + "n": 7168, + "k": 4608, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 32 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 1, + 32 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "amd/DeepSeek-R1-0528-MXFP4-Preview/down_proj", + "amd/DeepSeek-R1-0528-MXFP4-v2/down_proj", + "amd/DeepSeek-R1-0528-MXFP4/down_proj", + "amd/Kimi-K2.5-MXFP4/down_proj" + ] }, { "type": "gemm", "args": { - "m": 1048576, - "n": 192, - "k": 3072, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 512, + "n": 7168, + "k": 8192, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] }, { "type": "gemm", "args": { - "m": 1048576, - "n": 3072, - "k": 192, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 512, + "n": 7168, + "k": 16384, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/o_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/o_proj" + ] }, { "type": "gemm", "args": { - "m": 5760, - "n": 1536, - "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 512, + "n": 7168, + "k": 16384, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 32 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 1, + 32 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "amd/DeepSeek-R1-0528-MXFP4-Preview/o_proj", + "amd/DeepSeek-R1-0528-MXFP4-v2/o_proj", + "amd/DeepSeek-R1-0528-MXFP4/o_proj" + ] }, { "type": "gemm", "args": { - "m": 5760, - "n": 24576, - "k": 1536, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 512, + "n": 7168, + "k": 16384, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/o_proj", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] }, { "type": "gemm", "args": { - "m": 5760, - "n": 576, - "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 512, + "n": 7168, + "k": 16384, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] }, { "type": "gemm", "args": { - "m": 5760, - "n": 32768, - "k": 512, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 512, + "n": 7168, + "k": 65536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] }, { "type": "gemm", "args": { - "m": 5760, - "n": 7168, - "k": 16384, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 512, + "n": 8192, + "k": 512, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 32 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 1, + 32 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "amd/DeepSeek-R1-0528-MXFP4-Preview/kv_b_proj", + "amd/DeepSeek-R1-0528-MXFP4-v2/kv_b_proj", + "amd/DeepSeek-R1-0528-MXFP4/kv_b_proj" + ] }, { "type": "gemm", "args": { - "m": 5760, - "n": 7168, - "k": 576, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 512, + "n": 8192, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_b" + ] }, { "type": "gemm", "args": { - "m": 11, - "n": 7168, - "k": 576, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 512, + "n": 12288, + "k": 1536, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/Kimi-K2.5-MXFP4/q_b_proj", + "moonshotai/Kimi-K2.5/q_b_proj", + "nvidia/DeepSeek-R1-0528-FP4-V2/q_b_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/q_b_proj", + "nvidia/Kimi-K2.5-NVFP4/q_b_proj", + "nvidia/Kimi-K2.6-NVFP4/q_b_proj" + ] }, { "type": "gemm", "args": { - "m": 3072, - "n": 4608, - "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 512, + "n": 15360, + "k": 1024, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] }, { "type": "gemm", "args": { - "m": 3072, - "n": 7168, - "k": 4608, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 512, + "n": 16384, + "k": 512, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/Kimi-K2.5-MXFP4/kv_b_proj", + "moonshotai/Kimi-K2.5/kv_b_proj", + "nvidia/DeepSeek-R1-0528-FP4-V2/kv_b_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/kv_b_proj", + "nvidia/Kimi-K2.5-NVFP4/kv_b_proj", + "nvidia/Kimi-K2.6-NVFP4/kv_b_proj" + ] }, { "type": "gemm", "args": { - "m": 6144, - "n": 24576, + "m": 512, + "n": 16384, "k": 1536, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_b" + ] }, { "type": "gemm", "args": { - "m": 6144, - "n": 32768, - "k": 512, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 512, + "n": 24576, + "k": 1536, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/q_b_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/q_b_proj" + ] }, { "type": "gemm", "args": { - "m": 6144, - "n": 7168, - "k": 16384, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 512, + "n": 24576, + "k": 1536, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 32 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 1, + 32 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "amd/DeepSeek-R1-0528-MXFP4-Preview/q_b_proj", + "amd/DeepSeek-R1-0528-MXFP4-v2/q_b_proj", + "amd/DeepSeek-R1-0528-MXFP4/q_b_proj" + ] }, { "type": "gemm", "args": { - "m": 6144, - "n": 4608, - "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 512, + "n": 24576, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/q_b_proj" + ] }, { "type": "gemm", "args": { - "m": 6144, - "n": 7168, - "k": 4608, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 512, + "n": 30720, + "k": 1024, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] }, { "type": "gemm", "args": { - "m": 24, - "n": 24576, - "k": 1536, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 512, + "n": 32768, + "k": 512, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/kv_b_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/kv_b_proj" + ] }, { "type": "gemm", "args": { - "m": 24, + "m": 512, "n": 32768, "k": 512, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 32 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 1, + 32 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "amd/DeepSeek-R1-0528-MXFP4-Preview/kv_b_proj", + "amd/DeepSeek-R1-0528-MXFP4-v2/kv_b_proj", + "amd/DeepSeek-R1-0528-MXFP4/kv_b_proj" + ] }, { "type": "gemm", "args": { - "m": 24, - "n": 7168, - "k": 16384, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 512, + "n": 32768, + "k": 512, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/kv_b_proj" + ] }, { "type": "gemm", "args": { - "m": 12288, - "n": 3072, + "m": 512, + "n": 65536, "k": 1536, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_b" + ] }, { "type": "gemm", "args": { - "m": 12288, - "n": 4096, - "k": 512, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 512, + "n": 122880, + "k": 1024, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] }, { "type": "gemm", "args": { - "m": 12288, - "n": 7168, - "k": 2048, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 528, + "n": 576, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/kv_a_proj_with_mqa" + ] }, { "type": "gemm", "args": { - "m": 12288, - "n": 2304, + "m": 528, + "n": 1536, "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/gate_up_proj", + "deepseek-ai/DeepSeek-R1-0528/q_a_proj", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w3" + ] }, { "type": "gemm", "args": { - "m": 12288, - "n": 7168, - "k": 2304, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 528, + "n": 1536, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w3" + ] }, { "type": "gemm", "args": { - "m": 24, - "n": 3072, - "k": 1536, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 528, + "n": 4608, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/gate_up_proj" + ] }, { "type": "gemm", "args": { - "m": 24, - "n": 4096, - "k": 512, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 528, + "n": 7168, + "k": 4608, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/down_proj" + ] }, { "type": "gemm", "args": { - "m": 24, + "m": 528, "n": 7168, - "k": 2048, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "k": 16384, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/o_proj", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] }, { "type": "gemm", "args": { - "m": 24, - "n": 2304, - "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 528, + "n": 24576, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/q_b_proj" + ] }, { "type": "gemm", "args": { - "m": 24, - "n": 7168, - "k": 2304, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 528, + "n": 32768, + "k": 512, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/kv_b_proj" + ] }, { "type": "gemm", "args": { - "m": 24576, - "n": 1536, + "m": 537, + "n": 576, "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/Kimi-K2.5-MXFP4/kv_a_proj_with_mqa", + "moonshotai/Kimi-K2.5/kv_a_proj_with_mqa", + "nvidia/DeepSeek-R1-0528-FP4-V2/kv_a_proj_with_mqa", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/kv_a_proj_with_mqa", + "nvidia/Kimi-K2.5-NVFP4/kv_a_proj_with_mqa", + "nvidia/Kimi-K2.6-NVFP4/kv_a_proj_with_mqa" + ] }, { "type": "gemm", "args": { - "m": 24576, - "n": 3072, - "k": 1536, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 537, + "n": 576, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/kv_a_proj_with_mqa" + ] }, { "type": "gemm", "args": { - "m": 24576, - "n": 576, + "m": 537, + "n": 1152, "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-NVFP4-v2/gate_up_proj" + ] }, { "type": "gemm", "args": { - "m": 24576, - "n": 4096, - "k": 512, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 537, + "n": 1536, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/Kimi-K2.5-MXFP4/q_a_proj", + "moonshotai/Kimi-K2.5/q_a_proj", + "nvidia/DeepSeek-R1-0528-FP4-V2/q_a_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/q_a_proj", + "nvidia/Kimi-K2.5-NVFP4/q_a_proj", + "nvidia/Kimi-K2.6-NVFP4/q_a_proj" + ] }, { "type": "gemm", "args": { - "m": 24576, - "n": 7168, - "k": 2048, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 537, + "n": 1536, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/gate_up_proj", + "deepseek-ai/DeepSeek-R1-0528/q_a_proj", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w3" + ] }, { "type": "gemm", "args": { - "m": 24576, + "m": 537, "n": 2304, "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/gate_up_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/gate_up_proj", + "nvidia/Kimi-K2.5-NVFP4/gate_up_proj" + ] }, { "type": "gemm", "args": { - "m": 24576, - "n": 7168, - "k": 2304, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 537, + "n": 4608, + "k": 7168, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/gate_up_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/gate_up_proj", + "nvidia/Kimi-K2.5-NVFP4/gate_up_proj" + ] }, { "type": "gemm", "args": { - "m": 48, - "n": 3072, - "k": 1536, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 537, + "n": 4608, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/gate_up_proj" + ] }, { "type": "gemm", "args": { - "m": 48, - "n": 4096, - "k": 512, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 537, + "n": 7168, + "k": 1152, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/down_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/down_proj", + "nvidia/Kimi-K2.5-NVFP4/down_proj" + ] }, { "type": "gemm", "args": { - "m": 48, + "m": 537, "n": 7168, - "k": 2048, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "k": 2304, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/down_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/down_proj", + "nvidia/Kimi-K2.5-NVFP4/down_proj" + ] }, { "type": "gemm", "args": { - "m": 48, - "n": 2304, - "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 537, + "n": 7168, + "k": 2304, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/down_proj" + ] }, { "type": "gemm", "args": { - "m": 48, + "m": 537, "n": 7168, - "k": 2304, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "k": 4608, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/down_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/down_proj", + "nvidia/Kimi-K2.5-NVFP4/down_proj" + ] }, { "type": "gemm", "args": { - "m": 96, - "n": 1536, - "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 537, + "n": 7168, + "k": 4608, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 32 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 1, + 32 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "amd/DeepSeek-R1-0528-MXFP4-Preview/down_proj", + "amd/DeepSeek-R1-0528-MXFP4-v2/down_proj", + "amd/DeepSeek-R1-0528-MXFP4/down_proj", + "amd/Kimi-K2.5-MXFP4/down_proj" + ] }, { "type": "gemm", "args": { - "m": 96, - "n": 3072, - "k": 1536, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 537, + "n": 7168, + "k": 4608, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/down_proj" + ] }, { "type": "gemm", "args": { - "m": 96, - "n": 576, - "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 537, + "n": 7168, + "k": 8192, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/Kimi-K2.5-MXFP4/o_proj", + "moonshotai/Kimi-K2.5/o_proj", + "nvidia/Kimi-K2.5-NVFP4/o_proj", + "nvidia/Kimi-K2.6-NVFP4/o_proj" + ] }, { "type": "gemm", "args": { - "m": 96, - "n": 4096, - "k": 512, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 537, + "n": 7168, + "k": 16384, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/o_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/o_proj" + ] }, { "type": "gemm", "args": { - "m": 96, + "m": 537, "n": 7168, - "k": 2048, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "k": 16384, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/o_proj", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] }, { "type": "gemm", "args": { - "m": 96, - "n": 2304, - "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 537, + "n": 12288, + "k": 1536, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/Kimi-K2.5-MXFP4/q_b_proj", + "moonshotai/Kimi-K2.5/q_b_proj", + "nvidia/DeepSeek-R1-0528-FP4-V2/q_b_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/q_b_proj", + "nvidia/Kimi-K2.5-NVFP4/q_b_proj", + "nvidia/Kimi-K2.6-NVFP4/q_b_proj" + ] }, { "type": "gemm", "args": { - "m": 96, - "n": 7168, - "k": 2304, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 537, + "n": 16384, + "k": 512, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/Kimi-K2.5-MXFP4/kv_b_proj", + "moonshotai/Kimi-K2.5/kv_b_proj", + "nvidia/DeepSeek-R1-0528-FP4-V2/kv_b_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/kv_b_proj", + "nvidia/Kimi-K2.5-NVFP4/kv_b_proj", + "nvidia/Kimi-K2.6-NVFP4/kv_b_proj" + ] }, { "type": "gemm", "args": { - "m": 1101056, - "n": 1536, - "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 537, + "n": 24576, + "k": 1536, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/q_b_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/q_b_proj" + ] }, { "type": "gemm", "args": { - "m": 1101056, + "m": 537, "n": 24576, "k": 1536, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/q_b_proj" + ] }, { "type": "gemm", "args": { - "m": 1101056, - "n": 576, - "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 537, + "n": 32768, + "k": 512, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/kv_b_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/kv_b_proj" + ] }, { "type": "gemm", "args": { - "m": 1101056, + "m": 537, "n": 32768, "k": 512, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/kv_b_proj" + ] }, { "type": "gemm", "args": { - "m": 1101056, - "n": 7168, - "k": 16384, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 546, + "n": 576, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/Kimi-K2.5-MXFP4/kv_a_proj_with_mqa", + "moonshotai/Kimi-K2.5/kv_a_proj_with_mqa", + "nvidia/DeepSeek-R1-0528-FP4-V2/kv_a_proj_with_mqa", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/kv_a_proj_with_mqa", + "nvidia/Kimi-K2.5-NVFP4/kv_a_proj_with_mqa", + "nvidia/Kimi-K2.6-NVFP4/kv_a_proj_with_mqa" + ] }, { "type": "gemm", "args": { - "m": 1101056, - "n": 4608, + "m": 546, + "n": 576, "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 32 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 1, + 32 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "amd/DeepSeek-R1-0528-MXFP4-Preview/kv_a_proj_with_mqa", + "amd/DeepSeek-R1-0528-MXFP4-v2/kv_a_proj_with_mqa", + "amd/DeepSeek-R1-0528-MXFP4/kv_a_proj_with_mqa" + ] }, { "type": "gemm", "args": { - "m": 1101056, - "n": 7168, - "k": 4608, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 546, + "n": 1536, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/Kimi-K2.5-MXFP4/q_a_proj", + "moonshotai/Kimi-K2.5/q_a_proj", + "nvidia/DeepSeek-R1-0528-FP4-V2/q_a_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/q_a_proj", + "nvidia/Kimi-K2.5-NVFP4/q_a_proj", + "nvidia/Kimi-K2.6-NVFP4/q_a_proj" + ] }, { "type": "gemm", "args": { - "m": 2150, + "m": 546, "n": 1536, "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 32 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 1, + 32 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "amd/DeepSeek-R1-0528-MXFP4-Preview/q_a_proj", + "amd/DeepSeek-R1-0528-MXFP4-v2/q_a_proj", + "amd/DeepSeek-R1-0528-MXFP4/q_a_proj" + ] }, { "type": "gemm", "args": { - "m": 2150, - "n": 24576, - "k": 1536, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 546, + "n": 1536, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w3" + ] }, { "type": "gemm", "args": { - "m": 2150, - "n": 576, - "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 546, + "n": 7168, + "k": 9216, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/down_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/down_proj", + "nvidia/Kimi-K2.5-NVFP4/down_proj" + ] }, { "type": "gemm", "args": { - "m": 2150, - "n": 32768, - "k": 512, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 546, + "n": 7168, + "k": 16384, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/o_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/o_proj" + ] }, { "type": "gemm", "args": { - "m": 2150, + "m": 546, "n": 7168, "k": 16384, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 32 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 1, + 32 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "amd/DeepSeek-R1-0528-MXFP4-Preview/o_proj", + "amd/DeepSeek-R1-0528-MXFP4-v2/o_proj", + "amd/DeepSeek-R1-0528-MXFP4/o_proj" + ] }, { "type": "gemm", "args": { - "m": 2150, - "n": 4608, - "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 546, + "n": 7168, + "k": 16384, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] }, { "type": "gemm", "args": { - "m": 2150, - "n": 7168, - "k": 4608, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 546, + "n": 9216, + "k": 7168, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/gate_up_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/gate_up_proj", + "nvidia/Kimi-K2.5-NVFP4/gate_up_proj" + ] }, { "type": "gemm", "args": { - "m": 275264, - "n": 1536, + "m": 546, + "n": 9216, "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 32 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 1, + 32 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "amd/DeepSeek-R1-0528-MXFP4-Preview/gate_up_proj", + "amd/DeepSeek-R1-0528-MXFP4-v2/gate_up_proj", + "amd/DeepSeek-R1-0528-MXFP4/gate_up_proj", + "amd/Kimi-K2.5-MXFP4/gate_up_proj" + ] }, { "type": "gemm", "args": { - "m": 275264, + "m": 546, "n": 24576, "k": 1536, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/q_b_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/q_b_proj" + ] }, { "type": "gemm", "args": { - "m": 275264, - "n": 576, - "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 546, + "n": 24576, + "k": 1536, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 32 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 1, + 32 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "amd/DeepSeek-R1-0528-MXFP4-Preview/q_b_proj", + "amd/DeepSeek-R1-0528-MXFP4-v2/q_b_proj", + "amd/DeepSeek-R1-0528-MXFP4/q_b_proj" + ] }, { "type": "gemm", "args": { - "m": 275264, + "m": 546, "n": 32768, "k": 512, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/kv_b_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/kv_b_proj" + ] }, { "type": "gemm", "args": { - "m": 275264, - "n": 7168, - "k": 16384, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 546, + "n": 32768, + "k": 512, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 32 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 1, + 32 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "amd/DeepSeek-R1-0528-MXFP4-Preview/kv_b_proj", + "amd/DeepSeek-R1-0528-MXFP4-v2/kv_b_proj", + "amd/DeepSeek-R1-0528-MXFP4/kv_b_proj" + ] }, { "type": "gemm", "args": { - "m": 275264, - "n": 1152, + "m": 548, + "n": 576, "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/Kimi-K2.5-MXFP4/kv_a_proj_with_mqa", + "moonshotai/Kimi-K2.5/kv_a_proj_with_mqa", + "nvidia/DeepSeek-R1-0528-FP4-V2/kv_a_proj_with_mqa", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/kv_a_proj_with_mqa", + "nvidia/Kimi-K2.5-NVFP4/kv_a_proj_with_mqa", + "nvidia/Kimi-K2.6-NVFP4/kv_a_proj_with_mqa" + ] }, { "type": "gemm", "args": { - "m": 275264, - "n": 7168, - "k": 1152, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 548, + "n": 576, + "k": 7168, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 32 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 1, + 32 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "amd/DeepSeek-R1-0528-MXFP4-Preview/kv_a_proj_with_mqa", + "amd/DeepSeek-R1-0528-MXFP4-v2/kv_a_proj_with_mqa", + "amd/DeepSeek-R1-0528-MXFP4/kv_a_proj_with_mqa" + ] }, { "type": "gemm", "args": { - "m": 537, + "m": 548, "n": 1536, "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/Kimi-K2.5-MXFP4/q_a_proj", + "moonshotai/Kimi-K2.5/q_a_proj", + "nvidia/DeepSeek-R1-0528-FP4-V2/q_a_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/q_a_proj", + "nvidia/Kimi-K2.5-NVFP4/q_a_proj", + "nvidia/Kimi-K2.6-NVFP4/q_a_proj" + ] }, { "type": "gemm", "args": { - "m": 537, - "n": 24576, - "k": 1536, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 548, + "n": 1536, + "k": 7168, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 32 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 1, + 32 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "amd/DeepSeek-R1-0528-MXFP4-Preview/q_a_proj", + "amd/DeepSeek-R1-0528-MXFP4-v2/q_a_proj", + "amd/DeepSeek-R1-0528-MXFP4/q_a_proj" + ] }, { "type": "gemm", "args": { - "m": 537, - "n": 576, + "m": 548, + "n": 1536, "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w3" + ] }, { "type": "gemm", "args": { - "m": 537, - "n": 32768, - "k": 512, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 548, + "n": 4608, + "k": 7168, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/gate_up_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/gate_up_proj", + "nvidia/Kimi-K2.5-NVFP4/gate_up_proj" + ] }, { "type": "gemm", "args": { - "m": 537, - "n": 7168, - "k": 16384, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 548, + "n": 4608, + "k": 7168, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 32 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 1, + 32 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "amd/DeepSeek-R1-0528-MXFP4-Preview/gate_up_proj", + "amd/DeepSeek-R1-0528-MXFP4-v2/gate_up_proj", + "amd/DeepSeek-R1-0528-MXFP4/gate_up_proj", + "amd/Kimi-K2.5-MXFP4/gate_up_proj" + ] }, { "type": "gemm", "args": { - "m": 537, - "n": 1152, - "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 548, + "n": 7168, + "k": 4608, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/down_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/down_proj", + "nvidia/Kimi-K2.5-NVFP4/down_proj" + ] }, { "type": "gemm", "args": { - "m": 537, + "m": 548, "n": 7168, - "k": 1152, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "k": 4608, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 32 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 1, + 32 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "amd/DeepSeek-R1-0528-MXFP4-Preview/down_proj", + "amd/DeepSeek-R1-0528-MXFP4-v2/down_proj", + "amd/DeepSeek-R1-0528-MXFP4/down_proj", + "amd/Kimi-K2.5-MXFP4/down_proj" + ] }, { "type": "gemm", "args": { - "m": 576768, - "n": 1536, - "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 548, + "n": 7168, + "k": 16384, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/o_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/o_proj" + ] }, { "type": "gemm", "args": { - "m": 576768, - "n": 24576, - "k": 1536, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 548, + "n": 7168, + "k": 16384, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 32 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 1, + 32 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "amd/DeepSeek-R1-0528-MXFP4-Preview/o_proj", + "amd/DeepSeek-R1-0528-MXFP4-v2/o_proj", + "amd/DeepSeek-R1-0528-MXFP4/o_proj" + ] }, { "type": "gemm", "args": { - "m": 576768, - "n": 576, - "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 548, + "n": 7168, + "k": 16384, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] }, { "type": "gemm", "args": { - "m": 576768, - "n": 32768, - "k": 512, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 548, + "n": 24576, + "k": 1536, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/q_b_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/q_b_proj" + ] }, { "type": "gemm", "args": { - "m": 576768, - "n": 7168, - "k": 16384, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 548, + "n": 24576, + "k": 1536, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 32 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 1, + 32 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "amd/DeepSeek-R1-0528-MXFP4-Preview/q_b_proj", + "amd/DeepSeek-R1-0528-MXFP4-v2/q_b_proj", + "amd/DeepSeek-R1-0528-MXFP4/q_b_proj" + ] }, { "type": "gemm", "args": { - "m": 576768, - "n": 4608, - "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 548, + "n": 32768, + "k": 512, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/kv_b_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/kv_b_proj" + ] }, { "type": "gemm", "args": { - "m": 576768, - "n": 7168, - "k": 4608, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 548, + "n": 32768, + "k": 512, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 32 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 1, + 32 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "amd/DeepSeek-R1-0528-MXFP4-Preview/kv_b_proj", + "amd/DeepSeek-R1-0528-MXFP4-v2/kv_b_proj", + "amd/DeepSeek-R1-0528-MXFP4/kv_b_proj" + ] }, { "type": "gemm", "args": { - "m": 1126, - "n": 1536, + "m": 549, + "n": 576, "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/Kimi-K2.5-MXFP4/kv_a_proj_with_mqa", + "moonshotai/Kimi-K2.5/kv_a_proj_with_mqa", + "nvidia/DeepSeek-R1-0528-FP4-V2/kv_a_proj_with_mqa", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/kv_a_proj_with_mqa", + "nvidia/Kimi-K2.5-NVFP4/kv_a_proj_with_mqa", + "nvidia/Kimi-K2.6-NVFP4/kv_a_proj_with_mqa" + ] }, { "type": "gemm", "args": { - "m": 1126, - "n": 24576, - "k": 1536, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 549, + "n": 1536, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/Kimi-K2.5-MXFP4/q_a_proj", + "moonshotai/Kimi-K2.5/q_a_proj", + "nvidia/DeepSeek-R1-0528-FP4-V2/q_a_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/q_a_proj", + "nvidia/Kimi-K2.5-NVFP4/q_a_proj", + "nvidia/Kimi-K2.6-NVFP4/q_a_proj" + ] }, { "type": "gemm", "args": { - "m": 1126, - "n": 576, + "m": 549, + "n": 2304, "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/gate_up_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/gate_up_proj", + "nvidia/Kimi-K2.5-NVFP4/gate_up_proj" + ] }, { "type": "gemm", "args": { - "m": 1126, - "n": 32768, - "k": 512, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 549, + "n": 7168, + "k": 2304, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/down_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/down_proj", + "nvidia/Kimi-K2.5-NVFP4/down_proj" + ] }, { "type": "gemm", "args": { - "m": 1126, + "m": 549, "n": 7168, "k": 16384, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/o_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/o_proj" + ] }, { "type": "gemm", "args": { - "m": 1126, - "n": 4608, - "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 549, + "n": 24576, + "k": 1536, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/q_b_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/q_b_proj" + ] }, { "type": "gemm", "args": { - "m": 1126, - "n": 7168, - "k": 4608, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 549, + "n": 32768, + "k": 512, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/kv_b_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/kv_b_proj" + ] }, { "type": "gemm", "args": { - "m": 72096, - "n": 1536, + "m": 550, + "n": 576, "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/Kimi-K2.5-MXFP4/kv_a_proj_with_mqa", + "moonshotai/Kimi-K2.5/kv_a_proj_with_mqa", + "nvidia/DeepSeek-R1-0528-FP4-V2/kv_a_proj_with_mqa", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/kv_a_proj_with_mqa", + "nvidia/Kimi-K2.5-NVFP4/kv_a_proj_with_mqa", + "nvidia/Kimi-K2.6-NVFP4/kv_a_proj_with_mqa" + ] }, { "type": "gemm", "args": { - "m": 72096, - "n": 24576, - "k": 1536, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 550, + "n": 1536, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/Kimi-K2.5-MXFP4/q_a_proj", + "moonshotai/Kimi-K2.5/q_a_proj", + "nvidia/DeepSeek-R1-0528-FP4-V2/q_a_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/q_a_proj", + "nvidia/Kimi-K2.5-NVFP4/q_a_proj", + "nvidia/Kimi-K2.6-NVFP4/q_a_proj" + ] }, { "type": "gemm", "args": { - "m": 72096, - "n": 576, + "m": 550, + "n": 2304, "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/gate_up_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/gate_up_proj", + "nvidia/Kimi-K2.5-NVFP4/gate_up_proj" + ] }, { "type": "gemm", "args": { - "m": 72096, - "n": 32768, - "k": 512, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 550, + "n": 7168, + "k": 2304, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/down_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/down_proj", + "nvidia/Kimi-K2.5-NVFP4/down_proj" + ] }, { "type": "gemm", "args": { - "m": 72096, + "m": 550, "n": 7168, "k": 16384, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/o_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/o_proj" + ] }, { "type": "gemm", "args": { - "m": 72096, - "n": 7168, - "k": 576, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 550, + "n": 24576, + "k": 1536, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/q_b_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/q_b_proj" + ] }, { "type": "gemm", "args": { - "m": 140, - "n": 1536, - "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 550, + "n": 32768, + "k": 512, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/kv_b_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/kv_b_proj" + ] }, { "type": "gemm", "args": { - "m": 140, - "n": 24576, - "k": 1536, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 555, + "n": 576, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/Kimi-K2.5-MXFP4/kv_a_proj_with_mqa", + "moonshotai/Kimi-K2.5/kv_a_proj_with_mqa", + "nvidia/DeepSeek-R1-0528-FP4-V2/kv_a_proj_with_mqa", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/kv_a_proj_with_mqa", + "nvidia/Kimi-K2.5-NVFP4/kv_a_proj_with_mqa", + "nvidia/Kimi-K2.6-NVFP4/kv_a_proj_with_mqa" + ] }, { "type": "gemm", "args": { - "m": 140, + "m": 555, "n": 576, "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 32 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 1, + 32 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "amd/DeepSeek-R1-0528-MXFP4-Preview/kv_a_proj_with_mqa", + "amd/DeepSeek-R1-0528-MXFP4-v2/kv_a_proj_with_mqa", + "amd/DeepSeek-R1-0528-MXFP4/kv_a_proj_with_mqa" + ] }, { "type": "gemm", "args": { - "m": 140, - "n": 32768, - "k": 512, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 555, + "n": 1536, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/Kimi-K2.5-MXFP4/q_a_proj", + "moonshotai/Kimi-K2.5/q_a_proj", + "nvidia/DeepSeek-R1-0528-FP4-V2/q_a_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/q_a_proj", + "nvidia/Kimi-K2.5-NVFP4/q_a_proj", + "nvidia/Kimi-K2.6-NVFP4/q_a_proj" + ] }, { "type": "gemm", "args": { - "m": 140, - "n": 7168, - "k": 16384, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 555, + "n": 1536, + "k": 7168, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 32 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 1, + 32 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "amd/DeepSeek-R1-0528-MXFP4-Preview/q_a_proj", + "amd/DeepSeek-R1-0528-MXFP4-v2/q_a_proj", + "amd/DeepSeek-R1-0528-MXFP4/q_a_proj" + ] }, { "type": "gemm", "args": { - "m": 140, - "n": 7168, - "k": 576, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 555, + "n": 1536, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w3" + ] }, { "type": "gemm", "args": { - "m": 4129280, - "n": 1536, + "m": 555, + "n": 4608, "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/gate_up_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/gate_up_proj", + "nvidia/Kimi-K2.5-NVFP4/gate_up_proj" + ] }, { "type": "gemm", "args": { - "m": 4129280, - "n": 24576, - "k": 1536, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 555, + "n": 4608, + "k": 7168, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 32 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 1, + 32 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "amd/DeepSeek-R1-0528-MXFP4-Preview/gate_up_proj", + "amd/DeepSeek-R1-0528-MXFP4-v2/gate_up_proj", + "amd/DeepSeek-R1-0528-MXFP4/gate_up_proj", + "amd/Kimi-K2.5-MXFP4/gate_up_proj" + ] }, { "type": "gemm", "args": { - "m": 4129280, - "n": 576, - "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 555, + "n": 7168, + "k": 4608, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/down_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/down_proj", + "nvidia/Kimi-K2.5-NVFP4/down_proj" + ] }, { "type": "gemm", "args": { - "m": 4129280, - "n": 32768, - "k": 512, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 555, + "n": 7168, + "k": 4608, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 32 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 1, + 32 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "amd/DeepSeek-R1-0528-MXFP4-Preview/down_proj", + "amd/DeepSeek-R1-0528-MXFP4-v2/down_proj", + "amd/DeepSeek-R1-0528-MXFP4/down_proj", + "amd/Kimi-K2.5-MXFP4/down_proj" + ] }, { "type": "gemm", "args": { - "m": 4129280, + "m": 555, "n": 7168, "k": 16384, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/o_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/o_proj" + ] }, { "type": "gemm", "args": { - "m": 4129280, - "n": 4608, - "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 555, + "n": 7168, + "k": 16384, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 32 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 1, + 32 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "amd/DeepSeek-R1-0528-MXFP4-Preview/o_proj", + "amd/DeepSeek-R1-0528-MXFP4-v2/o_proj", + "amd/DeepSeek-R1-0528-MXFP4/o_proj" + ] }, { "type": "gemm", "args": { - "m": 4129280, + "m": 555, "n": 7168, - "k": 4608, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "k": 16384, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] }, { "type": "gemm", "args": { - "m": 8065, - "n": 1536, - "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 555, + "n": 24576, + "k": 1536, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/q_b_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/q_b_proj" + ] }, { "type": "gemm", "args": { - "m": 8065, + "m": 555, "n": 24576, "k": 1536, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 32 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 1, + 32 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "amd/DeepSeek-R1-0528-MXFP4-Preview/q_b_proj", + "amd/DeepSeek-R1-0528-MXFP4-v2/q_b_proj", + "amd/DeepSeek-R1-0528-MXFP4/q_b_proj" + ] }, { "type": "gemm", "args": { - "m": 8065, - "n": 576, - "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 555, + "n": 32768, + "k": 512, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/kv_b_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/kv_b_proj" + ] }, { "type": "gemm", "args": { - "m": 8065, + "m": 555, "n": 32768, "k": 512, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 32 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 1, + 32 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "amd/DeepSeek-R1-0528-MXFP4-Preview/kv_b_proj", + "amd/DeepSeek-R1-0528-MXFP4-v2/kv_b_proj", + "amd/DeepSeek-R1-0528-MXFP4/kv_b_proj" + ] }, { "type": "gemm", "args": { - "m": 8065, - "n": 7168, - "k": 16384, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 563, + "n": 576, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/Kimi-K2.5-MXFP4/kv_a_proj_with_mqa", + "moonshotai/Kimi-K2.5/kv_a_proj_with_mqa", + "nvidia/DeepSeek-R1-0528-FP4-V2/kv_a_proj_with_mqa", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/kv_a_proj_with_mqa", + "nvidia/Kimi-K2.5-NVFP4/kv_a_proj_with_mqa", + "nvidia/Kimi-K2.6-NVFP4/kv_a_proj_with_mqa" + ] + }, + { + "type": "gemm", + "args": { + "m": 563, + "n": 1536, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/Kimi-K2.5-MXFP4/q_a_proj", + "moonshotai/Kimi-K2.5/q_a_proj", + "nvidia/DeepSeek-R1-0528-FP4-V2/q_a_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/q_a_proj", + "nvidia/Kimi-K2.5-NVFP4/q_a_proj", + "nvidia/Kimi-K2.6-NVFP4/q_a_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 563, + "n": 2304, + "k": 7168, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/gate_up_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/gate_up_proj", + "nvidia/Kimi-K2.5-NVFP4/gate_up_proj" + ] }, { "type": "gemm", "args": { - "m": 8065, + "m": 563, "n": 4608, "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/gate_up_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/gate_up_proj", + "nvidia/Kimi-K2.5-NVFP4/gate_up_proj" + ] }, { "type": "gemm", "args": { - "m": 8065, + "m": 563, "n": 7168, - "k": 4608, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "k": 2304, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/down_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/down_proj", + "nvidia/Kimi-K2.5-NVFP4/down_proj" + ] }, { "type": "gemm", "args": { - "m": 1075, - "n": 1536, - "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 563, + "n": 7168, + "k": 4608, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/down_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/down_proj", + "nvidia/Kimi-K2.5-NVFP4/down_proj" + ] }, { "type": "gemm", "args": { - "m": 1075, - "n": 24576, - "k": 1536, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 563, + "n": 7168, + "k": 4608, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 32 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 1, + 32 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "amd/DeepSeek-R1-0528-MXFP4-Preview/down_proj", + "amd/DeepSeek-R1-0528-MXFP4-v2/down_proj", + "amd/DeepSeek-R1-0528-MXFP4/down_proj", + "amd/Kimi-K2.5-MXFP4/down_proj" + ] }, { "type": "gemm", "args": { - "m": 1075, - "n": 576, - "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 563, + "n": 7168, + "k": 4608, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/down_proj" + ] }, { "type": "gemm", "args": { - "m": 1075, - "n": 32768, - "k": 512, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 563, + "n": 7168, + "k": 8192, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/Kimi-K2.5-MXFP4/o_proj", + "moonshotai/Kimi-K2.5/o_proj", + "nvidia/Kimi-K2.5-NVFP4/o_proj", + "nvidia/Kimi-K2.6-NVFP4/o_proj" + ] }, { "type": "gemm", "args": { - "m": 1075, + "m": 563, "n": 7168, - "k": 16384, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "k": 8192, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] }, { "type": "gemm", "args": { - "m": 1075, - "n": 4608, - "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 563, + "n": 7168, + "k": 16384, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/o_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/o_proj" + ] }, { "type": "gemm", "args": { - "m": 1075, - "n": 7168, - "k": 4608, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 563, + "n": 12288, + "k": 1536, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/Kimi-K2.5-MXFP4/q_b_proj", + "moonshotai/Kimi-K2.5/q_b_proj", + "nvidia/DeepSeek-R1-0528-FP4-V2/q_b_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/q_b_proj", + "nvidia/Kimi-K2.5-NVFP4/q_b_proj", + "nvidia/Kimi-K2.6-NVFP4/q_b_proj" + ] }, { "type": "gemm", "args": { - "m": 550528, - "n": 1536, - "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 563, + "n": 16384, + "k": 512, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/Kimi-K2.5-MXFP4/kv_b_proj", + "moonshotai/Kimi-K2.5/kv_b_proj", + "nvidia/DeepSeek-R1-0528-FP4-V2/kv_b_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/kv_b_proj", + "nvidia/Kimi-K2.5-NVFP4/kv_b_proj", + "nvidia/Kimi-K2.6-NVFP4/kv_b_proj" + ] }, { "type": "gemm", "args": { - "m": 550528, + "m": 563, "n": 24576, "k": 1536, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 550528, - "n": 576, - "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/q_b_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/q_b_proj" + ] }, { "type": "gemm", "args": { - "m": 550528, + "m": 563, "n": 32768, "k": 512, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/kv_b_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/kv_b_proj" + ] }, { "type": "gemm", "args": { - "m": 550528, - "n": 7168, - "k": 16384, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 576, + "n": 576, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/kv_a_proj_with_mqa" + ] }, { "type": "gemm", "args": { - "m": 550528, - "n": 2304, + "m": 576, + "n": 1152, "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/gate_up_proj" + ] }, { "type": "gemm", "args": { - "m": 550528, - "n": 7168, - "k": 2304, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 576, + "n": 1536, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/gate_up_proj", + "deepseek-ai/DeepSeek-R1-0528/q_a_proj", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w3" + ] }, { "type": "gemm", "args": { - "m": 537, + "m": 576, "n": 2304, "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/gate_up_proj" + ] }, { "type": "gemm", "args": { - "m": 537, + "m": 576, "n": 7168, - "k": 2304, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "k": 1152, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/down_proj" + ] }, { "type": "gemm", "args": { - "m": 170496, - "n": 1536, - "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 576, + "n": 7168, + "k": 2304, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/down_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/down_proj", + "nvidia/Kimi-K2.5-NVFP4/down_proj" + ] }, { "type": "gemm", "args": { - "m": 170496, - "n": 24576, - "k": 1536, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 576, + "n": 7168, + "k": 16384, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/o_proj", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] }, { "type": "gemm", "args": { - "m": 170496, - "n": 576, - "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 576, + "n": 24576, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/q_b_proj" + ] }, { "type": "gemm", "args": { - "m": 170496, + "m": 576, "n": 32768, "k": 512, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/kv_b_proj" + ] }, { "type": "gemm", "args": { - "m": 170496, - "n": 7168, - "k": 16384, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 585, + "n": 576, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/Kimi-K2.5-MXFP4/kv_a_proj_with_mqa", + "moonshotai/Kimi-K2.5/kv_a_proj_with_mqa", + "nvidia/DeepSeek-R1-0528-FP4-V2/kv_a_proj_with_mqa", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/kv_a_proj_with_mqa", + "nvidia/Kimi-K2.5-NVFP4/kv_a_proj_with_mqa", + "nvidia/Kimi-K2.6-NVFP4/kv_a_proj_with_mqa" + ] }, { "type": "gemm", "args": { - "m": 170496, - "n": 4608, + "m": 585, + "n": 1536, "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/Kimi-K2.5-MXFP4/q_a_proj", + "moonshotai/Kimi-K2.5/q_a_proj", + "nvidia/DeepSeek-R1-0528-FP4-V2/q_a_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/q_a_proj", + "nvidia/Kimi-K2.5-NVFP4/q_a_proj", + "nvidia/Kimi-K2.6-NVFP4/q_a_proj" + ] }, { "type": "gemm", "args": { - "m": 170496, - "n": 7168, - "k": 4608, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 585, + "n": 2304, + "k": 7168, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/gate_up_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/gate_up_proj", + "nvidia/Kimi-K2.5-NVFP4/gate_up_proj" + ] }, { "type": "gemm", "args": { - "m": 166, - "n": 1536, - "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 585, + "n": 7168, + "k": 2304, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/down_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/down_proj", + "nvidia/Kimi-K2.5-NVFP4/down_proj" + ] }, { "type": "gemm", "args": { - "m": 166, - "n": 24576, - "k": 1536, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 585, + "n": 7168, + "k": 16384, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/o_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/o_proj" + ] }, { "type": "gemm", "args": { - "m": 166, - "n": 576, - "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 585, + "n": 24576, + "k": 1536, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/q_b_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/q_b_proj" + ] }, { "type": "gemm", "args": { - "m": 166, + "m": 585, "n": 32768, "k": 512, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/kv_b_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/kv_b_proj" + ] }, { "type": "gemm", "args": { - "m": 166, - "n": 7168, - "k": 16384, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 588, + "n": 576, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/Kimi-K2.5-MXFP4/kv_a_proj_with_mqa", + "moonshotai/Kimi-K2.5/kv_a_proj_with_mqa", + "nvidia/DeepSeek-R1-0528-FP4-V2/kv_a_proj_with_mqa", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/kv_a_proj_with_mqa", + "nvidia/Kimi-K2.5-NVFP4/kv_a_proj_with_mqa", + "nvidia/Kimi-K2.6-NVFP4/kv_a_proj_with_mqa" + ] }, { "type": "gemm", "args": { - "m": 166, - "n": 4608, + "m": 588, + "n": 576, "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 32 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 1, + 32 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "amd/DeepSeek-R1-0528-MXFP4-Preview/kv_a_proj_with_mqa", + "amd/DeepSeek-R1-0528-MXFP4-v2/kv_a_proj_with_mqa", + "amd/DeepSeek-R1-0528-MXFP4/kv_a_proj_with_mqa" + ] }, { "type": "gemm", "args": { - "m": 166, - "n": 7168, - "k": 4608, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 588, + "n": 1536, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/Kimi-K2.5-MXFP4/q_a_proj", + "moonshotai/Kimi-K2.5/q_a_proj", + "nvidia/DeepSeek-R1-0528-FP4-V2/q_a_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/q_a_proj", + "nvidia/Kimi-K2.5-NVFP4/q_a_proj", + "nvidia/Kimi-K2.6-NVFP4/q_a_proj" + ] }, { "type": "gemm", "args": { - "m": 21312, + "m": 588, "n": 1536, "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 32 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 1, + 32 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "amd/DeepSeek-R1-0528-MXFP4-Preview/q_a_proj", + "amd/DeepSeek-R1-0528-MXFP4-v2/q_a_proj", + "amd/DeepSeek-R1-0528-MXFP4/q_a_proj" + ] }, { "type": "gemm", "args": { - "m": 21312, - "n": 24576, - "k": 1536, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 588, + "n": 1536, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w3" + ] }, { "type": "gemm", "args": { - "m": 21312, - "n": 576, - "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 588, + "n": 7168, + "k": 9216, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/down_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/down_proj", + "nvidia/Kimi-K2.5-NVFP4/down_proj" + ] }, { "type": "gemm", "args": { - "m": 21312, - "n": 32768, - "k": 512, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 588, + "n": 7168, + "k": 16384, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/o_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/o_proj" + ] }, { "type": "gemm", "args": { - "m": 21312, + "m": 588, "n": 7168, "k": 16384, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 32 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 1, + 32 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "amd/DeepSeek-R1-0528-MXFP4-Preview/o_proj", + "amd/DeepSeek-R1-0528-MXFP4-v2/o_proj", + "amd/DeepSeek-R1-0528-MXFP4/o_proj" + ] }, { "type": "gemm", "args": { - "m": 21312, + "m": 588, "n": 7168, - "k": 576, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "k": 16384, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] }, { "type": "gemm", "args": { - "m": 20, - "n": 7168, - "k": 576, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 588, + "n": 9216, + "k": 7168, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/gate_up_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/gate_up_proj", + "nvidia/Kimi-K2.5-NVFP4/gate_up_proj" + ] }, { "type": "gemm", "args": { - "m": 1572864, - "n": 1536, + "m": 588, + "n": 9216, "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 32 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 1, + 32 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "amd/DeepSeek-R1-0528-MXFP4-Preview/gate_up_proj", + "amd/DeepSeek-R1-0528-MXFP4-v2/gate_up_proj", + "amd/DeepSeek-R1-0528-MXFP4/gate_up_proj", + "amd/Kimi-K2.5-MXFP4/gate_up_proj" + ] }, { "type": "gemm", "args": { - "m": 1572864, + "m": 588, "n": 24576, "k": 1536, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/q_b_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/q_b_proj" + ] }, { "type": "gemm", "args": { - "m": 1572864, - "n": 576, - "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 588, + "n": 24576, + "k": 1536, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 32 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 1, + 32 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "amd/DeepSeek-R1-0528-MXFP4-Preview/q_b_proj", + "amd/DeepSeek-R1-0528-MXFP4-v2/q_b_proj", + "amd/DeepSeek-R1-0528-MXFP4/q_b_proj" + ] }, { "type": "gemm", "args": { - "m": 1572864, + "m": 588, "n": 32768, "k": 512, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/kv_b_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/kv_b_proj" + ] }, { "type": "gemm", "args": { - "m": 1572864, - "n": 7168, - "k": 16384, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 588, + "n": 32768, + "k": 512, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 32 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 1, + 32 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "amd/DeepSeek-R1-0528-MXFP4-Preview/kv_b_proj", + "amd/DeepSeek-R1-0528-MXFP4-v2/kv_b_proj", + "amd/DeepSeek-R1-0528-MXFP4/kv_b_proj" + ] }, { "type": "gemm", "args": { - "m": 1572864, - "n": 4608, + "m": 607, + "n": 576, "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/Kimi-K2.5-MXFP4/kv_a_proj_with_mqa", + "moonshotai/Kimi-K2.5/kv_a_proj_with_mqa", + "nvidia/DeepSeek-R1-0528-FP4-V2/kv_a_proj_with_mqa", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/kv_a_proj_with_mqa", + "nvidia/Kimi-K2.5-NVFP4/kv_a_proj_with_mqa", + "nvidia/Kimi-K2.6-NVFP4/kv_a_proj_with_mqa" + ] }, { "type": "gemm", "args": { - "m": 1572864, - "n": 7168, - "k": 4608, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 607, + "n": 576, + "k": 7168, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 32 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 1, + 32 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "amd/DeepSeek-R1-0528-MXFP4-Preview/kv_a_proj_with_mqa", + "amd/DeepSeek-R1-0528-MXFP4-v2/kv_a_proj_with_mqa", + "amd/DeepSeek-R1-0528-MXFP4/kv_a_proj_with_mqa" + ] }, { "type": "gemm", "args": { - "m": 24576, - "n": 24576, - "k": 1536, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 607, + "n": 1536, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/Kimi-K2.5-MXFP4/q_a_proj", + "moonshotai/Kimi-K2.5/q_a_proj", + "nvidia/DeepSeek-R1-0528-FP4-V2/q_a_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/q_a_proj", + "nvidia/Kimi-K2.5-NVFP4/q_a_proj", + "nvidia/Kimi-K2.6-NVFP4/q_a_proj" + ] }, { "type": "gemm", "args": { - "m": 24576, - "n": 32768, - "k": 512, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 607, + "n": 1536, + "k": 7168, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 32 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 1, + 32 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "amd/DeepSeek-R1-0528-MXFP4-Preview/q_a_proj", + "amd/DeepSeek-R1-0528-MXFP4-v2/q_a_proj", + "amd/DeepSeek-R1-0528-MXFP4/q_a_proj" + ] }, { "type": "gemm", "args": { - "m": 24576, - "n": 7168, - "k": 16384, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 607, + "n": 1536, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w3" + ] }, { "type": "gemm", "args": { - "m": 24576, + "m": 607, "n": 4608, "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 24576, - "n": 7168, - "k": 4608, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/gate_up_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/gate_up_proj", + "nvidia/Kimi-K2.5-NVFP4/gate_up_proj" + ] }, { "type": "gemm", "args": { - "m": 49152, + "m": 607, "n": 4608, "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 32 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 1, + 32 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "amd/DeepSeek-R1-0528-MXFP4-Preview/gate_up_proj", + "amd/DeepSeek-R1-0528-MXFP4-v2/gate_up_proj", + "amd/DeepSeek-R1-0528-MXFP4/gate_up_proj", + "amd/Kimi-K2.5-MXFP4/gate_up_proj" + ] }, { "type": "gemm", "args": { - "m": 49152, + "m": 607, "n": 7168, "k": 4608, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 48, - "n": 24576, - "k": 1536, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/down_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/down_proj", + "nvidia/Kimi-K2.5-NVFP4/down_proj" + ] }, { "type": "gemm", "args": { - "m": 48, - "n": 32768, - "k": 512, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 607, + "n": 7168, + "k": 4608, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 32 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 1, + 32 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "amd/DeepSeek-R1-0528-MXFP4-Preview/down_proj", + "amd/DeepSeek-R1-0528-MXFP4-v2/down_proj", + "amd/DeepSeek-R1-0528-MXFP4/down_proj", + "amd/Kimi-K2.5-MXFP4/down_proj" + ] }, { "type": "gemm", "args": { - "m": 48, + "m": 607, "n": 7168, "k": 16384, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 196608, - "n": 3072, - "k": 1536, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/o_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/o_proj" + ] }, { "type": "gemm", "args": { - "m": 196608, - "n": 4096, - "k": 512, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 607, + "n": 7168, + "k": 16384, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 32 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 1, + 32 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "amd/DeepSeek-R1-0528-MXFP4-Preview/o_proj", + "amd/DeepSeek-R1-0528-MXFP4-v2/o_proj", + "amd/DeepSeek-R1-0528-MXFP4/o_proj" + ] }, { "type": "gemm", "args": { - "m": 196608, + "m": 607, "n": 7168, - "k": 2048, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "k": 16384, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] }, { "type": "gemm", "args": { - "m": 196608, - "n": 2304, - "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 607, + "n": 24576, + "k": 1536, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/q_b_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/q_b_proj" + ] }, { "type": "gemm", "args": { - "m": 196608, - "n": 7168, - "k": 2304, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 607, + "n": 24576, + "k": 1536, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 32 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 1, + 32 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "amd/DeepSeek-R1-0528-MXFP4-Preview/q_b_proj", + "amd/DeepSeek-R1-0528-MXFP4-v2/q_b_proj", + "amd/DeepSeek-R1-0528-MXFP4/q_b_proj" + ] }, { "type": "gemm", "args": { - "m": 192, - "n": 1536, - "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 607, + "n": 32768, + "k": 512, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/kv_b_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/kv_b_proj" + ] }, { "type": "gemm", "args": { - "m": 192, - "n": 3072, - "k": 1536, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 607, + "n": 32768, + "k": 512, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 32 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 1, + 32 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "amd/DeepSeek-R1-0528-MXFP4-Preview/kv_b_proj", + "amd/DeepSeek-R1-0528-MXFP4-v2/kv_b_proj", + "amd/DeepSeek-R1-0528-MXFP4/kv_b_proj" + ] }, { "type": "gemm", "args": { - "m": 192, + "m": 614, "n": 576, "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 192, - "n": 4096, - "k": 512, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/Kimi-K2.5-MXFP4/kv_a_proj_with_mqa", + "moonshotai/Kimi-K2.5/kv_a_proj_with_mqa", + "nvidia/DeepSeek-R1-0528-FP4-V2/kv_a_proj_with_mqa", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/kv_a_proj_with_mqa", + "nvidia/Kimi-K2.5-NVFP4/kv_a_proj_with_mqa", + "nvidia/Kimi-K2.6-NVFP4/kv_a_proj_with_mqa" + ] }, { "type": "gemm", "args": { - "m": 192, - "n": 7168, - "k": 2048, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 614, + "n": 576, + "k": 7168, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 32 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 1, + 32 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "amd/DeepSeek-R1-0528-MXFP4-Preview/kv_a_proj_with_mqa", + "amd/DeepSeek-R1-0528-MXFP4-v2/kv_a_proj_with_mqa", + "amd/DeepSeek-R1-0528-MXFP4/kv_a_proj_with_mqa" + ] }, { "type": "gemm", "args": { - "m": 192, - "n": 2304, + "m": 614, + "n": 576, "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/kv_a_proj_with_mqa" + ] }, { "type": "gemm", "args": { - "m": 192, - "n": 7168, - "k": 2304, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 614, + "n": 1536, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/Kimi-K2.5-MXFP4/q_a_proj", + "moonshotai/Kimi-K2.5/q_a_proj", + "nvidia/DeepSeek-R1-0528-FP4-V2/q_a_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/q_a_proj", + "nvidia/Kimi-K2.5-NVFP4/q_a_proj", + "nvidia/Kimi-K2.6-NVFP4/q_a_proj" + ] }, { "type": "gemm", "args": { - "m": 1280, + "m": 614, "n": 1536, "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 32 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 1, + 32 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "amd/DeepSeek-R1-0528-MXFP4-Preview/q_a_proj", + "amd/DeepSeek-R1-0528-MXFP4-v2/q_a_proj", + "amd/DeepSeek-R1-0528-MXFP4/q_a_proj" + ] }, { "type": "gemm", "args": { - "m": 1280, - "n": 24576, - "k": 1536, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 614, + "n": 1536, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/gate_up_proj", + "deepseek-ai/DeepSeek-R1-0528/q_a_proj", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w3" + ] }, { "type": "gemm", "args": { - "m": 1280, - "n": 576, + "m": 614, + "n": 1536, "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w3" + ] }, { "type": "gemm", "args": { - "m": 1280, - "n": 32768, - "k": 512, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 614, + "n": 4608, + "k": 7168, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/gate_up_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/gate_up_proj", + "nvidia/Kimi-K2.5-NVFP4/gate_up_proj" + ] }, { "type": "gemm", "args": { - "m": 1280, - "n": 7168, - "k": 16384, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 614, + "n": 4608, + "k": 7168, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 32 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 1, + 32 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "amd/DeepSeek-R1-0528-MXFP4-Preview/gate_up_proj", + "amd/DeepSeek-R1-0528-MXFP4-v2/gate_up_proj", + "amd/DeepSeek-R1-0528-MXFP4/gate_up_proj", + "amd/Kimi-K2.5-MXFP4/gate_up_proj" + ] }, { "type": "gemm", "args": { - "m": 1280, + "m": 614, "n": 4608, "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/gate_up_proj" + ] }, { "type": "gemm", "args": { - "m": 1280, + "m": 614, "n": 7168, "k": 4608, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 5120, - "n": 3072, - "k": 1536, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/down_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/down_proj", + "nvidia/Kimi-K2.5-NVFP4/down_proj" + ] }, { "type": "gemm", "args": { - "m": 5120, - "n": 4096, - "k": 512, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 614, + "n": 7168, + "k": 4608, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 32 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 1, + 32 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "amd/DeepSeek-R1-0528-MXFP4-Preview/down_proj", + "amd/DeepSeek-R1-0528-MXFP4-v2/down_proj", + "amd/DeepSeek-R1-0528-MXFP4/down_proj", + "amd/Kimi-K2.5-MXFP4/down_proj" + ] }, { "type": "gemm", "args": { - "m": 5120, + "m": 614, "n": 7168, - "k": 2048, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "k": 4608, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/down_proj" + ] }, { "type": "gemm", "args": { - "m": 5120, - "n": 2304, - "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 614, + "n": 7168, + "k": 9216, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/down_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/down_proj", + "nvidia/Kimi-K2.5-NVFP4/down_proj" + ] }, { "type": "gemm", "args": { - "m": 5120, + "m": 614, "n": 7168, - "k": 2304, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "k": 16384, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/o_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/o_proj" + ] }, { "type": "gemm", "args": { - "m": 5, - "n": 3072, - "k": 1536, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 614, + "n": 7168, + "k": 16384, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 32 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 1, + 32 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "amd/DeepSeek-R1-0528-MXFP4-Preview/o_proj", + "amd/DeepSeek-R1-0528-MXFP4-v2/o_proj", + "amd/DeepSeek-R1-0528-MXFP4/o_proj" + ] }, { "type": "gemm", "args": { - "m": 5, - "n": 4096, - "k": 512, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 614, + "n": 7168, + "k": 16384, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/o_proj", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] }, { "type": "gemm", "args": { - "m": 5, + "m": 614, "n": 7168, - "k": 2048, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "k": 16384, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] }, { "type": "gemm", "args": { - "m": 5, - "n": 2304, + "m": 614, + "n": 9216, "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/gate_up_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/gate_up_proj", + "nvidia/Kimi-K2.5-NVFP4/gate_up_proj" + ] }, { "type": "gemm", "args": { - "m": 5, - "n": 7168, - "k": 2304, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 614, + "n": 9216, + "k": 7168, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 32 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 1, + 32 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "amd/DeepSeek-R1-0528-MXFP4-Preview/gate_up_proj", + "amd/DeepSeek-R1-0528-MXFP4-v2/gate_up_proj", + "amd/DeepSeek-R1-0528-MXFP4/gate_up_proj", + "amd/Kimi-K2.5-MXFP4/gate_up_proj" + ] }, { "type": "gemm", "args": { - "m": 268, - "n": 1536, - "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 614, + "n": 24576, + "k": 1536, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/q_b_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/q_b_proj" + ] }, { "type": "gemm", "args": { - "m": 268, + "m": 614, "n": 24576, "k": 1536, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 32 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 1, + 32 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "amd/DeepSeek-R1-0528-MXFP4-Preview/q_b_proj", + "amd/DeepSeek-R1-0528-MXFP4-v2/q_b_proj", + "amd/DeepSeek-R1-0528-MXFP4/q_b_proj" + ] }, { "type": "gemm", "args": { - "m": 268, - "n": 576, - "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 614, + "n": 24576, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/q_b_proj" + ] }, { "type": "gemm", "args": { - "m": 268, + "m": 614, "n": 32768, "k": 512, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/kv_b_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/kv_b_proj" + ] }, { "type": "gemm", "args": { - "m": 268, - "n": 7168, - "k": 16384, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 614, + "n": 32768, + "k": 512, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 32 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 1, + 32 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "amd/DeepSeek-R1-0528-MXFP4-Preview/kv_b_proj", + "amd/DeepSeek-R1-0528-MXFP4-v2/kv_b_proj", + "amd/DeepSeek-R1-0528-MXFP4/kv_b_proj" + ] }, { "type": "gemm", "args": { - "m": 268, - "n": 1152, - "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 614, + "n": 32768, + "k": 512, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/kv_b_proj" + ] }, { "type": "gemm", "args": { - "m": 268, - "n": 7168, - "k": 1152, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 640, + "n": 576, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/kv_a_proj_with_mqa" + ] }, { "type": "gemm", "args": { - "m": 563, + "m": 640, "n": 1536, "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 563, - "n": 24576, - "k": 1536, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/gate_up_proj", + "deepseek-ai/DeepSeek-R1-0528/q_a_proj", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w3" + ] }, { "type": "gemm", "args": { - "m": 563, - "n": 576, + "m": 640, + "n": 2304, "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/gate_up_proj" + ] }, { "type": "gemm", "args": { - "m": 563, - "n": 32768, - "k": 512, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 640, + "n": 7168, + "k": 2304, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/down_proj" + ] }, { "type": "gemm", "args": { - "m": 563, + "m": 640, "n": 7168, "k": 16384, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/o_proj", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] }, { "type": "gemm", "args": { - "m": 563, - "n": 4608, - "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 640, + "n": 24576, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/q_b_proj" + ] }, { "type": "gemm", "args": { - "m": 563, - "n": 7168, - "k": 4608, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 640, + "n": 32768, + "k": 512, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/kv_b_proj" + ] }, { "type": "gemm", "args": { - "m": 70, - "n": 1536, + "m": 644, + "n": 576, "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/kv_a_proj_with_mqa" + ] }, { "type": "gemm", "args": { - "m": 70, - "n": 24576, - "k": 1536, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 644, + "n": 1536, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/gate_up_proj", + "deepseek-ai/DeepSeek-R1-0528/q_a_proj", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w3" + ] }, { "type": "gemm", "args": { - "m": 70, - "n": 576, + "m": 644, + "n": 1536, "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w3" + ] }, { "type": "gemm", "args": { - "m": 70, - "n": 32768, - "k": 512, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 644, + "n": 2304, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/gate_up_proj" + ] }, { "type": "gemm", "args": { - "m": 70, + "m": 644, "n": 7168, - "k": 16384, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "k": 2304, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/down_proj" + ] }, { "type": "gemm", "args": { - "m": 70, + "m": 644, "n": 7168, - "k": 576, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "k": 16384, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/o_proj", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] }, { "type": "gemm", "args": { - "m": 98304, + "m": 644, "n": 24576, "k": 1536, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/q_b_proj" + ] }, { "type": "gemm", "args": { - "m": 98304, + "m": 644, "n": 32768, "k": 512, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 98304, - "n": 7168, - "k": 16384, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 393216, - "n": 3072, - "k": 1536, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 393216, - "n": 4096, - "k": 512, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 393216, - "n": 7168, - "k": 2048, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/kv_b_proj" + ] }, { "type": "gemm", "args": { - "m": 393216, - "n": 2304, + "m": 654, + "n": 576, "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/Kimi-K2.5-MXFP4/kv_a_proj_with_mqa", + "moonshotai/Kimi-K2.5/kv_a_proj_with_mqa", + "nvidia/DeepSeek-R1-0528-FP4-V2/kv_a_proj_with_mqa", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/kv_a_proj_with_mqa", + "nvidia/Kimi-K2.5-NVFP4/kv_a_proj_with_mqa", + "nvidia/Kimi-K2.6-NVFP4/kv_a_proj_with_mqa" + ] }, { "type": "gemm", "args": { - "m": 393216, - "n": 7168, - "k": 2304, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 654, + "n": 576, + "k": 7168, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 32 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 1, + 32 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "amd/DeepSeek-R1-0528-MXFP4-Preview/kv_a_proj_with_mqa", + "amd/DeepSeek-R1-0528-MXFP4-v2/kv_a_proj_with_mqa", + "amd/DeepSeek-R1-0528-MXFP4/kv_a_proj_with_mqa" + ] }, { "type": "gemm", "args": { - "m": 368640, + "m": 654, "n": 1536, "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/Kimi-K2.5-MXFP4/q_a_proj", + "moonshotai/Kimi-K2.5/q_a_proj", + "nvidia/DeepSeek-R1-0528-FP4-V2/q_a_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/q_a_proj", + "nvidia/Kimi-K2.5-NVFP4/q_a_proj", + "nvidia/Kimi-K2.6-NVFP4/q_a_proj" + ] }, { "type": "gemm", "args": { - "m": 368640, - "n": 24576, - "k": 1536, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 654, + "n": 1536, + "k": 7168, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 32 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 1, + 32 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "amd/DeepSeek-R1-0528-MXFP4-Preview/q_a_proj", + "amd/DeepSeek-R1-0528-MXFP4-v2/q_a_proj", + "amd/DeepSeek-R1-0528-MXFP4/q_a_proj" + ] }, { "type": "gemm", "args": { - "m": 368640, - "n": 576, + "m": 654, + "n": 1536, "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w3" + ] }, { "type": "gemm", "args": { - "m": 368640, - "n": 32768, - "k": 512, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 654, + "n": 7168, + "k": 9216, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/down_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/down_proj", + "nvidia/Kimi-K2.5-NVFP4/down_proj" + ] }, { "type": "gemm", "args": { - "m": 368640, + "m": 654, "n": 7168, "k": 16384, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/o_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/o_proj" + ] }, { "type": "gemm", "args": { - "m": 368640, - "n": 4608, - "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 654, + "n": 7168, + "k": 16384, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 32 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 1, + 32 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "amd/DeepSeek-R1-0528-MXFP4-Preview/o_proj", + "amd/DeepSeek-R1-0528-MXFP4-v2/o_proj", + "amd/DeepSeek-R1-0528-MXFP4/o_proj" + ] }, { "type": "gemm", "args": { - "m": 368640, + "m": 654, "n": 7168, - "k": 4608, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "k": 16384, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] }, { "type": "gemm", "args": { - "m": 46080, - "n": 7168, - "k": 576, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 654, + "n": 9216, + "k": 7168, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/gate_up_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/gate_up_proj", + "nvidia/Kimi-K2.5-NVFP4/gate_up_proj" + ] }, { "type": "gemm", "args": { - "m": 2516992, - "n": 1536, + "m": 654, + "n": 9216, "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 32 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 1, + 32 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "amd/DeepSeek-R1-0528-MXFP4-Preview/gate_up_proj", + "amd/DeepSeek-R1-0528-MXFP4-v2/gate_up_proj", + "amd/DeepSeek-R1-0528-MXFP4/gate_up_proj", + "amd/Kimi-K2.5-MXFP4/gate_up_proj" + ] }, { "type": "gemm", "args": { - "m": 2516992, + "m": 654, "n": 24576, "k": 1536, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/q_b_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/q_b_proj" + ] }, { "type": "gemm", "args": { - "m": 2516992, - "n": 576, - "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 654, + "n": 24576, + "k": 1536, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 32 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 1, + 32 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "amd/DeepSeek-R1-0528-MXFP4-Preview/q_b_proj", + "amd/DeepSeek-R1-0528-MXFP4-v2/q_b_proj", + "amd/DeepSeek-R1-0528-MXFP4/q_b_proj" + ] }, { "type": "gemm", "args": { - "m": 2516992, + "m": 654, "n": 32768, "k": 512, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/kv_b_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/kv_b_proj" + ] }, { "type": "gemm", "args": { - "m": 2516992, - "n": 7168, - "k": 16384, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 654, + "n": 32768, + "k": 512, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 32 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 1, + 32 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "amd/DeepSeek-R1-0528-MXFP4-Preview/kv_b_proj", + "amd/DeepSeek-R1-0528-MXFP4-v2/kv_b_proj", + "amd/DeepSeek-R1-0528-MXFP4/kv_b_proj" + ] }, { "type": "gemm", "args": { - "m": 2516992, - "n": 4608, + "m": 666, + "n": 576, "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/Kimi-K2.5-MXFP4/kv_a_proj_with_mqa", + "moonshotai/Kimi-K2.5/kv_a_proj_with_mqa", + "nvidia/DeepSeek-R1-0528-FP4-V2/kv_a_proj_with_mqa", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/kv_a_proj_with_mqa", + "nvidia/Kimi-K2.5-NVFP4/kv_a_proj_with_mqa", + "nvidia/Kimi-K2.6-NVFP4/kv_a_proj_with_mqa" + ] }, { "type": "gemm", "args": { - "m": 2516992, - "n": 7168, - "k": 4608, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 666, + "n": 576, + "k": 7168, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 32 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 1, + 32 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "amd/DeepSeek-R1-0528-MXFP4-Preview/kv_a_proj_with_mqa", + "amd/DeepSeek-R1-0528-MXFP4-v2/kv_a_proj_with_mqa", + "amd/DeepSeek-R1-0528-MXFP4/kv_a_proj_with_mqa" + ] }, { "type": "gemm", "args": { - "m": 614, + "m": 666, "n": 1536, "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/Kimi-K2.5-MXFP4/q_a_proj", + "moonshotai/Kimi-K2.5/q_a_proj", + "nvidia/DeepSeek-R1-0528-FP4-V2/q_a_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/q_a_proj", + "nvidia/Kimi-K2.5-NVFP4/q_a_proj", + "nvidia/Kimi-K2.6-NVFP4/q_a_proj" + ] }, { "type": "gemm", "args": { - "m": 614, - "n": 24576, - "k": 1536, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 666, + "n": 1536, + "k": 7168, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 32 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 1, + 32 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "amd/DeepSeek-R1-0528-MXFP4-Preview/q_a_proj", + "amd/DeepSeek-R1-0528-MXFP4-v2/q_a_proj", + "amd/DeepSeek-R1-0528-MXFP4/q_a_proj" + ] }, { "type": "gemm", "args": { - "m": 614, - "n": 576, + "m": 666, + "n": 1536, "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w3" + ] }, { "type": "gemm", "args": { - "m": 614, - "n": 32768, - "k": 512, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 666, + "n": 7168, + "k": 9216, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/down_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/down_proj", + "nvidia/Kimi-K2.5-NVFP4/down_proj" + ] }, { "type": "gemm", "args": { - "m": 614, + "m": 666, "n": 7168, "k": 16384, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/o_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/o_proj" + ] }, { "type": "gemm", "args": { - "m": 614, - "n": 4608, - "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 666, + "n": 7168, + "k": 16384, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 32 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 1, + 32 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "amd/DeepSeek-R1-0528-MXFP4-Preview/o_proj", + "amd/DeepSeek-R1-0528-MXFP4-v2/o_proj", + "amd/DeepSeek-R1-0528-MXFP4/o_proj" + ] }, { "type": "gemm", "args": { - "m": 614, + "m": 666, "n": 7168, - "k": 4608, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "k": 16384, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] }, { "type": "gemm", "args": { - "m": 629248, - "n": 1536, + "m": 666, + "n": 9216, "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/gate_up_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/gate_up_proj", + "nvidia/Kimi-K2.5-NVFP4/gate_up_proj" + ] }, { "type": "gemm", "args": { - "m": 629248, + "m": 666, + "n": 9216, + "k": 7168, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 32 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 1, + 32 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "amd/DeepSeek-R1-0528-MXFP4-Preview/gate_up_proj", + "amd/DeepSeek-R1-0528-MXFP4-v2/gate_up_proj", + "amd/DeepSeek-R1-0528-MXFP4/gate_up_proj", + "amd/Kimi-K2.5-MXFP4/gate_up_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 666, "n": 24576, "k": 1536, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/q_b_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/q_b_proj" + ] }, { "type": "gemm", "args": { - "m": 629248, - "n": 576, - "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 666, + "n": 24576, + "k": 1536, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 32 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 1, + 32 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "amd/DeepSeek-R1-0528-MXFP4-Preview/q_b_proj", + "amd/DeepSeek-R1-0528-MXFP4-v2/q_b_proj", + "amd/DeepSeek-R1-0528-MXFP4/q_b_proj" + ] }, { "type": "gemm", "args": { - "m": 629248, + "m": 666, "n": 32768, "k": 512, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/kv_b_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/kv_b_proj" + ] }, { "type": "gemm", "args": { - "m": 629248, - "n": 7168, - "k": 16384, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 666, + "n": 32768, + "k": 512, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 32 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 1, + 32 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "amd/DeepSeek-R1-0528-MXFP4-Preview/kv_b_proj", + "amd/DeepSeek-R1-0528-MXFP4-v2/kv_b_proj", + "amd/DeepSeek-R1-0528-MXFP4/kv_b_proj" + ] }, { "type": "gemm", "args": { - "m": 629248, - "n": 1152, + "m": 668, + "n": 576, "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/Kimi-K2.5-MXFP4/kv_a_proj_with_mqa", + "moonshotai/Kimi-K2.5/kv_a_proj_with_mqa", + "nvidia/DeepSeek-R1-0528-FP4-V2/kv_a_proj_with_mqa", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/kv_a_proj_with_mqa", + "nvidia/Kimi-K2.5-NVFP4/kv_a_proj_with_mqa", + "nvidia/Kimi-K2.6-NVFP4/kv_a_proj_with_mqa" + ] }, { "type": "gemm", "args": { - "m": 629248, - "n": 7168, - "k": 1152, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 668, + "n": 1536, + "k": 1536, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/Kimi-K2.5-MXFP4/q_b_proj", + "moonshotai/Kimi-K2.5/q_b_proj", + "nvidia/DeepSeek-R1-0528-FP4-V2/q_b_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/q_b_proj", + "nvidia/Kimi-K2.5-NVFP4/q_b_proj", + "nvidia/Kimi-K2.6-NVFP4/q_b_proj" + ] }, { "type": "gemm", "args": { - "m": 153, + "m": 668, "n": 1536, "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/Kimi-K2.5-MXFP4/q_a_proj", + "moonshotai/Kimi-K2.5/q_a_proj", + "nvidia/DeepSeek-R1-0528-FP4-V2/q_a_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/q_a_proj", + "nvidia/Kimi-K2.5-NVFP4/q_a_proj", + "nvidia/Kimi-K2.6-NVFP4/q_a_proj" + ] }, { "type": "gemm", "args": { - "m": 153, - "n": 24576, - "k": 1536, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 668, + "n": 2048, + "k": 512, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/Kimi-K2.5-MXFP4/kv_b_proj", + "moonshotai/Kimi-K2.5/kv_b_proj", + "nvidia/DeepSeek-R1-0528-FP4-V2/kv_b_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/kv_b_proj", + "nvidia/Kimi-K2.5-NVFP4/kv_b_proj", + "nvidia/Kimi-K2.6-NVFP4/kv_b_proj" + ] }, { "type": "gemm", "args": { - "m": 153, - "n": 576, + "m": 668, + "n": 2304, "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/gate_up_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/gate_up_proj", + "nvidia/Kimi-K2.5-NVFP4/gate_up_proj" + ] }, { "type": "gemm", "args": { - "m": 153, - "n": 32768, - "k": 512, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 668, + "n": 7168, + "k": 1024, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/Kimi-K2.5-MXFP4/o_proj", + "moonshotai/Kimi-K2.5/o_proj", + "nvidia/Kimi-K2.5-NVFP4/o_proj", + "nvidia/Kimi-K2.6-NVFP4/o_proj" + ] }, { "type": "gemm", "args": { - "m": 153, + "m": 668, "n": 7168, - "k": 16384, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "k": 2304, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/down_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/down_proj", + "nvidia/Kimi-K2.5-NVFP4/down_proj" + ] }, { "type": "gemm", "args": { - "m": 153, - "n": 1152, - "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 768, + "n": 512, + "k": 2880, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/gpt-oss-120b-w-mxfp4-a-fp8/k_proj", + "amd/gpt-oss-120b-w-mxfp4-a-fp8/q_proj", + "amd/gpt-oss-120b-w-mxfp4-a-fp8/v_proj", + "openai/gpt-oss-120b/k_proj", + "openai/gpt-oss-120b/q_proj", + "openai/gpt-oss-120b/v_proj" + ] }, { "type": "gemm", "args": { - "m": 153, - "n": 7168, - "k": 1152, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 768, + "n": 576, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/kv_a_proj_with_mqa" + ] }, { "type": "gemm", "args": { - "m": 1363968, - "n": 1536, + "m": 768, + "n": 1088, "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] }, { "type": "gemm", "args": { - "m": 1363968, - "n": 24576, - "k": 1536, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 768, + "n": 1536, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/gate_up_proj", + "deepseek-ai/DeepSeek-R1-0528/q_a_proj", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w3" + ] }, { "type": "gemm", "args": { - "m": 1363968, - "n": 576, + "m": 768, + "n": 2304, "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/gate_up_proj" + ] }, { "type": "gemm", "args": { - "m": 1363968, - "n": 32768, - "k": 512, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 768, + "n": 2880, + "k": 4096, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/gpt-oss-120b-w-mxfp4-a-fp8/o_proj", + "openai/gpt-oss-120b/o_proj" + ] }, { "type": "gemm", "args": { - "m": 1363968, - "n": 7168, - "k": 16384, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 768, + "n": 4096, + "k": 2880, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/gpt-oss-120b-w-mxfp4-a-fp8/q_proj", + "openai/gpt-oss-120b/q_proj" + ] }, { "type": "gemm", "args": { - "m": 1363968, + "m": 768, "n": 4608, "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/gate_up_proj" + ] }, { "type": "gemm", "args": { - "m": 1363968, + "m": 768, "n": 7168, - "k": 4608, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "k": 2304, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/down_proj" + ] }, { "type": "gemm", "args": { - "m": 333, - "n": 1536, - "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 768, + "n": 7168, + "k": 4608, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/down_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/down_proj", + "nvidia/Kimi-K2.5-NVFP4/down_proj" + ] }, { "type": "gemm", "args": { - "m": 333, - "n": 24576, - "k": 1536, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 768, + "n": 7168, + "k": 4608, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 32 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 1, + 32 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "amd/DeepSeek-R1-0528-MXFP4-Preview/down_proj", + "amd/DeepSeek-R1-0528-MXFP4-v2/down_proj", + "amd/DeepSeek-R1-0528-MXFP4/down_proj", + "amd/Kimi-K2.5-MXFP4/down_proj" + ] }, { "type": "gemm", "args": { - "m": 333, - "n": 576, - "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 768, + "n": 7168, + "k": 4608, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/down_proj" + ] }, { "type": "gemm", "args": { - "m": 333, - "n": 32768, - "k": 512, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 768, + "n": 7168, + "k": 8192, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/Kimi-K2.5-MXFP4/o_proj", + "moonshotai/Kimi-K2.5/o_proj", + "nvidia/Kimi-K2.5-NVFP4/o_proj", + "nvidia/Kimi-K2.6-NVFP4/o_proj" + ] }, { "type": "gemm", "args": { - "m": 333, + "m": 768, "n": 7168, - "k": 16384, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "k": 8192, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] }, { "type": "gemm", "args": { - "m": 333, - "n": 4608, - "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 768, + "n": 7168, + "k": 16384, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/o_proj", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] }, { "type": "gemm", "args": { - "m": 333, + "m": 768, "n": 7168, - "k": 4608, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "k": 65536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] }, { "type": "gemm", "args": { - "m": 170496, - "n": 7168, - "k": 576, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 768, + "n": 12288, + "k": 1536, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/Kimi-K2.5-MXFP4/q_b_proj", + "moonshotai/Kimi-K2.5/q_b_proj", + "nvidia/DeepSeek-R1-0528-FP4-V2/q_b_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/q_b_proj", + "nvidia/Kimi-K2.5-NVFP4/q_b_proj", + "nvidia/Kimi-K2.6-NVFP4/q_b_proj" + ] }, { "type": "gemm", "args": { - "m": 41, - "n": 1536, - "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 768, + "n": 16384, + "k": 512, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/Kimi-K2.5-MXFP4/kv_b_proj", + "moonshotai/Kimi-K2.5/kv_b_proj", + "nvidia/DeepSeek-R1-0528-FP4-V2/kv_b_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/kv_b_proj", + "nvidia/Kimi-K2.5-NVFP4/kv_b_proj", + "nvidia/Kimi-K2.6-NVFP4/kv_b_proj" + ] }, { "type": "gemm", "args": { - "m": 41, + "m": 768, "n": 24576, "k": 1536, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 41, - "n": 576, - "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/q_b_proj" + ] }, { "type": "gemm", "args": { - "m": 41, + "m": 768, "n": 32768, "k": 512, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/kv_b_proj" + ] }, { "type": "gemm", "args": { - "m": 41, - "n": 7168, - "k": 16384, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 768, + "n": 65536, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_b" + ] }, { "type": "gemm", "args": { - "m": 41, - "n": 7168, - "k": 576, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 768, + "n": 122880, + "k": 1024, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] }, { "type": "gemm", "args": { - "m": 8808448, - "n": 1536, + "m": 806, + "n": 576, "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 8808448, - "n": 24576, - "k": 1536, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/Kimi-K2.5-MXFP4/kv_a_proj_with_mqa", + "moonshotai/Kimi-K2.5/kv_a_proj_with_mqa", + "nvidia/DeepSeek-R1-0528-FP4-V2/kv_a_proj_with_mqa", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/kv_a_proj_with_mqa", + "nvidia/Kimi-K2.5-NVFP4/kv_a_proj_with_mqa", + "nvidia/Kimi-K2.6-NVFP4/kv_a_proj_with_mqa" + ] }, { "type": "gemm", "args": { - "m": 8808448, - "n": 576, + "m": 806, + "n": 1536, "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/Kimi-K2.5-MXFP4/q_a_proj", + "moonshotai/Kimi-K2.5/q_a_proj", + "nvidia/DeepSeek-R1-0528-FP4-V2/q_a_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/q_a_proj", + "nvidia/Kimi-K2.5-NVFP4/q_a_proj", + "nvidia/Kimi-K2.6-NVFP4/q_a_proj" + ] }, { "type": "gemm", "args": { - "m": 8808448, - "n": 32768, - "k": 512, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 806, + "n": 2304, + "k": 7168, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/gate_up_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/gate_up_proj", + "nvidia/Kimi-K2.5-NVFP4/gate_up_proj" + ] }, { "type": "gemm", "args": { - "m": 8808448, + "m": 806, "n": 7168, - "k": 16384, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 8808448, - "n": 4608, - "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "k": 2304, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/down_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/down_proj", + "nvidia/Kimi-K2.5-NVFP4/down_proj" + ] }, { "type": "gemm", "args": { - "m": 8808448, + "m": 806, "n": 7168, - "k": 4608, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "k": 8192, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/Kimi-K2.5-MXFP4/o_proj", + "moonshotai/Kimi-K2.5/o_proj", + "nvidia/Kimi-K2.5-NVFP4/o_proj", + "nvidia/Kimi-K2.6-NVFP4/o_proj" + ] }, { "type": "gemm", "args": { - "m": 2202112, - "n": 1536, - "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 806, + "n": 12288, + "k": 1536, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/Kimi-K2.5-MXFP4/q_b_proj", + "moonshotai/Kimi-K2.5/q_b_proj", + "nvidia/DeepSeek-R1-0528-FP4-V2/q_b_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/q_b_proj", + "nvidia/Kimi-K2.5-NVFP4/q_b_proj", + "nvidia/Kimi-K2.6-NVFP4/q_b_proj" + ] }, { "type": "gemm", "args": { - "m": 2202112, - "n": 24576, - "k": 1536, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 806, + "n": 16384, + "k": 512, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/Kimi-K2.5-MXFP4/kv_b_proj", + "moonshotai/Kimi-K2.5/kv_b_proj", + "nvidia/DeepSeek-R1-0528-FP4-V2/kv_b_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/kv_b_proj", + "nvidia/Kimi-K2.5-NVFP4/kv_b_proj", + "nvidia/Kimi-K2.6-NVFP4/kv_b_proj" + ] }, { "type": "gemm", "args": { - "m": 2202112, + "m": 807, "n": 576, "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/Kimi-K2.5-MXFP4/kv_a_proj_with_mqa", + "moonshotai/Kimi-K2.5/kv_a_proj_with_mqa", + "nvidia/DeepSeek-R1-0528-FP4-V2/kv_a_proj_with_mqa", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/kv_a_proj_with_mqa", + "nvidia/Kimi-K2.5-NVFP4/kv_a_proj_with_mqa", + "nvidia/Kimi-K2.6-NVFP4/kv_a_proj_with_mqa" + ] }, { "type": "gemm", "args": { - "m": 2202112, - "n": 32768, - "k": 512, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 807, + "n": 1536, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/Kimi-K2.5-MXFP4/q_a_proj", + "moonshotai/Kimi-K2.5/q_a_proj", + "nvidia/DeepSeek-R1-0528-FP4-V2/q_a_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/q_a_proj", + "nvidia/Kimi-K2.5-NVFP4/q_a_proj", + "nvidia/Kimi-K2.6-NVFP4/q_a_proj" + ] }, { "type": "gemm", "args": { - "m": 2202112, - "n": 7168, - "k": 16384, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 807, + "n": 4608, + "k": 7168, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/gate_up_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/gate_up_proj", + "nvidia/Kimi-K2.5-NVFP4/gate_up_proj" + ] }, { "type": "gemm", "args": { - "m": 2202112, - "n": 1152, + "m": 807, + "n": 4608, "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 32 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 1, + 32 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "amd/DeepSeek-R1-0528-MXFP4-Preview/gate_up_proj", + "amd/DeepSeek-R1-0528-MXFP4-v2/gate_up_proj", + "amd/DeepSeek-R1-0528-MXFP4/gate_up_proj", + "amd/Kimi-K2.5-MXFP4/gate_up_proj" + ] }, { "type": "gemm", "args": { - "m": 2202112, + "m": 807, "n": 7168, - "k": 1152, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "k": 4608, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/down_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/down_proj", + "nvidia/Kimi-K2.5-NVFP4/down_proj" + ] }, { "type": "gemm", "args": { - "m": 90112, - "n": 1536, - "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 807, + "n": 7168, + "k": 16384, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/o_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/o_proj" + ] }, { "type": "gemm", "args": { - "m": 90112, + "m": 807, "n": 24576, "k": 1536, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/q_b_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/q_b_proj" + ] }, { "type": "gemm", "args": { - "m": 90112, - "n": 576, - "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 807, + "n": 32768, + "k": 512, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/kv_b_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/kv_b_proj" + ] }, { "type": "gemm", "args": { - "m": 90112, - "n": 32768, - "k": 512, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 845, + "n": 576, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/Kimi-K2.5-MXFP4/kv_a_proj_with_mqa", + "moonshotai/Kimi-K2.5/kv_a_proj_with_mqa", + "nvidia/DeepSeek-R1-0528-FP4-V2/kv_a_proj_with_mqa", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/kv_a_proj_with_mqa", + "nvidia/Kimi-K2.5-NVFP4/kv_a_proj_with_mqa", + "nvidia/Kimi-K2.6-NVFP4/kv_a_proj_with_mqa" + ] }, { "type": "gemm", "args": { - "m": 90112, - "n": 7168, - "k": 16384, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 845, + "n": 576, + "k": 7168, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 32 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 1, + 32 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "amd/DeepSeek-R1-0528-MXFP4-Preview/kv_a_proj_with_mqa", + "amd/DeepSeek-R1-0528-MXFP4-v2/kv_a_proj_with_mqa", + "amd/DeepSeek-R1-0528-MXFP4/kv_a_proj_with_mqa" + ] }, { "type": "gemm", "args": { - "m": 90112, - "n": 4608, + "m": 845, + "n": 1536, "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/Kimi-K2.5-MXFP4/q_a_proj", + "moonshotai/Kimi-K2.5/q_a_proj", + "nvidia/DeepSeek-R1-0528-FP4-V2/q_a_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/q_a_proj", + "nvidia/Kimi-K2.5-NVFP4/q_a_proj", + "nvidia/Kimi-K2.6-NVFP4/q_a_proj" + ] }, { "type": "gemm", "args": { - "m": 90112, - "n": 7168, - "k": 4608, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 845, + "n": 1536, + "k": 7168, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 32 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 1, + 32 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "amd/DeepSeek-R1-0528-MXFP4-Preview/q_a_proj", + "amd/DeepSeek-R1-0528-MXFP4-v2/q_a_proj", + "amd/DeepSeek-R1-0528-MXFP4/q_a_proj" + ] }, { "type": "gemm", "args": { - "m": 155648, + "m": 845, "n": 1536, "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w3" + ] }, { "type": "gemm", "args": { - "m": 155648, - "n": 24576, - "k": 1536, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 845, + "n": 4608, + "k": 7168, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/gate_up_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/gate_up_proj", + "nvidia/Kimi-K2.5-NVFP4/gate_up_proj" + ] }, { "type": "gemm", "args": { - "m": 155648, - "n": 576, + "m": 845, + "n": 4608, "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 32 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 1, + 32 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "amd/DeepSeek-R1-0528-MXFP4-Preview/gate_up_proj", + "amd/DeepSeek-R1-0528-MXFP4-v2/gate_up_proj", + "amd/DeepSeek-R1-0528-MXFP4/gate_up_proj", + "amd/Kimi-K2.5-MXFP4/gate_up_proj" + ] }, { "type": "gemm", "args": { - "m": 155648, - "n": 32768, - "k": 512, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 845, + "n": 7168, + "k": 4608, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/down_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/down_proj", + "nvidia/Kimi-K2.5-NVFP4/down_proj" + ] }, { "type": "gemm", "args": { - "m": 155648, + "m": 845, "n": 7168, - "k": 16384, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "k": 4608, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 32 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 1, + 32 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "amd/DeepSeek-R1-0528-MXFP4-Preview/down_proj", + "amd/DeepSeek-R1-0528-MXFP4-v2/down_proj", + "amd/DeepSeek-R1-0528-MXFP4/down_proj", + "amd/Kimi-K2.5-MXFP4/down_proj" + ] }, { "type": "gemm", "args": { - "m": 155648, - "n": 4608, - "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 845, + "n": 7168, + "k": 16384, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/o_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/o_proj" + ] }, { "type": "gemm", "args": { - "m": 155648, + "m": 845, "n": 7168, - "k": 4608, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "k": 16384, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 32 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 1, + 32 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "amd/DeepSeek-R1-0528-MXFP4-Preview/o_proj", + "amd/DeepSeek-R1-0528-MXFP4-v2/o_proj", + "amd/DeepSeek-R1-0528-MXFP4/o_proj" + ] }, { "type": "gemm", "args": { - "m": 19, - "n": 1536, - "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 845, + "n": 7168, + "k": 16384, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] }, { "type": "gemm", "args": { - "m": 19, + "m": 845, "n": 24576, "k": 1536, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/q_b_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/q_b_proj" + ] }, { "type": "gemm", "args": { - "m": 19, - "n": 576, - "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 845, + "n": 24576, + "k": 1536, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 32 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 1, + 32 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "amd/DeepSeek-R1-0528-MXFP4-Preview/q_b_proj", + "amd/DeepSeek-R1-0528-MXFP4-v2/q_b_proj", + "amd/DeepSeek-R1-0528-MXFP4/q_b_proj" + ] }, { "type": "gemm", "args": { - "m": 19, + "m": 845, "n": 32768, "k": 512, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/kv_b_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/kv_b_proj" + ] }, { "type": "gemm", "args": { - "m": 19, - "n": 7168, - "k": 16384, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 845, + "n": 32768, + "k": 512, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 32 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 1, + 32 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "amd/DeepSeek-R1-0528-MXFP4-Preview/kv_b_proj", + "amd/DeepSeek-R1-0528-MXFP4-v2/kv_b_proj", + "amd/DeepSeek-R1-0528-MXFP4/kv_b_proj" + ] }, { "type": "gemm", "args": { - "m": 19, - "n": 4608, + "m": 896, + "n": 576, "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/kv_a_proj_with_mqa" + ] }, { "type": "gemm", "args": { - "m": 19, - "n": 7168, - "k": 4608, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 896, + "n": 1536, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/gate_up_proj", + "deepseek-ai/DeepSeek-R1-0528/q_a_proj", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w3" + ] }, { "type": "gemm", "args": { - "m": 360448, + "m": 896, "n": 1536, "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w3" + ] }, { "type": "gemm", "args": { - "m": 360448, - "n": 3072, - "k": 1536, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 896, + "n": 2304, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/gate_up_proj" + ] }, { "type": "gemm", "args": { - "m": 360448, - "n": 576, - "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 896, + "n": 7168, + "k": 2304, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/down_proj" + ] }, { "type": "gemm", "args": { - "m": 360448, - "n": 4096, - "k": 512, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 896, + "n": 7168, + "k": 16384, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/o_proj", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] }, { "type": "gemm", "args": { - "m": 360448, - "n": 7168, - "k": 2048, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 896, + "n": 24576, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/q_b_proj" + ] }, { "type": "gemm", "args": { - "m": 360448, - "n": 2304, - "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 896, + "n": 32768, + "k": 512, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/kv_b_proj" + ] }, { "type": "gemm", "args": { - "m": 360448, - "n": 7168, - "k": 2304, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 924, + "n": 576, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/kv_a_proj_with_mqa" + ] }, { "type": "gemm", "args": { - "m": 44, - "n": 1536, + "m": 924, + "n": 1152, "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/gate_up_proj" + ] }, { "type": "gemm", "args": { - "m": 44, - "n": 3072, + "m": 924, + "n": 1536, "k": 1536, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/q_b_proj" + ] }, { "type": "gemm", "args": { - "m": 44, - "n": 576, + "m": 924, + "n": 1536, "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/gate_up_proj", + "deepseek-ai/DeepSeek-R1-0528/q_a_proj", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w3" + ] }, { "type": "gemm", "args": { - "m": 44, - "n": 4096, + "m": 924, + "n": 2048, "k": 512, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/kv_b_proj" + ] }, { "type": "gemm", "args": { - "m": 44, + "m": 924, "n": 7168, - "k": 2048, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "k": 1024, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/o_proj", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] }, { "type": "gemm", "args": { - "m": 44, - "n": 2304, - "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 924, + "n": 7168, + "k": 1152, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/down_proj" + ] }, { "type": "gemm", "args": { - "m": 44, - "n": 7168, - "k": 2304, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 960, + "n": 576, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/kv_a_proj_with_mqa" + ] }, { "type": "gemm", "args": { - "m": 622592, + "m": 960, "n": 1536, "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/gate_up_proj", + "deepseek-ai/DeepSeek-R1-0528/q_a_proj", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w3" + ] }, { "type": "gemm", "args": { - "m": 622592, - "n": 3072, - "k": 1536, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 960, + "n": 1536, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w3" + ] }, { "type": "gemm", "args": { - "m": 622592, - "n": 576, + "m": 960, + "n": 2304, "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/gate_up_proj" + ] }, { "type": "gemm", "args": { - "m": 622592, - "n": 4096, - "k": 512, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 960, + "n": 7168, + "k": 2304, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/down_proj" + ] }, { "type": "gemm", "args": { - "m": 622592, + "m": 960, "n": 7168, - "k": 2048, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "k": 16384, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/o_proj", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] }, { "type": "gemm", "args": { - "m": 622592, - "n": 2304, - "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 960, + "n": 24576, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/q_b_proj" + ] }, { "type": "gemm", "args": { - "m": 622592, - "n": 7168, - "k": 2304, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 960, + "n": 32768, + "k": 512, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/kv_b_proj" + ] }, { "type": "gemm", "args": { - "m": 76, - "n": 1536, - "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 1024, + "n": 64, + "k": 2880, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/gpt-oss-120b-w-mxfp4-a-fp8/k_proj", + "amd/gpt-oss-120b-w-mxfp4-a-fp8/v_proj", + "openai/gpt-oss-120b/k_proj", + "openai/gpt-oss-120b/v_proj" + ] }, { "type": "gemm", "args": { - "m": 76, - "n": 3072, - "k": 1536, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 1024, + "n": 128, + "k": 2880, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/gpt-oss-120b-w-mxfp4-a-fp8/k_proj", + "amd/gpt-oss-120b-w-mxfp4-a-fp8/v_proj", + "openai/gpt-oss-120b/k_proj", + "openai/gpt-oss-120b/v_proj" + ] }, { "type": "gemm", "args": { - "m": 76, - "n": 576, - "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 1024, + "n": 512, + "k": 2880, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/gpt-oss-120b-w-mxfp4-a-fp8/k_proj", + "amd/gpt-oss-120b-w-mxfp4-a-fp8/q_proj", + "amd/gpt-oss-120b-w-mxfp4-a-fp8/v_proj", + "openai/gpt-oss-120b/k_proj", + "openai/gpt-oss-120b/q_proj", + "openai/gpt-oss-120b/v_proj" + ] }, { "type": "gemm", "args": { - "m": 76, - "n": 4096, - "k": 512, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 1024, + "n": 512, + "k": 3072, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/MiniMax-M2.5-MXFP4/k_proj", + "amd/MiniMax-M2.5-MXFP4/v_proj", + "nvidia/MiniMax-M2.5-NVFP4/k_proj", + "nvidia/MiniMax-M2.5-NVFP4/v_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 1024, + "n": 512, + "k": 3072, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "MiniMaxAI/MiniMax-M2.5/k_proj", + "MiniMaxAI/MiniMax-M2.5/v_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 1024, + "n": 576, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/Kimi-K2.5-MXFP4/kv_a_proj_with_mqa", + "moonshotai/Kimi-K2.5/kv_a_proj_with_mqa", + "nvidia/DeepSeek-R1-0528-FP4-V2/kv_a_proj_with_mqa", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/kv_a_proj_with_mqa", + "nvidia/Kimi-K2.5-NVFP4/kv_a_proj_with_mqa", + "nvidia/Kimi-K2.6-NVFP4/kv_a_proj_with_mqa" + ] }, { "type": "gemm", "args": { - "m": 76, - "n": 7168, - "k": 2048, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 1024, + "n": 576, + "k": 7168, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 32 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 1, + 32 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "amd/DeepSeek-R1-0528-MXFP4-Preview/kv_a_proj_with_mqa", + "amd/DeepSeek-R1-0528-MXFP4-v2/kv_a_proj_with_mqa", + "amd/DeepSeek-R1-0528-MXFP4/kv_a_proj_with_mqa" + ] }, { "type": "gemm", "args": { - "m": 76, - "n": 2304, + "m": 1024, + "n": 576, "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/kv_a_proj_with_mqa" + ] }, { "type": "gemm", "args": { - "m": 76, - "n": 7168, - "k": 2304, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 1024, + "n": 768, + "k": 3072, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/MiniMax-M2.5-MXFP4/q_proj", + "nvidia/MiniMax-M2.5-NVFP4/q_proj" + ] }, { "type": "gemm", "args": { - "m": 40960, - "n": 1536, - "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 1024, + "n": 768, + "k": 3072, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/MiniMax-M2.5-NVFP4/experts.gate_up_proj" + ] }, { "type": "gemm", "args": { - "m": 40960, - "n": 3072, - "k": 1536, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 1024, + "n": 768, + "k": 3072, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 32 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 1, + 32 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "amd/MiniMax-M2.5-MXFP4/experts.gate_up_proj" + ] }, { "type": "gemm", "args": { - "m": 40960, - "n": 576, - "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 1024, + "n": 768, + "k": 3072, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "MiniMaxAI/MiniMax-M2.5/experts.gate_up_proj", + "MiniMaxAI/MiniMax-M2.5/q_proj" + ] }, { "type": "gemm", "args": { - "m": 40960, - "n": 4096, - "k": 512, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 1024, + "n": 1024, + "k": 2880, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/gpt-oss-120b-w-mxfp4-a-fp8/q_proj", + "openai/gpt-oss-120b/q_proj" + ] }, { "type": "gemm", "args": { - "m": 40960, - "n": 7168, - "k": 2048, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 1024, + "n": 1024, + "k": 3072, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/MiniMax-M2.5-MXFP4/k_proj", + "amd/MiniMax-M2.5-MXFP4/qkv_proj", + "amd/MiniMax-M2.5-MXFP4/v_proj", + "nvidia/MiniMax-M2.5-NVFP4/k_proj", + "nvidia/MiniMax-M2.5-NVFP4/qkv_proj", + "nvidia/MiniMax-M2.5-NVFP4/v_proj" + ] }, { "type": "gemm", "args": { - "m": 40960, - "n": 2304, + "m": 1024, + "n": 1088, "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] }, { "type": "gemm", "args": { - "m": 40960, - "n": 7168, - "k": 2304, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 1024, + "n": 1152, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/gate_up_proj" + ] }, { "type": "gemm", "args": { - "m": 681984, + "m": 1024, "n": 1536, - "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/q_b_proj" + ] }, { "type": "gemm", "args": { - "m": 681984, - "n": 24576, - "k": 1536, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 1024, + "n": 1536, + "k": 3072, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/MiniMax-M2.5-NVFP4/experts.gate_up_proj" + ] }, { "type": "gemm", "args": { - "m": 681984, - "n": 576, - "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 1024, + "n": 1536, + "k": 3072, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 32 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 1, + 32 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "amd/MiniMax-M2.5-MXFP4/experts.gate_up_proj" + ] }, { "type": "gemm", "args": { - "m": 681984, - "n": 32768, - "k": 512, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 1024, + "n": 1536, + "k": 3072, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "MiniMaxAI/MiniMax-M2.5/experts.gate_up_proj", + "MiniMaxAI/MiniMax-M2.5/q_proj" + ] }, { "type": "gemm", "args": { - "m": 681984, - "n": 7168, - "k": 16384, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 1024, + "n": 1536, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/Kimi-K2.5-MXFP4/q_a_proj", + "moonshotai/Kimi-K2.5/q_a_proj", + "nvidia/DeepSeek-R1-0528-FP4-V2/q_a_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/q_a_proj", + "nvidia/Kimi-K2.5-NVFP4/q_a_proj", + "nvidia/Kimi-K2.6-NVFP4/q_a_proj" + ] }, { "type": "gemm", "args": { - "m": 681984, - "n": 4608, + "m": 1024, + "n": 1536, "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 32 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 1, + 32 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "amd/DeepSeek-R1-0528-MXFP4-Preview/q_a_proj", + "amd/DeepSeek-R1-0528-MXFP4-v2/q_a_proj", + "amd/DeepSeek-R1-0528-MXFP4/q_a_proj" + ] }, { "type": "gemm", "args": { - "m": 681984, - "n": 7168, - "k": 4608, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 1024, + "n": 1536, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/gate_up_proj", + "deepseek-ai/DeepSeek-R1-0528/q_a_proj", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w3" + ] }, { "type": "gemm", "args": { - "m": 83, + "m": 1024, "n": 1536, "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w3" + ] }, { "type": "gemm", "args": { - "m": 83, - "n": 24576, - "k": 1536, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 1024, + "n": 2048, + "k": 512, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/kv_b_proj" + ] }, { "type": "gemm", "args": { - "m": 83, - "n": 576, + "m": 1024, + "n": 2304, "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/gate_up_proj" + ] }, { "type": "gemm", "args": { - "m": 83, - "n": 32768, + "m": 1024, + "n": 2880, "k": 512, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/gpt-oss-120b-w-mxfp4-a-fp8/o_proj", + "openai/gpt-oss-120b/o_proj" + ] }, { "type": "gemm", "args": { - "m": 83, - "n": 7168, - "k": 16384, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 1024, + "n": 2880, + "k": 1024, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/gpt-oss-120b-w-mxfp4-a-fp8/o_proj", + "openai/gpt-oss-120b/o_proj" + ] }, { "type": "gemm", "args": { - "m": 83, - "n": 4608, - "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 1024, + "n": 2880, + "k": 4096, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/gpt-oss-120b-w-mxfp4-a-fp8/o_proj", + "openai/gpt-oss-120b/o_proj" + ] }, { "type": "gemm", "args": { - "m": 83, - "n": 7168, - "k": 4608, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 1024, + "n": 3072, + "k": 768, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/MiniMax-M2.5-NVFP4/experts.down_proj" + ] }, { "type": "gemm", "args": { - "m": 85248, - "n": 1536, - "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 1024, + "n": 3072, + "k": 768, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 32 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 1, + 32 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "amd/MiniMax-M2.5-MXFP4/experts.down_proj" + ] }, { "type": "gemm", "args": { - "m": 85248, - "n": 24576, + "m": 1024, + "n": 3072, + "k": 768, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "MiniMaxAI/MiniMax-M2.5/experts.down_proj", + "MiniMaxAI/MiniMax-M2.5/o_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 1024, + "n": 3072, "k": 1536, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 32 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 1, + 32 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "amd/DeepSeek-R1-0528-MXFP4-Preview/q_b_proj", + "amd/DeepSeek-R1-0528-MXFP4-v2/q_b_proj", + "amd/DeepSeek-R1-0528-MXFP4/q_b_proj", + "amd/MiniMax-M2.5-MXFP4/experts.down_proj" + ] }, { "type": "gemm", "args": { - "m": 85248, - "n": 576, - "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 1024, + "n": 3072, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "MiniMaxAI/MiniMax-M2.5/experts.down_proj", + "MiniMaxAI/MiniMax-M2.5/o_proj", + "deepseek-ai/DeepSeek-R1-0528/q_b_proj" + ] }, { "type": "gemm", "args": { - "m": 85248, - "n": 32768, - "k": 512, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 1024, + "n": 3072, + "k": 3072, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/MiniMax-M2.5-MXFP4/o_proj", + "amd/MiniMax-M2.5-MXFP4/q_proj", + "nvidia/MiniMax-M2.5-NVFP4/o_proj", + "nvidia/MiniMax-M2.5-NVFP4/q_proj" + ] }, { "type": "gemm", "args": { - "m": 85248, - "n": 7168, - "k": 16384, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 1024, + "n": 3072, + "k": 3072, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 32 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 1, + 32 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "amd/MiniMax-M2.5-MXFP4/experts.gate_up_proj" + ] }, { "type": "gemm", "args": { - "m": 85248, - "n": 7168, - "k": 576, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 1024, + "n": 3072, + "k": 3072, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "MiniMaxAI/MiniMax-M2.5/experts.gate_up_proj", + "MiniMaxAI/MiniMax-M2.5/o_proj", + "MiniMaxAI/MiniMax-M2.5/q_proj" + ] }, { "type": "gemm", "args": { - "m": 10, - "n": 7168, - "k": 576, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 1024, + "n": 3072, + "k": 6144, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/GLM-5.1-MXFP4/gate_up_proj", + "amd/GLM-5.2-MXFP4/gate_up_proj", + "amd/MiniMax-M2.5-MXFP4/o_proj", + "amd/MiniMax-M3-MXFP4/gate_up_proj", + "nvidia/GLM-5.1-NVFP4/gate_up_proj", + "nvidia/GLM-5.2-NVFP4/gate_up_proj", + "nvidia/MiniMax-M2.5-NVFP4/o_proj" + ] }, { "type": "gemm", "args": { - "m": 307, - "n": 1536, - "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 1024, + "n": 4096, + "k": 512, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 32 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 1, + 32 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "amd/DeepSeek-R1-0528-MXFP4-Preview/kv_b_proj", + "amd/DeepSeek-R1-0528-MXFP4-v2/kv_b_proj", + "amd/DeepSeek-R1-0528-MXFP4/kv_b_proj", + "amd/Qwen3.5-397B-A17B-MXFP4-AttnFP8-V2/shared_expert.down_proj" + ] }, { "type": "gemm", "args": { - "m": 307, - "n": 24576, - "k": 1536, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 1024, + "n": 4096, + "k": 512, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "Qwen/Qwen3.5-397B-A17B-FP8/o_proj", + "Qwen/Qwen3.5-397B-A17B-FP8/shared_expert.down_proj", + "deepseek-ai/DeepSeek-R1-0528/kv_b_proj" + ] }, { "type": "gemm", "args": { - "m": 307, - "n": 576, - "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 1024, + "n": 4096, + "k": 2880, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/gpt-oss-120b-w-mxfp4-a-fp8/q_proj", + "openai/gpt-oss-120b/q_proj" + ] }, { "type": "gemm", "args": { - "m": 307, - "n": 32768, - "k": 512, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 1024, + "n": 4608, + "k": 7168, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/gate_up_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/gate_up_proj", + "nvidia/Kimi-K2.5-NVFP4/gate_up_proj" + ] }, { "type": "gemm", "args": { - "m": 307, - "n": 7168, - "k": 16384, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 1024, + "n": 4608, + "k": 7168, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 32 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 1, + 32 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "amd/DeepSeek-R1-0528-MXFP4-Preview/gate_up_proj", + "amd/DeepSeek-R1-0528-MXFP4-v2/gate_up_proj", + "amd/DeepSeek-R1-0528-MXFP4/gate_up_proj", + "amd/Kimi-K2.5-MXFP4/gate_up_proj" + ] }, { "type": "gemm", "args": { - "m": 307, + "m": 1024, "n": 4608, "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/gate_up_proj" + ] }, { "type": "gemm", "args": { - "m": 307, - "n": 7168, - "k": 4608, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 1024, + "n": 6144, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/q_b_proj", + "zai-org/GLM-5-FP8/down_proj", + "zai-org/GLM-5.1-FP8/down_proj", + "zai-org/GLM-5.2-FP8/down_proj" + ] }, { "type": "gemm", "args": { - "m": 314624, - "n": 1536, - "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 1024, + "n": 6144, + "k": 3072, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/GLM-5.1-MXFP4/down_proj", + "amd/GLM-5.2-MXFP4/down_proj", + "amd/MiniMax-M2.5-MXFP4/q_proj", + "amd/MiniMax-M3-MXFP4/down_proj", + "nvidia/GLM-5.1-NVFP4/down_proj", + "nvidia/GLM-5.2-NVFP4/down_proj", + "nvidia/MiniMax-M2.5-NVFP4/q_proj" + ] }, { "type": "gemm", "args": { - "m": 314624, - "n": 24576, - "k": 1536, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 1024, + "n": 7168, + "k": 1024, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/o_proj", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] }, { "type": "gemm", "args": { - "m": 314624, - "n": 576, - "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 1024, + "n": 7168, + "k": 1152, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/down_proj" + ] }, { "type": "gemm", "args": { - "m": 314624, - "n": 32768, - "k": 512, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 1024, + "n": 7168, + "k": 2048, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 32 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 1, + 32 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "amd/DeepSeek-R1-0528-MXFP4-Preview/o_proj", + "amd/DeepSeek-R1-0528-MXFP4-v2/o_proj", + "amd/DeepSeek-R1-0528-MXFP4/o_proj" + ] }, { "type": "gemm", "args": { - "m": 314624, + "m": 1024, "n": 7168, - "k": 16384, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "k": 2048, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/o_proj", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] }, { "type": "gemm", "args": { - "m": 314624, + "m": 1024, "n": 7168, - "k": 576, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "k": 2304, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/down_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/down_proj", + "nvidia/Kimi-K2.5-NVFP4/down_proj" + ] }, { "type": "gemm", "args": { - "m": 38, - "n": 1536, - "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 1024, + "n": 7168, + "k": 2304, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 32 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 1, + 32 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "amd/DeepSeek-R1-0528-MXFP4-Preview/down_proj", + "amd/DeepSeek-R1-0528-MXFP4-v2/down_proj", + "amd/DeepSeek-R1-0528-MXFP4/down_proj", + "amd/Kimi-K2.5-MXFP4/down_proj" + ] }, { "type": "gemm", "args": { - "m": 38, - "n": 24576, - "k": 1536, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 1024, + "n": 7168, + "k": 2304, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/down_proj" + ] }, { "type": "gemm", "args": { - "m": 38, - "n": 576, - "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 1024, + "n": 7168, + "k": 4096, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/o_proj", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] }, { "type": "gemm", "args": { - "m": 38, - "n": 32768, - "k": 512, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 1024, + "n": 7168, + "k": 4608, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/down_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/down_proj", + "nvidia/Kimi-K2.5-NVFP4/down_proj" + ] }, { "type": "gemm", "args": { - "m": 38, + "m": 1024, "n": 7168, - "k": 16384, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "k": 4608, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/down_proj" + ] }, { "type": "gemm", "args": { - "m": 38, + "m": 1024, "n": 7168, - "k": 576, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "k": 8192, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/Kimi-K2.5-MXFP4/o_proj", + "moonshotai/Kimi-K2.5/o_proj", + "nvidia/Kimi-K2.5-NVFP4/o_proj", + "nvidia/Kimi-K2.6-NVFP4/o_proj" + ] }, { "type": "gemm", "args": { - "m": 4614144, - "n": 1536, - "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 1024, + "n": 7168, + "k": 8192, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] }, { "type": "gemm", "args": { - "m": 4614144, - "n": 24576, - "k": 1536, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 1024, + "n": 7168, + "k": 9216, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/down_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/down_proj", + "nvidia/Kimi-K2.5-NVFP4/down_proj" + ] }, { "type": "gemm", "args": { - "m": 4614144, - "n": 576, - "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 1024, + "n": 7168, + "k": 16384, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/o_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/o_proj" + ] }, { "type": "gemm", "args": { - "m": 4614144, - "n": 32768, - "k": 512, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 1024, + "n": 7168, + "k": 16384, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 32 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 1, + 32 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "amd/DeepSeek-R1-0528-MXFP4-Preview/o_proj", + "amd/DeepSeek-R1-0528-MXFP4-v2/o_proj", + "amd/DeepSeek-R1-0528-MXFP4/o_proj" + ] }, { "type": "gemm", "args": { - "m": 4614144, + "m": 1024, "n": 7168, "k": 16384, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/o_proj", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] }, { "type": "gemm", "args": { - "m": 4614144, - "n": 4608, - "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 1024, + "n": 7168, + "k": 16384, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] }, { "type": "gemm", "args": { - "m": 4614144, + "m": 1024, "n": 7168, - "k": 4608, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "k": 65536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] }, { "type": "gemm", "args": { - "m": 1153536, - "n": 1536, - "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 1024, + "n": 8192, + "k": 512, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/kv_b_proj" + ] }, { "type": "gemm", "args": { - "m": 1153536, - "n": 24576, + "m": 1024, + "n": 8192, "k": 1536, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_b" + ] }, { "type": "gemm", "args": { - "m": 1153536, - "n": 576, + "m": 1024, + "n": 9216, "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/gate_up_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/gate_up_proj", + "nvidia/Kimi-K2.5-NVFP4/gate_up_proj" + ] }, { "type": "gemm", "args": { - "m": 1153536, - "n": 32768, - "k": 512, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 1024, + "n": 9216, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/gate_up_proj" + ] }, { "type": "gemm", "args": { - "m": 1153536, - "n": 7168, - "k": 16384, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 1024, + "n": 12288, + "k": 1536, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/Kimi-K2.5-MXFP4/q_b_proj", + "moonshotai/Kimi-K2.5/q_b_proj", + "nvidia/DeepSeek-R1-0528-FP4-V2/q_b_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/q_b_proj", + "nvidia/Kimi-K2.5-NVFP4/q_b_proj", + "nvidia/Kimi-K2.6-NVFP4/q_b_proj" + ] }, { "type": "gemm", "args": { - "m": 1153536, - "n": 1152, - "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 1024, + "n": 15360, + "k": 1024, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] }, { "type": "gemm", "args": { - "m": 1153536, - "n": 7168, - "k": 1152, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 1024, + "n": 16384, + "k": 512, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/Kimi-K2.5-MXFP4/kv_b_proj", + "moonshotai/Kimi-K2.5/kv_b_proj", + "nvidia/DeepSeek-R1-0528-FP4-V2/kv_b_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/kv_b_proj", + "nvidia/Kimi-K2.5-NVFP4/kv_b_proj", + "nvidia/Kimi-K2.6-NVFP4/kv_b_proj" + ] }, { "type": "gemm", "args": { - "m": 140, - "n": 1152, - "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 1024, + "n": 16384, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_b" + ] }, { "type": "gemm", "args": { - "m": 140, - "n": 7168, - "k": 1152, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 1024, + "n": 24576, + "k": 1536, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/q_b_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/q_b_proj" + ] }, { "type": "gemm", "args": { - "m": 8388608, - "n": 4608, - "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 1024, + "n": 24576, + "k": 1536, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 32 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 1, + 32 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "amd/DeepSeek-R1-0528-MXFP4-Preview/q_b_proj", + "amd/DeepSeek-R1-0528-MXFP4-v2/q_b_proj", + "amd/DeepSeek-R1-0528-MXFP4/q_b_proj" + ] }, { "type": "gemm", "args": { - "m": 8388608, - "n": 7168, - "k": 4608, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 1024, + "n": 24576, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/q_b_proj" + ] }, { "type": "gemm", "args": { - "m": 2097152, - "n": 1152, - "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 1024, + "n": 30720, + "k": 1024, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] }, { "type": "gemm", "args": { - "m": 2097152, - "n": 7168, - "k": 1152, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 1024, + "n": 32768, + "k": 512, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/kv_b_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/kv_b_proj" + ] }, { "type": "gemm", "args": { - "m": 256, - "n": 1152, - "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 1024, + "n": 32768, + "k": 512, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 32 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 1, + 32 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "amd/DeepSeek-R1-0528-MXFP4-Preview/kv_b_proj", + "amd/DeepSeek-R1-0528-MXFP4-v2/kv_b_proj", + "amd/DeepSeek-R1-0528-MXFP4/kv_b_proj" + ] }, { "type": "gemm", "args": { - "m": 256, - "n": 7168, - "k": 1152, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 1024, + "n": 32768, + "k": 512, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/kv_b_proj" + ] }, { "type": "gemm", "args": { - "m": 550528, - "n": 1536, - "k": 7168, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 1024, + "n": 65536, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_b" + ] }, { "type": "gemm", "args": { - "m": 550528, - "n": 24576, - "k": 1536, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 1024, + "n": 122880, + "k": 1024, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] }, { "type": "gemm", "args": { - "m": 550528, + "m": 1075, "n": 576, "k": 7168, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/kv_a_proj_with_mqa" + ] }, { "type": "gemm", "args": { - "m": 550528, - "n": 32768, - "k": 512, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 1075, + "n": 1536, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/gate_up_proj", + "deepseek-ai/DeepSeek-R1-0528/q_a_proj", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w3" + ] }, { "type": "gemm", "args": { - "m": 550528, - "n": 7168, - "k": 16384, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 1075, + "n": 2304, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/gate_up_proj" + ] }, { "type": "gemm", "args": { - "m": 550528, - "n": 2304, + "m": 1075, + "n": 4608, "k": 7168, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/gate_up_proj" + ] }, { "type": "gemm", "args": { - "m": 550528, + "m": 1075, "n": 7168, "k": 2304, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/down_proj" + ] }, { "type": "gemm", "args": { "m": 1075, - "n": 1536, - "k": 7168, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "n": 7168, + "k": 4608, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/down_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/down_proj", + "nvidia/Kimi-K2.5-NVFP4/down_proj" + ] }, { "type": "gemm", "args": { "m": 1075, - "n": 24576, - "k": 1536, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "n": 7168, + "k": 4608, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 32 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 1, + 32 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "amd/DeepSeek-R1-0528-MXFP4-Preview/down_proj", + "amd/DeepSeek-R1-0528-MXFP4-v2/down_proj", + "amd/DeepSeek-R1-0528-MXFP4/down_proj", + "amd/Kimi-K2.5-MXFP4/down_proj" + ] }, { "type": "gemm", "args": { "m": 1075, - "n": 576, - "k": 7168, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "n": 7168, + "k": 8192, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/Kimi-K2.5-MXFP4/o_proj", + "moonshotai/Kimi-K2.5/o_proj", + "nvidia/Kimi-K2.5-NVFP4/o_proj", + "nvidia/Kimi-K2.6-NVFP4/o_proj" + ] }, { "type": "gemm", "args": { "m": 1075, - "n": 32768, - "k": 512, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "n": 7168, + "k": 8192, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] }, { "type": "gemm", @@ -56545,22790 +117832,72537 @@ "m": 1075, "n": 7168, "k": 16384, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/o_proj", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] }, { "type": "gemm", "args": { "m": 1075, - "n": 2304, - "k": 7168, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "n": 12288, + "k": 1536, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/Kimi-K2.5-MXFP4/q_b_proj", + "moonshotai/Kimi-K2.5/q_b_proj", + "nvidia/DeepSeek-R1-0528-FP4-V2/q_b_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/q_b_proj", + "nvidia/Kimi-K2.5-NVFP4/q_b_proj", + "nvidia/Kimi-K2.6-NVFP4/q_b_proj" + ] }, { "type": "gemm", "args": { "m": 1075, - "n": 7168, - "k": 2304, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "n": 16384, + "k": 512, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/Kimi-K2.5-MXFP4/kv_b_proj", + "moonshotai/Kimi-K2.5/kv_b_proj", + "nvidia/DeepSeek-R1-0528-FP4-V2/kv_b_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/kv_b_proj", + "nvidia/Kimi-K2.5-NVFP4/kv_b_proj", + "nvidia/Kimi-K2.6-NVFP4/kv_b_proj" + ] }, { "type": "gemm", "args": { - "m": 275328, - "n": 1536, - "k": 7168, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 1075, + "n": 24576, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/q_b_proj" + ] }, { "type": "gemm", "args": { - "m": 275328, - "n": 24576, - "k": 1536, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 1075, + "n": 32768, + "k": 512, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/kv_b_proj" + ] }, { "type": "gemm", "args": { - "m": 275328, + "m": 1126, "n": 576, "k": 7168, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/Kimi-K2.5-MXFP4/kv_a_proj_with_mqa", + "moonshotai/Kimi-K2.5/kv_a_proj_with_mqa", + "nvidia/DeepSeek-R1-0528-FP4-V2/kv_a_proj_with_mqa", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/kv_a_proj_with_mqa", + "nvidia/Kimi-K2.5-NVFP4/kv_a_proj_with_mqa", + "nvidia/Kimi-K2.6-NVFP4/kv_a_proj_with_mqa" + ] }, { "type": "gemm", "args": { - "m": 275328, - "n": 32768, - "k": 512, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 1126, + "n": 576, + "k": 7168, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 32 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 1, + 32 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "amd/DeepSeek-R1-0528-MXFP4-Preview/kv_a_proj_with_mqa", + "amd/DeepSeek-R1-0528-MXFP4-v2/kv_a_proj_with_mqa", + "amd/DeepSeek-R1-0528-MXFP4/kv_a_proj_with_mqa" + ] }, { "type": "gemm", "args": { - "m": 275328, - "n": 7168, - "k": 16384, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 1126, + "n": 576, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/kv_a_proj_with_mqa" + ] }, { "type": "gemm", "args": { - "m": 275328, - "n": 2304, + "m": 1126, + "n": 1536, "k": 7168, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/Kimi-K2.5-MXFP4/q_a_proj", + "moonshotai/Kimi-K2.5/q_a_proj", + "nvidia/DeepSeek-R1-0528-FP4-V2/q_a_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/q_a_proj", + "nvidia/Kimi-K2.5-NVFP4/q_a_proj", + "nvidia/Kimi-K2.6-NVFP4/q_a_proj" + ] }, { "type": "gemm", "args": { - "m": 275328, - "n": 7168, - "k": 2304, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 1126, + "n": 1536, + "k": 7168, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 32 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 1, + 32 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "amd/DeepSeek-R1-0528-MXFP4-Preview/q_a_proj", + "amd/DeepSeek-R1-0528-MXFP4-v2/q_a_proj", + "amd/DeepSeek-R1-0528-MXFP4/q_a_proj" + ] }, { "type": "gemm", "args": { - "m": 537, + "m": 1126, "n": 1536, "k": 7168, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/gate_up_proj", + "deepseek-ai/DeepSeek-R1-0528/q_a_proj", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w3" + ] }, { "type": "gemm", "args": { - "m": 537, - "n": 24576, - "k": 1536, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 1126, + "n": 1536, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w3" + ] }, { "type": "gemm", "args": { - "m": 537, - "n": 576, + "m": 1126, + "n": 4608, "k": 7168, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/gate_up_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/gate_up_proj", + "nvidia/Kimi-K2.5-NVFP4/gate_up_proj" + ] }, { "type": "gemm", "args": { - "m": 537, - "n": 32768, - "k": 512, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 1126, + "n": 4608, + "k": 7168, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 32 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 1, + 32 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "amd/DeepSeek-R1-0528-MXFP4-Preview/gate_up_proj", + "amd/DeepSeek-R1-0528-MXFP4-v2/gate_up_proj", + "amd/DeepSeek-R1-0528-MXFP4/gate_up_proj", + "amd/Kimi-K2.5-MXFP4/gate_up_proj" + ] }, { "type": "gemm", "args": { - "m": 537, - "n": 7168, - "k": 16384, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 1126, + "n": 4608, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/gate_up_proj" + ] }, { "type": "gemm", "args": { - "m": 537, - "n": 2304, - "k": 7168, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 1126, + "n": 7168, + "k": 4608, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/down_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/down_proj", + "nvidia/Kimi-K2.5-NVFP4/down_proj" + ] }, { "type": "gemm", "args": { - "m": 537, + "m": 1126, "n": 7168, - "k": 2304, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "k": 4608, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 32 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 1, + 32 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "amd/DeepSeek-R1-0528-MXFP4-Preview/down_proj", + "amd/DeepSeek-R1-0528-MXFP4-v2/down_proj", + "amd/DeepSeek-R1-0528-MXFP4/down_proj", + "amd/Kimi-K2.5-MXFP4/down_proj" + ] }, { "type": "gemm", "args": { - "m": 157312, - "n": 1536, - "k": 7168, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 1126, + "n": 7168, + "k": 4608, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/down_proj" + ] }, { "type": "gemm", "args": { - "m": 157312, - "n": 24576, - "k": 1536, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 1126, + "n": 7168, + "k": 9216, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/down_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/down_proj", + "nvidia/Kimi-K2.5-NVFP4/down_proj" + ] }, { "type": "gemm", "args": { - "m": 157312, - "n": 576, - "k": 7168, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 1126, + "n": 7168, + "k": 16384, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/o_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/o_proj" + ] }, { "type": "gemm", "args": { - "m": 157312, - "n": 32768, - "k": 512, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 1126, + "n": 7168, + "k": 16384, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 32 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 1, + 32 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "amd/DeepSeek-R1-0528-MXFP4-Preview/o_proj", + "amd/DeepSeek-R1-0528-MXFP4-v2/o_proj", + "amd/DeepSeek-R1-0528-MXFP4/o_proj" + ] }, { "type": "gemm", "args": { - "m": 157312, + "m": 1126, "n": 7168, "k": 16384, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/o_proj", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] }, { "type": "gemm", "args": { - "m": 157312, - "n": 2304, - "k": 7168, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 1126, + "n": 7168, + "k": 16384, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] }, { "type": "gemm", "args": { - "m": 157312, - "n": 7168, - "k": 2304, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 1126, + "n": 9216, + "k": 7168, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/gate_up_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/gate_up_proj", + "nvidia/Kimi-K2.5-NVFP4/gate_up_proj" + ] }, { "type": "gemm", "args": { - "m": 307, - "n": 2304, + "m": 1126, + "n": 9216, "k": 7168, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 32 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 1, + 32 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "amd/DeepSeek-R1-0528-MXFP4-Preview/gate_up_proj", + "amd/DeepSeek-R1-0528-MXFP4-v2/gate_up_proj", + "amd/DeepSeek-R1-0528-MXFP4/gate_up_proj", + "amd/Kimi-K2.5-MXFP4/gate_up_proj" + ] }, { "type": "gemm", "args": { - "m": 307, - "n": 7168, - "k": 2304, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 1126, + "n": 24576, + "k": 1536, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/q_b_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/q_b_proj" + ] }, { "type": "gemm", "args": { - "m": 78720, - "n": 1536, - "k": 7168, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 1126, + "n": 24576, + "k": 1536, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 32 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 1, + 32 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "amd/DeepSeek-R1-0528-MXFP4-Preview/q_b_proj", + "amd/DeepSeek-R1-0528-MXFP4-v2/q_b_proj", + "amd/DeepSeek-R1-0528-MXFP4/q_b_proj" + ] }, { "type": "gemm", "args": { - "m": 78720, + "m": 1126, "n": 24576, "k": 1536, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/q_b_proj" + ] }, { "type": "gemm", "args": { - "m": 78720, - "n": 576, - "k": 7168, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 1126, + "n": 32768, + "k": 512, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/kv_b_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/kv_b_proj" + ] }, { "type": "gemm", "args": { - "m": 78720, + "m": 1126, "n": 32768, "k": 512, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 32 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 1, + 32 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "amd/DeepSeek-R1-0528-MXFP4-Preview/kv_b_proj", + "amd/DeepSeek-R1-0528-MXFP4-v2/kv_b_proj", + "amd/DeepSeek-R1-0528-MXFP4/kv_b_proj" + ] }, { "type": "gemm", "args": { - "m": 78720, - "n": 7168, - "k": 16384, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 1126, + "n": 32768, + "k": 512, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/kv_b_proj" + ] }, { "type": "gemm", "args": { - "m": 78720, - "n": 2304, + "m": 1127, + "n": 576, "k": 7168, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/Kimi-K2.5-MXFP4/kv_a_proj_with_mqa", + "moonshotai/Kimi-K2.5/kv_a_proj_with_mqa", + "nvidia/DeepSeek-R1-0528-FP4-V2/kv_a_proj_with_mqa", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/kv_a_proj_with_mqa", + "nvidia/Kimi-K2.5-NVFP4/kv_a_proj_with_mqa", + "nvidia/Kimi-K2.6-NVFP4/kv_a_proj_with_mqa" + ] }, { "type": "gemm", "args": { - "m": 78720, - "n": 7168, - "k": 2304, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 1127, + "n": 576, + "k": 7168, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 32 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 1, + 32 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "amd/DeepSeek-R1-0528-MXFP4-Preview/kv_a_proj_with_mqa", + "amd/DeepSeek-R1-0528-MXFP4-v2/kv_a_proj_with_mqa", + "amd/DeepSeek-R1-0528-MXFP4/kv_a_proj_with_mqa" + ] }, { "type": "gemm", "args": { - "m": 153, - "n": 2304, + "m": 1127, + "n": 1536, "k": 7168, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/Kimi-K2.5-MXFP4/q_a_proj", + "moonshotai/Kimi-K2.5/q_a_proj", + "nvidia/DeepSeek-R1-0528-FP4-V2/q_a_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/q_a_proj", + "nvidia/Kimi-K2.5-NVFP4/q_a_proj", + "nvidia/Kimi-K2.6-NVFP4/q_a_proj" + ] }, { "type": "gemm", "args": { - "m": 153, - "n": 7168, - "k": 2304, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 1127, + "n": 1536, + "k": 7168, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 32 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 1, + 32 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "amd/DeepSeek-R1-0528-MXFP4-Preview/q_a_proj", + "amd/DeepSeek-R1-0528-MXFP4-v2/q_a_proj", + "amd/DeepSeek-R1-0528-MXFP4/q_a_proj" + ] }, { "type": "gemm", "args": { - "m": 19680, + "m": 1127, "n": 1536, "k": 7168, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w3" + ] }, { "type": "gemm", "args": { - "m": 19680, - "n": 24576, - "k": 1536, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 1127, + "n": 7168, + "k": 9216, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/down_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/down_proj", + "nvidia/Kimi-K2.5-NVFP4/down_proj" + ] }, { "type": "gemm", "args": { - "m": 19680, - "n": 576, - "k": 7168, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 1127, + "n": 7168, + "k": 16384, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/o_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/o_proj" + ] }, { "type": "gemm", "args": { - "m": 19680, - "n": 32768, - "k": 512, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 1127, + "n": 7168, + "k": 16384, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 32 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 1, + 32 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "amd/DeepSeek-R1-0528-MXFP4-Preview/o_proj", + "amd/DeepSeek-R1-0528-MXFP4-v2/o_proj", + "amd/DeepSeek-R1-0528-MXFP4/o_proj" + ] }, { "type": "gemm", "args": { - "m": 19680, + "m": 1127, "n": 7168, "k": 16384, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] }, { "type": "gemm", "args": { - "m": 19680, - "n": 7168, - "k": 576, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 1127, + "n": 9216, + "k": 7168, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/gate_up_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/gate_up_proj", + "nvidia/Kimi-K2.5-NVFP4/gate_up_proj" + ] }, { "type": "gemm", "args": { - "m": 38, - "n": 1536, + "m": 1127, + "n": 9216, "k": 7168, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 32 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 1, + 32 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "amd/DeepSeek-R1-0528-MXFP4-Preview/gate_up_proj", + "amd/DeepSeek-R1-0528-MXFP4-v2/gate_up_proj", + "amd/DeepSeek-R1-0528-MXFP4/gate_up_proj", + "amd/Kimi-K2.5-MXFP4/gate_up_proj" + ] }, { "type": "gemm", "args": { - "m": 38, + "m": 1127, "n": 24576, "k": 1536, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/q_b_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/q_b_proj" + ] }, { "type": "gemm", "args": { - "m": 38, - "n": 576, - "k": 7168, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 1127, + "n": 24576, + "k": 1536, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 32 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 1, + 32 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "amd/DeepSeek-R1-0528-MXFP4-Preview/q_b_proj", + "amd/DeepSeek-R1-0528-MXFP4-v2/q_b_proj", + "amd/DeepSeek-R1-0528-MXFP4/q_b_proj" + ] }, { "type": "gemm", "args": { - "m": 38, + "m": 1127, "n": 32768, "k": 512, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/kv_b_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/kv_b_proj" + ] }, { "type": "gemm", "args": { - "m": 38, - "n": 7168, - "k": 16384, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 1127, + "n": 32768, + "k": 512, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 32 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 1, + 32 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "amd/DeepSeek-R1-0528-MXFP4-Preview/kv_b_proj", + "amd/DeepSeek-R1-0528-MXFP4-v2/kv_b_proj", + "amd/DeepSeek-R1-0528-MXFP4/kv_b_proj" + ] }, { "type": "gemm", "args": { - "m": 38, - "n": 7168, - "k": 576, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 1152, + "n": 576, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/kv_a_proj_with_mqa" + ] }, { "type": "gemm", "args": { - "m": 4608, + "m": 1152, + "n": 1536, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/gate_up_proj", + "deepseek-ai/DeepSeek-R1-0528/q_a_proj", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w3" + ] + }, + { + "type": "gemm", + "args": { + "m": 1152, "n": 2304, "k": 7168, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/gate_up_proj" + ] }, { "type": "gemm", "args": { - "m": 4608, + "m": 1152, "n": 7168, "k": 2304, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/down_proj" + ] }, { "type": "gemm", "args": { - "m": 9, - "n": 2304, - "k": 7168, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 1152, + "n": 7168, + "k": 16384, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/o_proj", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] }, { "type": "gemm", "args": { - "m": 9, - "n": 7168, - "k": 2304, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 1152, + "n": 24576, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/q_b_proj" + ] }, { "type": "gemm", "args": { - "m": 36864, - "n": 1536, - "k": 7168, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 1152, + "n": 32768, + "k": 512, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/kv_b_proj" + ] }, { "type": "gemm", "args": { - "m": 36864, - "n": 3072, - "k": 1536, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 1197, + "n": 576, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/Kimi-K2.5-MXFP4/kv_a_proj_with_mqa", + "moonshotai/Kimi-K2.5/kv_a_proj_with_mqa", + "nvidia/DeepSeek-R1-0528-FP4-V2/kv_a_proj_with_mqa", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/kv_a_proj_with_mqa", + "nvidia/Kimi-K2.5-NVFP4/kv_a_proj_with_mqa", + "nvidia/Kimi-K2.6-NVFP4/kv_a_proj_with_mqa" + ] }, { "type": "gemm", "args": { - "m": 36864, + "m": 1197, "n": 576, "k": 7168, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 32 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 1, + 32 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "amd/DeepSeek-R1-0528-MXFP4-Preview/kv_a_proj_with_mqa", + "amd/DeepSeek-R1-0528-MXFP4-v2/kv_a_proj_with_mqa", + "amd/DeepSeek-R1-0528-MXFP4/kv_a_proj_with_mqa" + ] }, { "type": "gemm", "args": { - "m": 36864, - "n": 4096, - "k": 512, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 1197, + "n": 1536, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/Kimi-K2.5-MXFP4/q_a_proj", + "moonshotai/Kimi-K2.5/q_a_proj", + "nvidia/DeepSeek-R1-0528-FP4-V2/q_a_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/q_a_proj", + "nvidia/Kimi-K2.5-NVFP4/q_a_proj", + "nvidia/Kimi-K2.6-NVFP4/q_a_proj" + ] }, { "type": "gemm", "args": { - "m": 36864, - "n": 7168, - "k": 2048, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 1197, + "n": 1536, + "k": 7168, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 32 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 1, + 32 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "amd/DeepSeek-R1-0528-MXFP4-Preview/q_a_proj", + "amd/DeepSeek-R1-0528-MXFP4-v2/q_a_proj", + "amd/DeepSeek-R1-0528-MXFP4/q_a_proj" + ] }, { "type": "gemm", "args": { - "m": 36864, - "n": 2304, + "m": 1197, + "n": 1536, "k": 7168, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w3" + ] }, { "type": "gemm", "args": { - "m": 36864, + "m": 1197, "n": 7168, - "k": 2304, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "k": 9216, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/down_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/down_proj", + "nvidia/Kimi-K2.5-NVFP4/down_proj" + ] }, { "type": "gemm", "args": { - "m": 72, - "n": 3072, - "k": 1536, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 1197, + "n": 7168, + "k": 16384, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/o_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/o_proj" + ] }, { "type": "gemm", "args": { - "m": 72, - "n": 4096, - "k": 512, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 1197, + "n": 7168, + "k": 16384, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 32 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 1, + 32 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "amd/DeepSeek-R1-0528-MXFP4-Preview/o_proj", + "amd/DeepSeek-R1-0528-MXFP4-v2/o_proj", + "amd/DeepSeek-R1-0528-MXFP4/o_proj" + ] }, { "type": "gemm", "args": { - "m": 72, + "m": 1197, "n": 7168, - "k": 2048, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "k": 16384, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] }, { "type": "gemm", "args": { - "m": 2304, - "n": 1536, + "m": 1197, + "n": 9216, "k": 7168, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/gate_up_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/gate_up_proj", + "nvidia/Kimi-K2.5-NVFP4/gate_up_proj" + ] }, { "type": "gemm", "args": { - "m": 2304, + "m": 1197, + "n": 9216, + "k": 7168, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 32 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 1, + 32 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "amd/DeepSeek-R1-0528-MXFP4-Preview/gate_up_proj", + "amd/DeepSeek-R1-0528-MXFP4-v2/gate_up_proj", + "amd/DeepSeek-R1-0528-MXFP4/gate_up_proj", + "amd/Kimi-K2.5-MXFP4/gate_up_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 1197, "n": 24576, "k": 1536, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/q_b_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/q_b_proj" + ] }, { "type": "gemm", "args": { - "m": 2304, - "n": 576, - "k": 7168, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 1197, + "n": 24576, + "k": 1536, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 32 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 1, + 32 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "amd/DeepSeek-R1-0528-MXFP4-Preview/q_b_proj", + "amd/DeepSeek-R1-0528-MXFP4-v2/q_b_proj", + "amd/DeepSeek-R1-0528-MXFP4/q_b_proj" + ] }, { "type": "gemm", "args": { - "m": 2304, + "m": 1197, "n": 32768, "k": 512, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/kv_b_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/kv_b_proj" + ] }, { "type": "gemm", "args": { - "m": 2304, - "n": 7168, - "k": 16384, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 1197, + "n": 32768, + "k": 512, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 32 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 1, + 32 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "amd/DeepSeek-R1-0528-MXFP4-Preview/kv_b_proj", + "amd/DeepSeek-R1-0528-MXFP4-v2/kv_b_proj", + "amd/DeepSeek-R1-0528-MXFP4/kv_b_proj" + ] }, { "type": "gemm", "args": { - "m": 2304, - "n": 2304, + "m": 1229, + "n": 576, "k": 7168, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/Kimi-K2.5-MXFP4/kv_a_proj_with_mqa", + "moonshotai/Kimi-K2.5/kv_a_proj_with_mqa", + "nvidia/DeepSeek-R1-0528-FP4-V2/kv_a_proj_with_mqa", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/kv_a_proj_with_mqa", + "nvidia/Kimi-K2.5-NVFP4/kv_a_proj_with_mqa", + "nvidia/Kimi-K2.6-NVFP4/kv_a_proj_with_mqa" + ] }, { "type": "gemm", "args": { - "m": 2304, - "n": 7168, - "k": 2304, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 1229, + "n": 576, + "k": 7168, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 32 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 1, + 32 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "amd/DeepSeek-R1-0528-MXFP4-Preview/kv_a_proj_with_mqa", + "amd/DeepSeek-R1-0528-MXFP4-v2/kv_a_proj_with_mqa", + "amd/DeepSeek-R1-0528-MXFP4/kv_a_proj_with_mqa" + ] }, { "type": "gemm", "args": { - "m": 18432, + "m": 1229, "n": 1536, "k": 7168, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/Kimi-K2.5-MXFP4/q_a_proj", + "moonshotai/Kimi-K2.5/q_a_proj", + "nvidia/DeepSeek-R1-0528-FP4-V2/q_a_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/q_a_proj", + "nvidia/Kimi-K2.5-NVFP4/q_a_proj", + "nvidia/Kimi-K2.6-NVFP4/q_a_proj" + ] }, { "type": "gemm", "args": { - "m": 18432, - "n": 3072, - "k": 1536, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 18432, - "n": 576, + "m": 1229, + "n": 1536, "k": 7168, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 32 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 1, + 32 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "amd/DeepSeek-R1-0528-MXFP4-Preview/q_a_proj", + "amd/DeepSeek-R1-0528-MXFP4-v2/q_a_proj", + "amd/DeepSeek-R1-0528-MXFP4/q_a_proj" + ] }, { "type": "gemm", "args": { - "m": 18432, - "n": 4096, - "k": 512, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 1229, + "n": 1536, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w3" + ] }, { "type": "gemm", "args": { - "m": 18432, + "m": 1229, "n": 7168, - "k": 2048, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 18432, - "n": 2304, - "k": 7168, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "k": 9216, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/down_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/down_proj", + "nvidia/Kimi-K2.5-NVFP4/down_proj" + ] }, { "type": "gemm", "args": { - "m": 18432, + "m": 1229, "n": 7168, - "k": 2304, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "k": 16384, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/o_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/o_proj" + ] }, { "type": "gemm", "args": { - "m": 36, - "n": 3072, - "k": 1536, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 1229, + "n": 7168, + "k": 16384, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 32 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 1, + 32 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "amd/DeepSeek-R1-0528-MXFP4-Preview/o_proj", + "amd/DeepSeek-R1-0528-MXFP4-v2/o_proj", + "amd/DeepSeek-R1-0528-MXFP4/o_proj" + ] }, { "type": "gemm", "args": { - "m": 36, - "n": 4096, - "k": 512, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 1229, + "n": 7168, + "k": 16384, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] }, { "type": "gemm", "args": { - "m": 36, - "n": 7168, - "k": 2048, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 1229, + "n": 9216, + "k": 7168, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/gate_up_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/gate_up_proj", + "nvidia/Kimi-K2.5-NVFP4/gate_up_proj" + ] }, { "type": "gemm", "args": { - "m": 1152, - "n": 1536, + "m": 1229, + "n": 9216, "k": 7168, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 32 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 1, + 32 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "amd/DeepSeek-R1-0528-MXFP4-Preview/gate_up_proj", + "amd/DeepSeek-R1-0528-MXFP4-v2/gate_up_proj", + "amd/DeepSeek-R1-0528-MXFP4/gate_up_proj", + "amd/Kimi-K2.5-MXFP4/gate_up_proj" + ] }, { "type": "gemm", "args": { - "m": 1152, + "m": 1229, "n": 24576, "k": 1536, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/q_b_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/q_b_proj" + ] }, { "type": "gemm", "args": { - "m": 1152, - "n": 576, - "k": 7168, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 1229, + "n": 24576, + "k": 1536, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 32 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 1, + 32 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "amd/DeepSeek-R1-0528-MXFP4-Preview/q_b_proj", + "amd/DeepSeek-R1-0528-MXFP4-v2/q_b_proj", + "amd/DeepSeek-R1-0528-MXFP4/q_b_proj" + ] }, { "type": "gemm", "args": { - "m": 1152, + "m": 1229, "n": 32768, "k": 512, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/kv_b_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/kv_b_proj" + ] }, { "type": "gemm", "args": { - "m": 1152, - "n": 7168, - "k": 16384, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 1229, + "n": 32768, + "k": 512, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 32 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 1, + 32 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "amd/DeepSeek-R1-0528-MXFP4-Preview/kv_b_proj", + "amd/DeepSeek-R1-0528-MXFP4-v2/kv_b_proj", + "amd/DeepSeek-R1-0528-MXFP4/kv_b_proj" + ] }, { "type": "gemm", "args": { - "m": 1152, - "n": 2304, + "m": 1242, + "n": 576, "k": 7168, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 1152, - "n": 7168, - "k": 2304, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/Kimi-K2.5-MXFP4/kv_a_proj_with_mqa", + "moonshotai/Kimi-K2.5/kv_a_proj_with_mqa", + "nvidia/DeepSeek-R1-0528-FP4-V2/kv_a_proj_with_mqa", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/kv_a_proj_with_mqa", + "nvidia/Kimi-K2.5-NVFP4/kv_a_proj_with_mqa", + "nvidia/Kimi-K2.6-NVFP4/kv_a_proj_with_mqa" + ] }, { "type": "gemm", "args": { - "m": 9216, - "n": 3072, - "k": 1536, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 1242, + "n": 1536, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/Kimi-K2.5-MXFP4/q_a_proj", + "moonshotai/Kimi-K2.5/q_a_proj", + "nvidia/DeepSeek-R1-0528-FP4-V2/q_a_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/q_a_proj", + "nvidia/Kimi-K2.5-NVFP4/q_a_proj", + "nvidia/Kimi-K2.6-NVFP4/q_a_proj" + ] }, { "type": "gemm", "args": { - "m": 9216, - "n": 4096, - "k": 512, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 1242, + "n": 2304, + "k": 7168, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/gate_up_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/gate_up_proj", + "nvidia/Kimi-K2.5-NVFP4/gate_up_proj" + ] }, { "type": "gemm", "args": { - "m": 9216, + "m": 1242, "n": 7168, - "k": 2048, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 9216, - "n": 2304, - "k": 7168, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "k": 2304, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/down_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/down_proj", + "nvidia/Kimi-K2.5-NVFP4/down_proj" + ] }, { "type": "gemm", "args": { - "m": 9216, + "m": 1242, "n": 7168, - "k": 2304, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "k": 16384, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/o_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/o_proj" + ] }, { "type": "gemm", "args": { - "m": 18, - "n": 3072, + "m": 1242, + "n": 24576, "k": 1536, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/q_b_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/q_b_proj" + ] }, { "type": "gemm", "args": { - "m": 18, - "n": 4096, + "m": 1242, + "n": 32768, "k": 512, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/kv_b_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/kv_b_proj" + ] }, { "type": "gemm", "args": { - "m": 18, - "n": 7168, - "k": 2048, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 1280, + "n": 512, + "k": 2880, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/gpt-oss-120b-w-mxfp4-a-fp8/k_proj", + "amd/gpt-oss-120b-w-mxfp4-a-fp8/q_proj", + "amd/gpt-oss-120b-w-mxfp4-a-fp8/v_proj", + "openai/gpt-oss-120b/k_proj", + "openai/gpt-oss-120b/q_proj", + "openai/gpt-oss-120b/v_proj" + ] }, { "type": "gemm", "args": { - "m": 268, - "n": 2304, + "m": 1280, + "n": 576, "k": 7168, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 268, - "n": 7168, - "k": 2304, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/Kimi-K2.5-MXFP4/kv_a_proj_with_mqa", + "moonshotai/Kimi-K2.5/kv_a_proj_with_mqa", + "nvidia/DeepSeek-R1-0528-FP4-V2/kv_a_proj_with_mqa", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/kv_a_proj_with_mqa", + "nvidia/Kimi-K2.5-NVFP4/kv_a_proj_with_mqa", + "nvidia/Kimi-K2.6-NVFP4/kv_a_proj_with_mqa" + ] }, { "type": "gemm", "args": { - "m": 137664, + "m": 1280, "n": 1536, "k": 7168, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/Kimi-K2.5-MXFP4/q_a_proj", + "moonshotai/Kimi-K2.5/q_a_proj", + "nvidia/DeepSeek-R1-0528-FP4-V2/q_a_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/q_a_proj", + "nvidia/Kimi-K2.5-NVFP4/q_a_proj", + "nvidia/Kimi-K2.6-NVFP4/q_a_proj" + ] }, { "type": "gemm", "args": { - "m": 137664, - "n": 24576, - "k": 1536, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 1280, + "n": 2880, + "k": 4096, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/gpt-oss-120b-w-mxfp4-a-fp8/o_proj", + "openai/gpt-oss-120b/o_proj" + ] }, { "type": "gemm", "args": { - "m": 137664, - "n": 576, - "k": 7168, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 1280, + "n": 4096, + "k": 2880, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/gpt-oss-120b-w-mxfp4-a-fp8/q_proj", + "openai/gpt-oss-120b/q_proj" + ] }, { "type": "gemm", "args": { - "m": 137664, - "n": 32768, - "k": 512, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 1280, + "n": 4608, + "k": 7168, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/gate_up_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/gate_up_proj", + "nvidia/Kimi-K2.5-NVFP4/gate_up_proj" + ] }, { "type": "gemm", "args": { - "m": 137664, - "n": 7168, - "k": 16384, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 1280, + "n": 4608, + "k": 7168, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 32 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 1, + 32 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "amd/DeepSeek-R1-0528-MXFP4-Preview/gate_up_proj", + "amd/DeepSeek-R1-0528-MXFP4-v2/gate_up_proj", + "amd/DeepSeek-R1-0528-MXFP4/gate_up_proj", + "amd/Kimi-K2.5-MXFP4/gate_up_proj" + ] }, { "type": "gemm", "args": { - "m": 137664, - "n": 1152, + "m": 1280, + "n": 4608, "k": 7168, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/gate_up_proj" + ] }, { "type": "gemm", "args": { - "m": 137664, + "m": 1280, "n": 7168, - "k": 1152, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "k": 4608, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/down_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/down_proj", + "nvidia/Kimi-K2.5-NVFP4/down_proj" + ] }, { "type": "gemm", "args": { - "m": 134, - "n": 1152, - "k": 7168, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 1280, + "n": 7168, + "k": 8192, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/Kimi-K2.5-MXFP4/o_proj", + "moonshotai/Kimi-K2.5/o_proj", + "nvidia/Kimi-K2.5-NVFP4/o_proj", + "nvidia/Kimi-K2.6-NVFP4/o_proj" + ] }, { "type": "gemm", "args": { - "m": 134, + "m": 1280, "n": 7168, - "k": 1152, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "k": 16384, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/o_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/o_proj" + ] }, { "type": "gemm", "args": { - "m": 144256, - "n": 1536, - "k": 7168, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 1280, + "n": 12288, + "k": 1536, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/Kimi-K2.5-MXFP4/q_b_proj", + "moonshotai/Kimi-K2.5/q_b_proj", + "nvidia/DeepSeek-R1-0528-FP4-V2/q_b_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/q_b_proj", + "nvidia/Kimi-K2.5-NVFP4/q_b_proj", + "nvidia/Kimi-K2.6-NVFP4/q_b_proj" + ] }, { "type": "gemm", "args": { - "m": 144256, - "n": 24576, - "k": 1536, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 1280, + "n": 16384, + "k": 512, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/Kimi-K2.5-MXFP4/kv_b_proj", + "moonshotai/Kimi-K2.5/kv_b_proj", + "nvidia/DeepSeek-R1-0528-FP4-V2/kv_b_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/kv_b_proj", + "nvidia/Kimi-K2.5-NVFP4/kv_b_proj", + "nvidia/Kimi-K2.6-NVFP4/kv_b_proj" + ] }, { "type": "gemm", "args": { - "m": 144256, - "n": 576, - "k": 7168, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 1280, + "n": 24576, + "k": 1536, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/q_b_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/q_b_proj" + ] }, { "type": "gemm", "args": { - "m": 144256, + "m": 1280, "n": 32768, "k": 512, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/kv_b_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/kv_b_proj" + ] }, { "type": "gemm", "args": { - "m": 144256, - "n": 7168, - "k": 16384, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 1536, + "n": 512, + "k": 2880, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/gpt-oss-120b-w-mxfp4-a-fp8/k_proj", + "amd/gpt-oss-120b-w-mxfp4-a-fp8/q_proj", + "amd/gpt-oss-120b-w-mxfp4-a-fp8/v_proj", + "openai/gpt-oss-120b/k_proj", + "openai/gpt-oss-120b/q_proj", + "openai/gpt-oss-120b/v_proj" + ] }, { "type": "gemm", "args": { - "m": 144256, - "n": 2304, + "m": 1536, + "n": 576, "k": 7168, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/Kimi-K2.5-MXFP4/kv_a_proj_with_mqa", + "moonshotai/Kimi-K2.5/kv_a_proj_with_mqa", + "nvidia/DeepSeek-R1-0528-FP4-V2/kv_a_proj_with_mqa", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/kv_a_proj_with_mqa", + "nvidia/Kimi-K2.5-NVFP4/kv_a_proj_with_mqa", + "nvidia/Kimi-K2.6-NVFP4/kv_a_proj_with_mqa" + ] }, { "type": "gemm", "args": { - "m": 144256, - "n": 7168, - "k": 2304, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 1536, + "n": 1536, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/Kimi-K2.5-MXFP4/q_a_proj", + "moonshotai/Kimi-K2.5/q_a_proj", + "nvidia/DeepSeek-R1-0528-FP4-V2/q_a_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/q_a_proj", + "nvidia/Kimi-K2.5-NVFP4/q_a_proj", + "nvidia/Kimi-K2.6-NVFP4/q_a_proj" + ] }, { "type": "gemm", "args": { - "m": 140, - "n": 1536, + "m": 1536, + "n": 2304, "k": 7168, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/gate_up_proj" + ] }, { "type": "gemm", "args": { - "m": 140, - "n": 24576, - "k": 1536, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 1536, + "n": 2880, + "k": 4096, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/gpt-oss-120b-w-mxfp4-a-fp8/o_proj", + "openai/gpt-oss-120b/o_proj" + ] }, { "type": "gemm", "args": { - "m": 140, - "n": 576, - "k": 7168, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 1536, + "n": 3072, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "MiniMaxAI/MiniMax-M2.5/experts.down_proj", + "MiniMaxAI/MiniMax-M2.5/o_proj", + "deepseek-ai/DeepSeek-R1-0528/q_b_proj" + ] }, { "type": "gemm", "args": { - "m": 140, - "n": 32768, + "m": 1536, + "n": 4096, "k": 512, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "Qwen/Qwen3.5-397B-A17B-FP8/o_proj", + "Qwen/Qwen3.5-397B-A17B-FP8/shared_expert.down_proj", + "deepseek-ai/DeepSeek-R1-0528/kv_b_proj" + ] }, { "type": "gemm", "args": { - "m": 140, - "n": 7168, - "k": 16384, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 1536, + "n": 4096, + "k": 2880, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/gpt-oss-120b-w-mxfp4-a-fp8/q_proj", + "openai/gpt-oss-120b/q_proj" + ] }, { "type": "gemm", "args": { - "m": 140, - "n": 2304, + "m": 1536, + "n": 4608, "k": 7168, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/gate_up_proj" + ] }, { "type": "gemm", "args": { - "m": 140, + "m": 1536, "n": 7168, - "k": 2304, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "k": 2048, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/o_proj", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] }, { "type": "gemm", "args": { - "m": 36064, - "n": 1536, - "k": 7168, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 1536, + "n": 7168, + "k": 2304, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/down_proj" + ] }, { "type": "gemm", "args": { - "m": 36064, - "n": 24576, - "k": 1536, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 1536, + "n": 7168, + "k": 4608, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/down_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/down_proj", + "nvidia/Kimi-K2.5-NVFP4/down_proj" + ] }, { "type": "gemm", "args": { - "m": 36064, - "n": 576, - "k": 7168, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 1536, + "n": 7168, + "k": 16384, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/o_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/o_proj" + ] }, { "type": "gemm", "args": { - "m": 36064, - "n": 32768, - "k": 512, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 1536, + "n": 24576, + "k": 1536, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/q_b_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/q_b_proj" + ] }, { "type": "gemm", "args": { - "m": 36064, - "n": 7168, - "k": 16384, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 1536, + "n": 32768, + "k": 512, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/kv_b_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/kv_b_proj" + ] }, { "type": "gemm", "args": { - "m": 36064, - "n": 7168, - "k": 576, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 1543, + "n": 1088, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] }, { "type": "gemm", "args": { - "m": 35, + "m": 1543, "n": 1536, "k": 7168, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 35, - "n": 24576, - "k": 1536, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/Kimi-K2.5-MXFP4/q_a_proj", + "moonshotai/Kimi-K2.5/q_a_proj", + "nvidia/DeepSeek-R1-0528-FP4-V2/q_a_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/q_a_proj", + "nvidia/Kimi-K2.5-NVFP4/q_a_proj", + "nvidia/Kimi-K2.6-NVFP4/q_a_proj" + ] }, { "type": "gemm", "args": { - "m": 35, - "n": 576, + "m": 1543, + "n": 1536, "k": 7168, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 32 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 1, + 32 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "amd/DeepSeek-R1-0528-MXFP4-Preview/q_a_proj", + "amd/DeepSeek-R1-0528-MXFP4-v2/q_a_proj", + "amd/DeepSeek-R1-0528-MXFP4/q_a_proj" + ] }, { "type": "gemm", "args": { - "m": 35, - "n": 32768, - "k": 512, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 1543, + "n": 1536, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w3" + ] }, { "type": "gemm", "args": { - "m": 35, + "m": 1543, "n": 7168, - "k": 16384, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "k": 65536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] }, { "type": "gemm", "args": { - "m": 35, - "n": 7168, - "k": 576, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 1543, + "n": 65536, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_b" + ] }, { "type": "gemm", "args": { - "m": 8192, - "n": 7168, - "k": 576, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 1543, + "n": 122880, + "k": 1024, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] }, { "type": "gemm", "args": { - "m": 8, - "n": 7168, - "k": 576, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 1613, + "n": 576, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/Kimi-K2.5-MXFP4/kv_a_proj_with_mqa", + "moonshotai/Kimi-K2.5/kv_a_proj_with_mqa", + "nvidia/DeepSeek-R1-0528-FP4-V2/kv_a_proj_with_mqa", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/kv_a_proj_with_mqa", + "nvidia/Kimi-K2.5-NVFP4/kv_a_proj_with_mqa", + "nvidia/Kimi-K2.6-NVFP4/kv_a_proj_with_mqa" + ] }, { "type": "gemm", "args": { - "m": 3456, + "m": 1613, "n": 1536, "k": 7168, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/Kimi-K2.5-MXFP4/q_a_proj", + "moonshotai/Kimi-K2.5/q_a_proj", + "nvidia/DeepSeek-R1-0528-FP4-V2/q_a_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/q_a_proj", + "nvidia/Kimi-K2.5-NVFP4/q_a_proj", + "nvidia/Kimi-K2.6-NVFP4/q_a_proj" + ] }, { "type": "gemm", "args": { - "m": 3456, - "n": 24576, - "k": 1536, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 1613, + "n": 4608, + "k": 7168, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/gate_up_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/gate_up_proj", + "nvidia/Kimi-K2.5-NVFP4/gate_up_proj" + ] }, { "type": "gemm", "args": { - "m": 3456, - "n": 576, + "m": 1613, + "n": 4608, "k": 7168, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 32 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 1, + 32 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "amd/DeepSeek-R1-0528-MXFP4-Preview/gate_up_proj", + "amd/DeepSeek-R1-0528-MXFP4-v2/gate_up_proj", + "amd/DeepSeek-R1-0528-MXFP4/gate_up_proj", + "amd/Kimi-K2.5-MXFP4/gate_up_proj" + ] }, { "type": "gemm", "args": { - "m": 3456, - "n": 32768, - "k": 512, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 1613, + "n": 7168, + "k": 4608, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/down_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/down_proj", + "nvidia/Kimi-K2.5-NVFP4/down_proj" + ] }, { "type": "gemm", "args": { - "m": 3456, + "m": 1613, "n": 7168, - "k": 16384, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "k": 8192, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/Kimi-K2.5-MXFP4/o_proj", + "moonshotai/Kimi-K2.5/o_proj", + "nvidia/Kimi-K2.5-NVFP4/o_proj", + "nvidia/Kimi-K2.6-NVFP4/o_proj" + ] }, { "type": "gemm", "args": { - "m": 3456, - "n": 2304, - "k": 7168, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 1613, + "n": 7168, + "k": 8192, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] }, { "type": "gemm", "args": { - "m": 3456, + "m": 1613, "n": 7168, - "k": 2304, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "k": 16384, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/o_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/o_proj" + ] }, { "type": "gemm", "args": { - "m": 3, - "n": 2304, - "k": 7168, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 1613, + "n": 12288, + "k": 1536, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/Kimi-K2.5-MXFP4/q_b_proj", + "moonshotai/Kimi-K2.5/q_b_proj", + "nvidia/DeepSeek-R1-0528-FP4-V2/q_b_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/q_b_proj", + "nvidia/Kimi-K2.5-NVFP4/q_b_proj", + "nvidia/Kimi-K2.6-NVFP4/q_b_proj" + ] }, { "type": "gemm", "args": { - "m": 3, - "n": 7168, - "k": 2304, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 1613, + "n": 16384, + "k": 512, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/Kimi-K2.5-MXFP4/kv_b_proj", + "moonshotai/Kimi-K2.5/kv_b_proj", + "nvidia/DeepSeek-R1-0528-FP4-V2/kv_b_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/kv_b_proj", + "nvidia/Kimi-K2.5-NVFP4/kv_b_proj", + "nvidia/Kimi-K2.6-NVFP4/kv_b_proj" + ] }, { "type": "gemm", "args": { - "m": 27648, - "n": 1536, - "k": 7168, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 1613, + "n": 24576, + "k": 1536, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/q_b_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/q_b_proj" + ] }, { "type": "gemm", "args": { - "m": 27648, - "n": 3072, - "k": 1536, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 1613, + "n": 32768, + "k": 512, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/kv_b_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/kv_b_proj" + ] }, { "type": "gemm", "args": { - "m": 27648, + "m": 1614, "n": 576, "k": 7168, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/Kimi-K2.5-MXFP4/kv_a_proj_with_mqa", + "moonshotai/Kimi-K2.5/kv_a_proj_with_mqa", + "nvidia/DeepSeek-R1-0528-FP4-V2/kv_a_proj_with_mqa", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/kv_a_proj_with_mqa", + "nvidia/Kimi-K2.5-NVFP4/kv_a_proj_with_mqa", + "nvidia/Kimi-K2.6-NVFP4/kv_a_proj_with_mqa" + ] }, { "type": "gemm", "args": { - "m": 27648, - "n": 4096, - "k": 512, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 1614, + "n": 576, + "k": 7168, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 32 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 1, + 32 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "amd/DeepSeek-R1-0528-MXFP4-Preview/kv_a_proj_with_mqa", + "amd/DeepSeek-R1-0528-MXFP4-v2/kv_a_proj_with_mqa", + "amd/DeepSeek-R1-0528-MXFP4/kv_a_proj_with_mqa" + ] }, { "type": "gemm", "args": { - "m": 27648, - "n": 7168, - "k": 2048, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 1614, + "n": 1536, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/Kimi-K2.5-MXFP4/q_a_proj", + "moonshotai/Kimi-K2.5/q_a_proj", + "nvidia/DeepSeek-R1-0528-FP4-V2/q_a_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/q_a_proj", + "nvidia/Kimi-K2.5-NVFP4/q_a_proj", + "nvidia/Kimi-K2.6-NVFP4/q_a_proj" + ] }, { "type": "gemm", "args": { - "m": 27648, - "n": 2304, + "m": 1614, + "n": 1536, "k": 7168, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 32 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 1, + 32 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "amd/DeepSeek-R1-0528-MXFP4-Preview/q_a_proj", + "amd/DeepSeek-R1-0528-MXFP4-v2/q_a_proj", + "amd/DeepSeek-R1-0528-MXFP4/q_a_proj" + ] }, { "type": "gemm", "args": { - "m": 27648, - "n": 7168, - "k": 2304, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 1614, + "n": 1536, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w3" + ] }, { "type": "gemm", "args": { - "m": 27, - "n": 1536, - "k": 7168, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 1614, + "n": 7168, + "k": 9216, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/down_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/down_proj", + "nvidia/Kimi-K2.5-NVFP4/down_proj" + ] }, { "type": "gemm", "args": { - "m": 27, - "n": 3072, - "k": 1536, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 1614, + "n": 7168, + "k": 16384, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/o_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/o_proj" + ] }, { "type": "gemm", "args": { - "m": 27, - "n": 576, - "k": 7168, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 1614, + "n": 7168, + "k": 16384, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 32 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 1, + 32 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "amd/DeepSeek-R1-0528-MXFP4-Preview/o_proj", + "amd/DeepSeek-R1-0528-MXFP4-v2/o_proj", + "amd/DeepSeek-R1-0528-MXFP4/o_proj" + ] }, { "type": "gemm", "args": { - "m": 27, - "n": 4096, - "k": 512, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 1614, + "n": 7168, + "k": 16384, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] }, { "type": "gemm", "args": { - "m": 27, - "n": 7168, - "k": 2048, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 1614, + "n": 9216, + "k": 7168, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/gate_up_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/gate_up_proj", + "nvidia/Kimi-K2.5-NVFP4/gate_up_proj" + ] }, { "type": "gemm", "args": { - "m": 27, - "n": 2304, + "m": 1614, + "n": 9216, "k": 7168, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 32 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 1, + 32 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "amd/DeepSeek-R1-0528-MXFP4-Preview/gate_up_proj", + "amd/DeepSeek-R1-0528-MXFP4-v2/gate_up_proj", + "amd/DeepSeek-R1-0528-MXFP4/gate_up_proj", + "amd/Kimi-K2.5-MXFP4/gate_up_proj" + ] }, { "type": "gemm", "args": { - "m": 27, - "n": 7168, - "k": 2304, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 1614, + "n": 24576, + "k": 1536, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/q_b_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/q_b_proj" + ] }, { "type": "gemm", "args": { - "m": 3072, - "n": 3072, + "m": 1614, + "n": 24576, "k": 1536, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 32 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 1, + 32 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "amd/DeepSeek-R1-0528-MXFP4-Preview/q_b_proj", + "amd/DeepSeek-R1-0528-MXFP4-v2/q_b_proj", + "amd/DeepSeek-R1-0528-MXFP4/q_b_proj" + ] }, { "type": "gemm", "args": { - "m": 3072, - "n": 4096, + "m": 1614, + "n": 32768, "k": 512, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/kv_b_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/kv_b_proj" + ] }, { "type": "gemm", "args": { - "m": 3072, - "n": 7168, - "k": 2048, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 1614, + "n": 32768, + "k": 512, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 32 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 1, + 32 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "amd/DeepSeek-R1-0528-MXFP4-Preview/kv_b_proj", + "amd/DeepSeek-R1-0528-MXFP4-v2/kv_b_proj", + "amd/DeepSeek-R1-0528-MXFP4/kv_b_proj" + ] }, { "type": "gemm", "args": { - "m": 3072, - "n": 2304, + "m": 1792, + "n": 1088, "k": 7168, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] }, { "type": "gemm", "args": { - "m": 3072, - "n": 7168, - "k": 2304, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 1792, + "n": 1536, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w3" + ] }, { "type": "gemm", "args": { - "m": 3, - "n": 3072, - "k": 1536, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 1792, + "n": 7168, + "k": 65536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] }, { "type": "gemm", "args": { - "m": 3, - "n": 4096, - "k": 512, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 1792, + "n": 65536, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_b" + ] }, { "type": "gemm", "args": { - "m": 3, - "n": 7168, - "k": 2048, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 1792, + "n": 122880, + "k": 1024, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] }, { "type": "gemm", "args": { - "m": 681984, - "n": 1536, + "m": 1920, + "n": 576, "k": 7168, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/kv_a_proj_with_mqa" + ] }, { "type": "gemm", "args": { - "m": 681984, - "n": 24576, - "k": 1536, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 1920, + "n": 1152, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/gate_up_proj" + ] }, { "type": "gemm", "args": { - "m": 681984, - "n": 576, + "m": 1920, + "n": 1536, "k": 7168, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/gate_up_proj", + "deepseek-ai/DeepSeek-R1-0528/q_a_proj", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w3" + ] }, { "type": "gemm", "args": { - "m": 681984, - "n": 32768, - "k": 512, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 1920, + "n": 7168, + "k": 1152, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/down_proj" + ] }, { "type": "gemm", "args": { - "m": 681984, + "m": 1920, "n": 7168, "k": 16384, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/o_proj", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] }, { "type": "gemm", "args": { - "m": 681984, - "n": 2304, - "k": 7168, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 1920, + "n": 24576, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/q_b_proj" + ] }, { "type": "gemm", "args": { - "m": 681984, - "n": 7168, - "k": 2304, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 1920, + "n": 32768, + "k": 512, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/kv_b_proj" + ] }, { "type": "gemm", "args": { - "m": 166, - "n": 1536, - "k": 7168, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 2048, + "n": 64, + "k": 2880, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/gpt-oss-120b-w-mxfp4-a-fp8/k_proj", + "amd/gpt-oss-120b-w-mxfp4-a-fp8/v_proj", + "openai/gpt-oss-120b/k_proj", + "openai/gpt-oss-120b/v_proj" + ] }, { "type": "gemm", "args": { - "m": 166, - "n": 24576, - "k": 1536, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 2048, + "n": 128, + "k": 2880, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/gpt-oss-120b-w-mxfp4-a-fp8/k_proj", + "amd/gpt-oss-120b-w-mxfp4-a-fp8/v_proj", + "openai/gpt-oss-120b/k_proj", + "openai/gpt-oss-120b/v_proj" + ] }, { "type": "gemm", "args": { - "m": 166, - "n": 576, - "k": 7168, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 2048, + "n": 128, + "k": 3072, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "MiniMaxAI/MiniMax-M2.5/k_proj", + "MiniMaxAI/MiniMax-M2.5/v_proj" + ] }, { "type": "gemm", "args": { - "m": 166, - "n": 32768, - "k": 512, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 2048, + "n": 192, + "k": 3072, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "MiniMaxAI/MiniMax-M2.5/experts.gate_up_proj" + ] }, { "type": "gemm", "args": { - "m": 166, - "n": 7168, - "k": 16384, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 2048, + "n": 512, + "k": 2880, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/gpt-oss-120b-w-mxfp4-a-fp8/k_proj", + "amd/gpt-oss-120b-w-mxfp4-a-fp8/q_proj", + "amd/gpt-oss-120b-w-mxfp4-a-fp8/v_proj", + "openai/gpt-oss-120b/k_proj", + "openai/gpt-oss-120b/q_proj", + "openai/gpt-oss-120b/v_proj" + ] }, { "type": "gemm", "args": { - "m": 166, - "n": 2304, - "k": 7168, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 2048, + "n": 512, + "k": 3072, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "MiniMaxAI/MiniMax-M2.5/k_proj", + "MiniMaxAI/MiniMax-M2.5/v_proj" + ] }, { "type": "gemm", "args": { - "m": 166, - "n": 7168, - "k": 2304, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 2048, + "n": 576, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/Kimi-K2.5-MXFP4/kv_a_proj_with_mqa", + "moonshotai/Kimi-K2.5/kv_a_proj_with_mqa", + "nvidia/DeepSeek-R1-0528-FP4-V2/kv_a_proj_with_mqa", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/kv_a_proj_with_mqa", + "nvidia/Kimi-K2.5-NVFP4/kv_a_proj_with_mqa", + "nvidia/Kimi-K2.6-NVFP4/kv_a_proj_with_mqa" + ] }, { "type": "gemm", "args": { - "m": 340992, - "n": 1536, + "m": 2048, + "n": 576, "k": 7168, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 32 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 1, + 32 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "amd/DeepSeek-R1-0528-MXFP4-Preview/kv_a_proj_with_mqa", + "amd/DeepSeek-R1-0528-MXFP4-v2/kv_a_proj_with_mqa", + "amd/DeepSeek-R1-0528-MXFP4/kv_a_proj_with_mqa" + ] }, { "type": "gemm", "args": { - "m": 340992, - "n": 24576, - "k": 1536, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 2048, + "n": 576, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/kv_a_proj_with_mqa" + ] }, { "type": "gemm", "args": { - "m": 340992, - "n": 576, - "k": 7168, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 2048, + "n": 768, + "k": 3072, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "MiniMaxAI/MiniMax-M2.5/experts.gate_up_proj", + "MiniMaxAI/MiniMax-M2.5/q_proj" + ] }, { "type": "gemm", "args": { - "m": 340992, - "n": 32768, - "k": 512, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 2048, + "n": 1024, + "k": 2880, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/gpt-oss-120b-w-mxfp4-a-fp8/q_proj", + "openai/gpt-oss-120b/q_proj" + ] }, { "type": "gemm", "args": { - "m": 340992, - "n": 7168, - "k": 16384, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 2048, + "n": 1088, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] }, { "type": "gemm", "args": { - "m": 340992, + "m": 2048, "n": 1152, "k": 7168, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/gate_up_proj" + ] }, { "type": "gemm", "args": { - "m": 340992, - "n": 7168, - "k": 1152, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 2048, + "n": 1536, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/q_b_proj" + ] }, { "type": "gemm", "args": { - "m": 83, + "m": 2048, "n": 1536, "k": 7168, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/Kimi-K2.5-MXFP4/q_a_proj", + "moonshotai/Kimi-K2.5/q_a_proj", + "nvidia/DeepSeek-R1-0528-FP4-V2/q_a_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/q_a_proj", + "nvidia/Kimi-K2.5-NVFP4/q_a_proj", + "nvidia/Kimi-K2.6-NVFP4/q_a_proj" + ] }, { "type": "gemm", "args": { - "m": 83, - "n": 24576, - "k": 1536, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 2048, + "n": 1536, + "k": 7168, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 32 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 1, + 32 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "amd/DeepSeek-R1-0528-MXFP4-Preview/q_a_proj", + "amd/DeepSeek-R1-0528-MXFP4-v2/q_a_proj", + "amd/DeepSeek-R1-0528-MXFP4/q_a_proj" + ] }, { "type": "gemm", "args": { - "m": 83, - "n": 576, + "m": 2048, + "n": 1536, "k": 7168, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/gate_up_proj", + "deepseek-ai/DeepSeek-R1-0528/q_a_proj", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w3" + ] }, { "type": "gemm", "args": { - "m": 83, - "n": 32768, - "k": 512, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 2048, + "n": 1536, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w3" + ] }, { "type": "gemm", "args": { - "m": 83, - "n": 7168, - "k": 16384, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 2048, + "n": 2048, + "k": 512, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/kv_b_proj" + ] }, { "type": "gemm", "args": { - "m": 83, - "n": 1152, + "m": 2048, + "n": 2304, "k": 7168, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/gate_up_proj" + ] }, { "type": "gemm", "args": { - "m": 83, - "n": 7168, - "k": 1152, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 2048, + "n": 2880, + "k": 512, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/gpt-oss-120b-w-mxfp4-a-fp8/o_proj", + "openai/gpt-oss-120b/o_proj" + ] }, { "type": "gemm", "args": { - "m": 340992, - "n": 2304, - "k": 7168, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 2048, + "n": 2880, + "k": 1024, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/gpt-oss-120b-w-mxfp4-a-fp8/o_proj", + "openai/gpt-oss-120b/o_proj" + ] }, { "type": "gemm", "args": { - "m": 340992, - "n": 7168, - "k": 2304, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 2048, + "n": 2880, + "k": 4096, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/gpt-oss-120b-w-mxfp4-a-fp8/o_proj", + "openai/gpt-oss-120b/o_proj" + ] }, { "type": "gemm", "args": { - "m": 83, - "n": 2304, - "k": 7168, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 2048, + "n": 3072, + "k": 192, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "MiniMaxAI/MiniMax-M2.5/experts.down_proj" + ] }, { "type": "gemm", "args": { - "m": 83, - "n": 7168, - "k": 2304, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 2048, + "n": 3072, + "k": 768, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "MiniMaxAI/MiniMax-M2.5/experts.down_proj", + "MiniMaxAI/MiniMax-M2.5/o_proj" + ] }, { "type": "gemm", "args": { - "m": 170496, - "n": 1536, - "k": 7168, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 2048, + "n": 3072, + "k": 1536, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 32 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 1, + 32 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "amd/DeepSeek-R1-0528-MXFP4-Preview/q_b_proj", + "amd/DeepSeek-R1-0528-MXFP4-v2/q_b_proj", + "amd/DeepSeek-R1-0528-MXFP4/q_b_proj", + "amd/MiniMax-M2.5-MXFP4/experts.down_proj" + ] }, { "type": "gemm", "args": { - "m": 170496, - "n": 24576, - "k": 1536, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 2048, + "n": 3072, + "k": 3072, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "MiniMaxAI/MiniMax-M2.5/experts.gate_up_proj", + "MiniMaxAI/MiniMax-M2.5/o_proj", + "MiniMaxAI/MiniMax-M2.5/q_proj" + ] }, { "type": "gemm", "args": { - "m": 170496, - "n": 576, - "k": 7168, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 2048, + "n": 4096, + "k": 512, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 32 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 1, + 32 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "amd/DeepSeek-R1-0528-MXFP4-Preview/kv_b_proj", + "amd/DeepSeek-R1-0528-MXFP4-v2/kv_b_proj", + "amd/DeepSeek-R1-0528-MXFP4/kv_b_proj", + "amd/Qwen3.5-397B-A17B-MXFP4-AttnFP8-V2/shared_expert.down_proj" + ] }, { "type": "gemm", "args": { - "m": 170496, - "n": 32768, - "k": 512, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 2048, + "n": 4096, + "k": 2880, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/gpt-oss-120b-w-mxfp4-a-fp8/q_proj", + "openai/gpt-oss-120b/q_proj" + ] }, { "type": "gemm", "args": { - "m": 170496, - "n": 7168, - "k": 16384, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 2048, + "n": 4608, + "k": 7168, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 32 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 1, + 32 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "amd/DeepSeek-R1-0528-MXFP4-Preview/gate_up_proj", + "amd/DeepSeek-R1-0528-MXFP4-v2/gate_up_proj", + "amd/DeepSeek-R1-0528-MXFP4/gate_up_proj", + "amd/Kimi-K2.5-MXFP4/gate_up_proj" + ] }, { "type": "gemm", "args": { - "m": 170496, - "n": 1152, + "m": 2048, + "n": 4608, "k": 7168, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/gate_up_proj" + ] }, { "type": "gemm", "args": { - "m": 170496, - "n": 7168, - "k": 1152, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 2048, + "n": 6144, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/q_b_proj", + "zai-org/GLM-5-FP8/down_proj", + "zai-org/GLM-5.1-FP8/down_proj", + "zai-org/GLM-5.2-FP8/down_proj" + ] }, { "type": "gemm", "args": { - "m": 41, - "n": 1536, - "k": 7168, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 2048, + "n": 7168, + "k": 1024, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/o_proj", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] }, { "type": "gemm", "args": { - "m": 41, - "n": 24576, - "k": 1536, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 2048, + "n": 7168, + "k": 1152, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/down_proj" + ] }, { "type": "gemm", "args": { - "m": 41, - "n": 576, - "k": 7168, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 2048, + "n": 7168, + "k": 2048, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 32 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 1, + 32 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "amd/DeepSeek-R1-0528-MXFP4-Preview/o_proj", + "amd/DeepSeek-R1-0528-MXFP4-v2/o_proj", + "amd/DeepSeek-R1-0528-MXFP4/o_proj" + ] }, { "type": "gemm", "args": { - "m": 41, - "n": 32768, - "k": 512, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 2048, + "n": 7168, + "k": 2304, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 32 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 1, + 32 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "amd/DeepSeek-R1-0528-MXFP4-Preview/down_proj", + "amd/DeepSeek-R1-0528-MXFP4-v2/down_proj", + "amd/DeepSeek-R1-0528-MXFP4/down_proj", + "amd/Kimi-K2.5-MXFP4/down_proj" + ] }, { "type": "gemm", "args": { - "m": 41, + "m": 2048, "n": 7168, - "k": 16384, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "k": 2304, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/down_proj" + ] }, { "type": "gemm", "args": { - "m": 41, - "n": 1152, - "k": 7168, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 2048, + "n": 7168, + "k": 4096, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/o_proj", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] }, { "type": "gemm", "args": { - "m": 41, + "m": 2048, "n": 7168, - "k": 1152, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "k": 4608, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/down_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/down_proj", + "nvidia/Kimi-K2.5-NVFP4/down_proj" + ] }, { "type": "gemm", "args": { - "m": 85248, - "n": 1536, - "k": 7168, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 2048, + "n": 7168, + "k": 4608, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/down_proj" + ] }, { "type": "gemm", "args": { - "m": 85248, - "n": 24576, - "k": 1536, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 2048, + "n": 7168, + "k": 8192, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/Kimi-K2.5-MXFP4/o_proj", + "moonshotai/Kimi-K2.5/o_proj", + "nvidia/Kimi-K2.5-NVFP4/o_proj", + "nvidia/Kimi-K2.6-NVFP4/o_proj" + ] }, { "type": "gemm", "args": { - "m": 85248, - "n": 576, - "k": 7168, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 2048, + "n": 7168, + "k": 8192, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] }, { "type": "gemm", "args": { - "m": 85248, - "n": 32768, - "k": 512, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 2048, + "n": 7168, + "k": 16384, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/o_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/o_proj" + ] }, { "type": "gemm", "args": { - "m": 85248, + "m": 2048, "n": 7168, "k": 16384, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/o_proj", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] }, { "type": "gemm", "args": { - "m": 85248, + "m": 2048, "n": 7168, - "k": 576, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "k": 16384, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] }, { "type": "gemm", "args": { - "m": 20, + "m": 2048, "n": 7168, - "k": 576, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "k": 65536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] }, { "type": "gemm", "args": { - "m": 22, - "n": 1536, - "k": 7168, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 2048, + "n": 8192, + "k": 512, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/kv_b_proj" + ] }, { "type": "gemm", "args": { - "m": 22, - "n": 24576, + "m": 2048, + "n": 8192, "k": 1536, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_b" + ] }, { "type": "gemm", "args": { - "m": 22, - "n": 576, - "k": 7168, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 2048, + "n": 12288, + "k": 1536, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/Kimi-K2.5-MXFP4/q_b_proj", + "moonshotai/Kimi-K2.5/q_b_proj", + "nvidia/DeepSeek-R1-0528-FP4-V2/q_b_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/q_b_proj", + "nvidia/Kimi-K2.5-NVFP4/q_b_proj", + "nvidia/Kimi-K2.6-NVFP4/q_b_proj" + ] }, { "type": "gemm", "args": { - "m": 22, - "n": 32768, - "k": 512, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 2048, + "n": 15360, + "k": 1024, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] }, { "type": "gemm", "args": { - "m": 22, - "n": 7168, - "k": 16384, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 2048, + "n": 16384, + "k": 512, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/Kimi-K2.5-MXFP4/kv_b_proj", + "moonshotai/Kimi-K2.5/kv_b_proj", + "nvidia/DeepSeek-R1-0528-FP4-V2/kv_b_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/kv_b_proj", + "nvidia/Kimi-K2.5-NVFP4/kv_b_proj", + "nvidia/Kimi-K2.6-NVFP4/kv_b_proj" + ] }, { "type": "gemm", "args": { - "m": 22, - "n": 2304, - "k": 7168, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 2048, + "n": 16384, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_b" + ] }, { "type": "gemm", "args": { - "m": 22, - "n": 7168, - "k": 2304, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 2048, + "n": 24576, + "k": 1536, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/q_b_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/q_b_proj" + ] }, { "type": "gemm", "args": { - "m": 23040, - "n": 1536, - "k": 7168, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 2048, + "n": 24576, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/q_b_proj" + ] }, { "type": "gemm", "args": { - "m": 23040, - "n": 24576, - "k": 1536, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 2048, + "n": 30720, + "k": 1024, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] }, { "type": "gemm", "args": { - "m": 23040, - "n": 576, - "k": 7168, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 2048, + "n": 32768, + "k": 512, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/kv_b_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/kv_b_proj" + ] }, { "type": "gemm", "args": { - "m": 23040, + "m": 2048, "n": 32768, "k": 512, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/kv_b_proj" + ] }, { "type": "gemm", "args": { - "m": 23040, - "n": 7168, - "k": 16384, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 2048, + "n": 65536, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_b" + ] }, { "type": "gemm", "args": { - "m": 23040, - "n": 7168, - "k": 576, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 2048, + "n": 122880, + "k": 1024, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] }, { "type": "gemm", "args": { - "m": 5, - "n": 7168, - "k": 576, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 2150, + "n": 576, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/Kimi-K2.5-MXFP4/kv_a_proj_with_mqa", + "moonshotai/Kimi-K2.5/kv_a_proj_with_mqa", + "nvidia/DeepSeek-R1-0528-FP4-V2/kv_a_proj_with_mqa", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/kv_a_proj_with_mqa", + "nvidia/Kimi-K2.5-NVFP4/kv_a_proj_with_mqa", + "nvidia/Kimi-K2.6-NVFP4/kv_a_proj_with_mqa" + ] }, { "type": "gemm", "args": { - "m": 15360, - "n": 2304, + "m": 2150, + "n": 576, "k": 7168, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 32 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 1, + 32 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "amd/DeepSeek-R1-0528-MXFP4-Preview/kv_a_proj_with_mqa", + "amd/DeepSeek-R1-0528-MXFP4-v2/kv_a_proj_with_mqa", + "amd/DeepSeek-R1-0528-MXFP4/kv_a_proj_with_mqa" + ] }, { "type": "gemm", "args": { - "m": 15360, - "n": 7168, - "k": 2304, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 2150, + "n": 576, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/kv_a_proj_with_mqa" + ] }, { "type": "gemm", "args": { - "m": 122880, + "m": 2150, "n": 1536, "k": 7168, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/Kimi-K2.5-MXFP4/q_a_proj", + "moonshotai/Kimi-K2.5/q_a_proj", + "nvidia/DeepSeek-R1-0528-FP4-V2/q_a_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/q_a_proj", + "nvidia/Kimi-K2.5-NVFP4/q_a_proj", + "nvidia/Kimi-K2.6-NVFP4/q_a_proj" + ] }, { "type": "gemm", "args": { - "m": 122880, - "n": 3072, - "k": 1536, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 2150, + "n": 1536, + "k": 7168, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 32 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 1, + 32 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "amd/DeepSeek-R1-0528-MXFP4-Preview/q_a_proj", + "amd/DeepSeek-R1-0528-MXFP4-v2/q_a_proj", + "amd/DeepSeek-R1-0528-MXFP4/q_a_proj" + ] }, { "type": "gemm", "args": { - "m": 122880, - "n": 576, + "m": 2150, + "n": 1536, "k": 7168, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/gate_up_proj", + "deepseek-ai/DeepSeek-R1-0528/q_a_proj", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w3" + ] }, { "type": "gemm", "args": { - "m": 122880, - "n": 4096, - "k": 512, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 2150, + "n": 1536, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w3" + ] }, { "type": "gemm", "args": { - "m": 122880, - "n": 7168, - "k": 2048, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 2150, + "n": 4608, + "k": 7168, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/gate_up_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/gate_up_proj", + "nvidia/Kimi-K2.5-NVFP4/gate_up_proj" + ] }, { "type": "gemm", "args": { - "m": 122880, - "n": 2304, + "m": 2150, + "n": 4608, "k": 7168, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 32 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 1, + 32 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "amd/DeepSeek-R1-0528-MXFP4-Preview/gate_up_proj", + "amd/DeepSeek-R1-0528-MXFP4-v2/gate_up_proj", + "amd/DeepSeek-R1-0528-MXFP4/gate_up_proj", + "amd/Kimi-K2.5-MXFP4/gate_up_proj" + ] }, { "type": "gemm", "args": { - "m": 122880, - "n": 7168, - "k": 2304, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 2150, + "n": 4608, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/gate_up_proj" + ] }, { "type": "gemm", "args": { - "m": 30, - "n": 3072, - "k": 1536, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 2150, + "n": 7168, + "k": 4608, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/down_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/down_proj", + "nvidia/Kimi-K2.5-NVFP4/down_proj" + ] }, { "type": "gemm", "args": { - "m": 30, - "n": 4096, - "k": 512, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 2150, + "n": 7168, + "k": 4608, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 32 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 1, + 32 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "amd/DeepSeek-R1-0528-MXFP4-Preview/down_proj", + "amd/DeepSeek-R1-0528-MXFP4-v2/down_proj", + "amd/DeepSeek-R1-0528-MXFP4/down_proj", + "amd/Kimi-K2.5-MXFP4/down_proj" + ] }, { "type": "gemm", "args": { - "m": 30, + "m": 2150, "n": 7168, - "k": 2048, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "k": 4608, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/down_proj" + ] }, { "type": "gemm", "args": { - "m": 30, - "n": 2304, - "k": 7168, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 2150, + "n": 7168, + "k": 9216, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/down_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/down_proj", + "nvidia/Kimi-K2.5-NVFP4/down_proj" + ] }, { "type": "gemm", "args": { - "m": 30, + "m": 2150, "n": 7168, - "k": 2304, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "k": 16384, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/o_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/o_proj" + ] }, { "type": "gemm", "args": { - "m": 6144, - "n": 24576, - "k": 1536, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 2150, + "n": 7168, + "k": 16384, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 32 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 1, + 32 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "amd/DeepSeek-R1-0528-MXFP4-Preview/o_proj", + "amd/DeepSeek-R1-0528-MXFP4-v2/o_proj", + "amd/DeepSeek-R1-0528-MXFP4/o_proj" + ] }, { "type": "gemm", "args": { - "m": 6144, - "n": 32768, - "k": 512, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 2150, + "n": 7168, + "k": 16384, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/o_proj", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] }, { "type": "gemm", "args": { - "m": 6144, + "m": 2150, "n": 7168, "k": 16384, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] }, { "type": "gemm", "args": { - "m": 6144, - "n": 2304, + "m": 2150, + "n": 9216, "k": 7168, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/gate_up_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/gate_up_proj", + "nvidia/Kimi-K2.5-NVFP4/gate_up_proj" + ] }, { "type": "gemm", "args": { - "m": 6144, - "n": 7168, - "k": 2304, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 2150, + "n": 9216, + "k": 7168, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 32 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 1, + 32 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "amd/DeepSeek-R1-0528-MXFP4-Preview/gate_up_proj", + "amd/DeepSeek-R1-0528-MXFP4-v2/gate_up_proj", + "amd/DeepSeek-R1-0528-MXFP4/gate_up_proj", + "amd/Kimi-K2.5-MXFP4/gate_up_proj" + ] }, { "type": "gemm", "args": { - "m": 49152, - "n": 3072, + "m": 2150, + "n": 24576, "k": 1536, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/q_b_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/q_b_proj" + ] }, { "type": "gemm", "args": { - "m": 49152, - "n": 4096, - "k": 512, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 2150, + "n": 24576, + "k": 1536, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 32 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 1, + 32 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "amd/DeepSeek-R1-0528-MXFP4-Preview/q_b_proj", + "amd/DeepSeek-R1-0528-MXFP4-v2/q_b_proj", + "amd/DeepSeek-R1-0528-MXFP4/q_b_proj" + ] }, { "type": "gemm", "args": { - "m": 49152, - "n": 7168, - "k": 2048, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 2150, + "n": 24576, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/q_b_proj" + ] }, { "type": "gemm", "args": { - "m": 1258496, - "n": 1536, - "k": 7168, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 2150, + "n": 32768, + "k": 512, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/kv_b_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/kv_b_proj" + ] }, { "type": "gemm", "args": { - "m": 1258496, - "n": 24576, - "k": 1536, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 2150, + "n": 32768, + "k": 512, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 32 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 1, + 32 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "amd/DeepSeek-R1-0528-MXFP4-Preview/kv_b_proj", + "amd/DeepSeek-R1-0528-MXFP4-v2/kv_b_proj", + "amd/DeepSeek-R1-0528-MXFP4/kv_b_proj" + ] }, { "type": "gemm", "args": { - "m": 1258496, - "n": 576, - "k": 7168, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 2150, + "n": 32768, + "k": 512, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/kv_b_proj" + ] }, { "type": "gemm", "args": { - "m": 1258496, - "n": 32768, - "k": 512, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 2152, + "n": 576, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/Kimi-K2.5-MXFP4/kv_a_proj_with_mqa", + "moonshotai/Kimi-K2.5/kv_a_proj_with_mqa", + "nvidia/DeepSeek-R1-0528-FP4-V2/kv_a_proj_with_mqa", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/kv_a_proj_with_mqa", + "nvidia/Kimi-K2.5-NVFP4/kv_a_proj_with_mqa", + "nvidia/Kimi-K2.6-NVFP4/kv_a_proj_with_mqa" + ] }, { "type": "gemm", "args": { - "m": 1258496, - "n": 7168, - "k": 16384, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 2152, + "n": 1536, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/Kimi-K2.5-MXFP4/q_a_proj", + "moonshotai/Kimi-K2.5/q_a_proj", + "nvidia/DeepSeek-R1-0528-FP4-V2/q_a_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/q_a_proj", + "nvidia/Kimi-K2.5-NVFP4/q_a_proj", + "nvidia/Kimi-K2.6-NVFP4/q_a_proj" + ] }, { "type": "gemm", "args": { - "m": 1258496, + "m": 2152, "n": 2304, "k": 7168, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/gate_up_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/gate_up_proj", + "nvidia/Kimi-K2.5-NVFP4/gate_up_proj" + ] }, { "type": "gemm", "args": { - "m": 1258496, + "m": 2152, "n": 7168, "k": 2304, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/down_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/down_proj", + "nvidia/Kimi-K2.5-NVFP4/down_proj" + ] }, { "type": "gemm", "args": { - "m": 629248, - "n": 1536, - "k": 7168, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 2152, + "n": 7168, + "k": 16384, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/o_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/o_proj" + ] }, { "type": "gemm", "args": { - "m": 629248, + "m": 2152, "n": 24576, "k": 1536, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 629248, - "n": 576, - "k": 7168, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/q_b_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/q_b_proj" + ] }, { "type": "gemm", "args": { - "m": 629248, + "m": 2152, "n": 32768, "k": 512, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/kv_b_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/kv_b_proj" + ] }, { "type": "gemm", "args": { - "m": 629248, - "n": 7168, - "k": 16384, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 2198, + "n": 576, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/Kimi-K2.5-MXFP4/kv_a_proj_with_mqa", + "moonshotai/Kimi-K2.5/kv_a_proj_with_mqa", + "nvidia/DeepSeek-R1-0528-FP4-V2/kv_a_proj_with_mqa", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/kv_a_proj_with_mqa", + "nvidia/Kimi-K2.5-NVFP4/kv_a_proj_with_mqa", + "nvidia/Kimi-K2.6-NVFP4/kv_a_proj_with_mqa" + ] }, { "type": "gemm", "args": { - "m": 629248, - "n": 1152, + "m": 2198, + "n": 576, "k": 7168, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 32 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 1, + 32 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "amd/DeepSeek-R1-0528-MXFP4-Preview/kv_a_proj_with_mqa", + "amd/DeepSeek-R1-0528-MXFP4-v2/kv_a_proj_with_mqa", + "amd/DeepSeek-R1-0528-MXFP4/kv_a_proj_with_mqa" + ] }, { "type": "gemm", "args": { - "m": 629248, - "n": 7168, - "k": 1152, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 2198, + "n": 1536, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/Kimi-K2.5-MXFP4/q_a_proj", + "moonshotai/Kimi-K2.5/q_a_proj", + "nvidia/DeepSeek-R1-0528-FP4-V2/q_a_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/q_a_proj", + "nvidia/Kimi-K2.5-NVFP4/q_a_proj", + "nvidia/Kimi-K2.6-NVFP4/q_a_proj" + ] }, { "type": "gemm", "args": { - "m": 170496, - "n": 7168, - "k": 576, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 2198, + "n": 1536, + "k": 7168, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 32 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 1, + 32 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "amd/DeepSeek-R1-0528-MXFP4-Preview/q_a_proj", + "amd/DeepSeek-R1-0528-MXFP4-v2/q_a_proj", + "amd/DeepSeek-R1-0528-MXFP4/q_a_proj" + ] }, { "type": "gemm", "args": { - "m": 629760, + "m": 2198, "n": 1536, "k": 7168, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w3" + ] }, { "type": "gemm", "args": { - "m": 629760, - "n": 24576, - "k": 1536, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 2198, + "n": 7168, + "k": 9216, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/down_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/down_proj", + "nvidia/Kimi-K2.5-NVFP4/down_proj" + ] }, { "type": "gemm", "args": { - "m": 629760, - "n": 576, - "k": 7168, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 2198, + "n": 7168, + "k": 16384, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/o_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/o_proj" + ] }, { "type": "gemm", "args": { - "m": 629760, - "n": 32768, - "k": 512, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 2198, + "n": 7168, + "k": 16384, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 32 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 1, + 32 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "amd/DeepSeek-R1-0528-MXFP4-Preview/o_proj", + "amd/DeepSeek-R1-0528-MXFP4-v2/o_proj", + "amd/DeepSeek-R1-0528-MXFP4/o_proj" + ] }, { "type": "gemm", "args": { - "m": 629760, + "m": 2198, "n": 7168, "k": 16384, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] }, { "type": "gemm", "args": { - "m": 629760, - "n": 2304, + "m": 2198, + "n": 9216, "k": 7168, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/gate_up_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/gate_up_proj", + "nvidia/Kimi-K2.5-NVFP4/gate_up_proj" + ] }, { "type": "gemm", "args": { - "m": 629760, - "n": 7168, - "k": 2304, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 2198, + "n": 9216, + "k": 7168, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 32 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 1, + 32 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "amd/DeepSeek-R1-0528-MXFP4-Preview/gate_up_proj", + "amd/DeepSeek-R1-0528-MXFP4-v2/gate_up_proj", + "amd/DeepSeek-R1-0528-MXFP4/gate_up_proj", + "amd/Kimi-K2.5-MXFP4/gate_up_proj" + ] }, { "type": "gemm", "args": { - "m": 76, - "n": 2304, - "k": 7168, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 2198, + "n": 24576, + "k": 1536, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/q_b_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/q_b_proj" + ] }, { "type": "gemm", "args": { - "m": 76, - "n": 7168, - "k": 2304, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 2198, + "n": 24576, + "k": 1536, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 32 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 1, + 32 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "amd/DeepSeek-R1-0528-MXFP4-Preview/q_b_proj", + "amd/DeepSeek-R1-0528-MXFP4-v2/q_b_proj", + "amd/DeepSeek-R1-0528-MXFP4/q_b_proj" + ] }, { "type": "gemm", "args": { - "m": 314880, - "n": 1536, - "k": 7168, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 2198, + "n": 32768, + "k": 512, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/kv_b_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/kv_b_proj" + ] }, { "type": "gemm", "args": { - "m": 314880, - "n": 24576, - "k": 1536, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 2198, + "n": 32768, + "k": 512, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 32 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 1, + 32 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "amd/DeepSeek-R1-0528-MXFP4-Preview/kv_b_proj", + "amd/DeepSeek-R1-0528-MXFP4-v2/kv_b_proj", + "amd/DeepSeek-R1-0528-MXFP4/kv_b_proj" + ] }, { "type": "gemm", "args": { - "m": 314880, + "m": 2202, "n": 576, "k": 7168, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/Kimi-K2.5-MXFP4/kv_a_proj_with_mqa", + "moonshotai/Kimi-K2.5/kv_a_proj_with_mqa", + "nvidia/DeepSeek-R1-0528-FP4-V2/kv_a_proj_with_mqa", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/kv_a_proj_with_mqa", + "nvidia/Kimi-K2.5-NVFP4/kv_a_proj_with_mqa", + "nvidia/Kimi-K2.6-NVFP4/kv_a_proj_with_mqa" + ] }, { "type": "gemm", "args": { - "m": 314880, - "n": 32768, - "k": 512, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 2202, + "n": 576, + "k": 7168, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 32 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 1, + 32 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "amd/DeepSeek-R1-0528-MXFP4-Preview/kv_a_proj_with_mqa", + "amd/DeepSeek-R1-0528-MXFP4-v2/kv_a_proj_with_mqa", + "amd/DeepSeek-R1-0528-MXFP4/kv_a_proj_with_mqa" + ] }, { "type": "gemm", "args": { - "m": 314880, - "n": 7168, - "k": 16384, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 2202, + "n": 1536, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/Kimi-K2.5-MXFP4/q_a_proj", + "moonshotai/Kimi-K2.5/q_a_proj", + "nvidia/DeepSeek-R1-0528-FP4-V2/q_a_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/q_a_proj", + "nvidia/Kimi-K2.5-NVFP4/q_a_proj", + "nvidia/Kimi-K2.6-NVFP4/q_a_proj" + ] }, { "type": "gemm", "args": { - "m": 314880, - "n": 1152, + "m": 2202, + "n": 1536, "k": 7168, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 32 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 1, + 32 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "amd/DeepSeek-R1-0528-MXFP4-Preview/q_a_proj", + "amd/DeepSeek-R1-0528-MXFP4-v2/q_a_proj", + "amd/DeepSeek-R1-0528-MXFP4/q_a_proj" + ] }, { "type": "gemm", "args": { - "m": 314880, - "n": 7168, - "k": 1152, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 2202, + "n": 1536, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w3" + ] }, { "type": "gemm", "args": { - "m": 38, - "n": 1152, - "k": 7168, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 2202, + "n": 7168, + "k": 9216, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/down_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/down_proj", + "nvidia/Kimi-K2.5-NVFP4/down_proj" + ] }, { "type": "gemm", "args": { - "m": 38, + "m": 2202, "n": 7168, - "k": 1152, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "k": 16384, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/o_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/o_proj" + ] }, { "type": "gemm", "args": { - "m": 41, - "n": 2304, - "k": 7168, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 2202, + "n": 7168, + "k": 16384, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 32 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 1, + 32 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "amd/DeepSeek-R1-0528-MXFP4-Preview/o_proj", + "amd/DeepSeek-R1-0528-MXFP4-v2/o_proj", + "amd/DeepSeek-R1-0528-MXFP4/o_proj" + ] }, { "type": "gemm", "args": { - "m": 41, + "m": 2202, "n": 7168, - "k": 2304, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "k": 16384, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] }, { "type": "gemm", "args": { - "m": 10, - "n": 7168, - "k": 576, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 2202, + "n": 9216, + "k": 7168, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/gate_up_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/gate_up_proj", + "nvidia/Kimi-K2.5-NVFP4/gate_up_proj" + ] }, { "type": "gemm", "args": { - "m": 64512, - "n": 1536, + "m": 2202, + "n": 9216, "k": 7168, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 32 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 1, + 32 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "amd/DeepSeek-R1-0528-MXFP4-Preview/gate_up_proj", + "amd/DeepSeek-R1-0528-MXFP4-v2/gate_up_proj", + "amd/DeepSeek-R1-0528-MXFP4/gate_up_proj", + "amd/Kimi-K2.5-MXFP4/gate_up_proj" + ] }, { "type": "gemm", "args": { - "m": 64512, + "m": 2202, "n": 24576, "k": 1536, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/q_b_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/q_b_proj" + ] }, { "type": "gemm", "args": { - "m": 64512, - "n": 576, - "k": 7168, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 2202, + "n": 24576, + "k": 1536, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 32 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 1, + 32 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "amd/DeepSeek-R1-0528-MXFP4-Preview/q_b_proj", + "amd/DeepSeek-R1-0528-MXFP4-v2/q_b_proj", + "amd/DeepSeek-R1-0528-MXFP4/q_b_proj" + ] }, { "type": "gemm", "args": { - "m": 64512, + "m": 2202, "n": 32768, "k": 512, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/kv_b_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/kv_b_proj" + ] }, { "type": "gemm", "args": { - "m": 64512, - "n": 7168, - "k": 16384, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 2202, + "n": 32768, + "k": 512, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 32 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 1, + 32 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "amd/DeepSeek-R1-0528-MXFP4-Preview/kv_b_proj", + "amd/DeepSeek-R1-0528-MXFP4-v2/kv_b_proj", + "amd/DeepSeek-R1-0528-MXFP4/kv_b_proj" + ] }, { "type": "gemm", "args": { - "m": 64512, - "n": 2304, + "m": 2253, + "n": 576, "k": 7168, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/Kimi-K2.5-MXFP4/kv_a_proj_with_mqa", + "moonshotai/Kimi-K2.5/kv_a_proj_with_mqa", + "nvidia/DeepSeek-R1-0528-FP4-V2/kv_a_proj_with_mqa", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/kv_a_proj_with_mqa", + "nvidia/Kimi-K2.5-NVFP4/kv_a_proj_with_mqa", + "nvidia/Kimi-K2.6-NVFP4/kv_a_proj_with_mqa" + ] }, { "type": "gemm", "args": { - "m": 64512, - "n": 7168, - "k": 2304, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 2253, + "n": 576, + "k": 7168, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 32 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 1, + 32 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "amd/DeepSeek-R1-0528-MXFP4-Preview/kv_a_proj_with_mqa", + "amd/DeepSeek-R1-0528-MXFP4-v2/kv_a_proj_with_mqa", + "amd/DeepSeek-R1-0528-MXFP4/kv_a_proj_with_mqa" + ] }, { "type": "gemm", "args": { - "m": 7, - "n": 2304, + "m": 2253, + "n": 1536, "k": 7168, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/Kimi-K2.5-MXFP4/q_a_proj", + "moonshotai/Kimi-K2.5/q_a_proj", + "nvidia/DeepSeek-R1-0528-FP4-V2/q_a_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/q_a_proj", + "nvidia/Kimi-K2.5-NVFP4/q_a_proj", + "nvidia/Kimi-K2.6-NVFP4/q_a_proj" + ] }, { "type": "gemm", "args": { - "m": 7, - "n": 7168, - "k": 2304, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 2253, + "n": 1536, + "k": 7168, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 32 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 1, + 32 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "amd/DeepSeek-R1-0528-MXFP4-Preview/q_a_proj", + "amd/DeepSeek-R1-0528-MXFP4-v2/q_a_proj", + "amd/DeepSeek-R1-0528-MXFP4/q_a_proj" + ] }, { "type": "gemm", "args": { - "m": 516096, + "m": 2253, "n": 1536, "k": 7168, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w3" + ] }, { "type": "gemm", "args": { - "m": 516096, - "n": 3072, - "k": 1536, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 2253, + "n": 7168, + "k": 9216, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/down_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/down_proj", + "nvidia/Kimi-K2.5-NVFP4/down_proj" + ] }, { "type": "gemm", "args": { - "m": 516096, - "n": 576, - "k": 7168, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 2253, + "n": 7168, + "k": 16384, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/o_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/o_proj" + ] }, { "type": "gemm", "args": { - "m": 516096, - "n": 4096, - "k": 512, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 2253, + "n": 7168, + "k": 16384, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 32 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 1, + 32 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "amd/DeepSeek-R1-0528-MXFP4-Preview/o_proj", + "amd/DeepSeek-R1-0528-MXFP4-v2/o_proj", + "amd/DeepSeek-R1-0528-MXFP4/o_proj" + ] }, { "type": "gemm", "args": { - "m": 516096, + "m": 2253, "n": 7168, - "k": 2048, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "k": 16384, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] }, { "type": "gemm", "args": { - "m": 516096, - "n": 2304, + "m": 2253, + "n": 9216, "k": 7168, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/gate_up_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/gate_up_proj", + "nvidia/Kimi-K2.5-NVFP4/gate_up_proj" + ] }, { "type": "gemm", "args": { - "m": 516096, - "n": 7168, - "k": 2304, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 2253, + "n": 9216, + "k": 7168, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 32 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 1, + 32 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "amd/DeepSeek-R1-0528-MXFP4-Preview/gate_up_proj", + "amd/DeepSeek-R1-0528-MXFP4-v2/gate_up_proj", + "amd/DeepSeek-R1-0528-MXFP4/gate_up_proj", + "amd/Kimi-K2.5-MXFP4/gate_up_proj" + ] }, { "type": "gemm", "args": { - "m": 63, - "n": 1536, - "k": 7168, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 2253, + "n": 24576, + "k": 1536, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/q_b_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/q_b_proj" + ] }, { "type": "gemm", "args": { - "m": 63, - "n": 3072, + "m": 2253, + "n": 24576, "k": 1536, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 32 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 1, + 32 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "amd/DeepSeek-R1-0528-MXFP4-Preview/q_b_proj", + "amd/DeepSeek-R1-0528-MXFP4-v2/q_b_proj", + "amd/DeepSeek-R1-0528-MXFP4/q_b_proj" + ] }, { "type": "gemm", "args": { - "m": 63, - "n": 576, - "k": 7168, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 2253, + "n": 32768, + "k": 512, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/kv_b_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/kv_b_proj" + ] }, { "type": "gemm", "args": { - "m": 63, - "n": 4096, + "m": 2253, + "n": 32768, "k": 512, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 32 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 1, + 32 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "amd/DeepSeek-R1-0528-MXFP4-Preview/kv_b_proj", + "amd/DeepSeek-R1-0528-MXFP4-v2/kv_b_proj", + "amd/DeepSeek-R1-0528-MXFP4/kv_b_proj" + ] }, { "type": "gemm", "args": { - "m": 63, - "n": 7168, - "k": 2048, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 2304, + "n": 576, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/kv_a_proj_with_mqa" + ] }, { "type": "gemm", "args": { - "m": 63, + "m": 2304, + "n": 1536, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/gate_up_proj", + "deepseek-ai/DeepSeek-R1-0528/q_a_proj", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w3" + ] + }, + { + "type": "gemm", + "args": { + "m": 2304, "n": 2304, "k": 7168, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/gate_up_proj" + ] }, { "type": "gemm", "args": { - "m": 63, + "m": 2304, "n": 7168, "k": 2304, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/down_proj" + ] }, { "type": "gemm", "args": { - "m": 18432, + "m": 2304, + "n": 7168, + "k": 16384, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/o_proj", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 2304, "n": 24576, "k": 1536, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/q_b_proj" + ] }, { "type": "gemm", "args": { - "m": 18432, + "m": 2304, "n": 32768, "k": 512, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/kv_b_proj" + ] }, { "type": "gemm", "args": { - "m": 18432, - "n": 7168, - "k": 16384, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 2305, + "n": 576, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/Kimi-K2.5-MXFP4/kv_a_proj_with_mqa", + "moonshotai/Kimi-K2.5/kv_a_proj_with_mqa", + "nvidia/DeepSeek-R1-0528-FP4-V2/kv_a_proj_with_mqa", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/kv_a_proj_with_mqa", + "nvidia/Kimi-K2.5-NVFP4/kv_a_proj_with_mqa", + "nvidia/Kimi-K2.6-NVFP4/kv_a_proj_with_mqa" + ] }, { "type": "gemm", "args": { - "m": 147456, - "n": 3072, - "k": 1536, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 2305, + "n": 576, + "k": 7168, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 32 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 1, + 32 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "amd/DeepSeek-R1-0528-MXFP4-Preview/kv_a_proj_with_mqa", + "amd/DeepSeek-R1-0528-MXFP4-v2/kv_a_proj_with_mqa", + "amd/DeepSeek-R1-0528-MXFP4/kv_a_proj_with_mqa" + ] }, { "type": "gemm", "args": { - "m": 147456, - "n": 4096, - "k": 512, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 2305, + "n": 1536, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/Kimi-K2.5-MXFP4/q_a_proj", + "moonshotai/Kimi-K2.5/q_a_proj", + "nvidia/DeepSeek-R1-0528-FP4-V2/q_a_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/q_a_proj", + "nvidia/Kimi-K2.5-NVFP4/q_a_proj", + "nvidia/Kimi-K2.6-NVFP4/q_a_proj" + ] }, { "type": "gemm", "args": { - "m": 147456, - "n": 7168, - "k": 2048, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 2305, + "n": 1536, + "k": 7168, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 32 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 1, + 32 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "amd/DeepSeek-R1-0528-MXFP4-Preview/q_a_proj", + "amd/DeepSeek-R1-0528-MXFP4-v2/q_a_proj", + "amd/DeepSeek-R1-0528-MXFP4/q_a_proj" + ] }, { "type": "gemm", "args": { - "m": 6, - "n": 3072, - "k": 1536, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 2305, + "n": 1536, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w3" + ] }, { "type": "gemm", "args": { - "m": 6, - "n": 4096, - "k": 512, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 2305, + "n": 7168, + "k": 9216, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/down_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/down_proj", + "nvidia/Kimi-K2.5-NVFP4/down_proj" + ] }, { "type": "gemm", "args": { - "m": 6, + "m": 2305, "n": 7168, - "k": 2048, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "k": 16384, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/o_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/o_proj" + ] }, { "type": "gemm", "args": { - "m": 6, - "n": 2304, - "k": 7168, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 2305, + "n": 7168, + "k": 16384, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 32 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 1, + 32 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "amd/DeepSeek-R1-0528-MXFP4-Preview/o_proj", + "amd/DeepSeek-R1-0528-MXFP4-v2/o_proj", + "amd/DeepSeek-R1-0528-MXFP4/o_proj" + ] }, { "type": "gemm", "args": { - "m": 6, + "m": 2305, "n": 7168, - "k": 2304, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "k": 16384, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] }, { "type": "gemm", "args": { - "m": 4096, - "n": 8192, - "k": 512, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 2305, + "n": 9216, + "k": 7168, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/gate_up_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/gate_up_proj", + "nvidia/Kimi-K2.5-NVFP4/gate_up_proj" + ] }, { "type": "gemm", "args": { - "m": 4096, - "n": 7168, - "k": 4096, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 2305, + "n": 9216, + "k": 7168, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 32 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 1, + 32 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "amd/DeepSeek-R1-0528-MXFP4-Preview/gate_up_proj", + "amd/DeepSeek-R1-0528-MXFP4-v2/gate_up_proj", + "amd/DeepSeek-R1-0528-MXFP4/gate_up_proj", + "amd/Kimi-K2.5-MXFP4/gate_up_proj" + ] }, { "type": "gemm", "args": { - "m": 21845, - "n": 1536, - "k": 7168, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 2305, + "n": 24576, + "k": 1536, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/q_b_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/q_b_proj" + ] }, { "type": "gemm", "args": { - "m": 21845, + "m": 2305, "n": 24576, "k": 1536, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 32 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 1, + 32 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "amd/DeepSeek-R1-0528-MXFP4-Preview/q_b_proj", + "amd/DeepSeek-R1-0528-MXFP4-v2/q_b_proj", + "amd/DeepSeek-R1-0528-MXFP4/q_b_proj" + ] }, { "type": "gemm", "args": { - "m": 21845, - "n": 576, - "k": 7168, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 2305, + "n": 32768, + "k": 512, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/kv_b_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/kv_b_proj" + ] }, { "type": "gemm", "args": { - "m": 21845, + "m": 2305, "n": 32768, "k": 512, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 32 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 1, + 32 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "amd/DeepSeek-R1-0528-MXFP4-Preview/kv_b_proj", + "amd/DeepSeek-R1-0528-MXFP4-v2/kv_b_proj", + "amd/DeepSeek-R1-0528-MXFP4/kv_b_proj" + ] }, { "type": "gemm", "args": { - "m": 21845, - "n": 7168, - "k": 16384, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 2342, + "n": 576, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/Kimi-K2.5-MXFP4/kv_a_proj_with_mqa", + "moonshotai/Kimi-K2.5/kv_a_proj_with_mqa", + "nvidia/DeepSeek-R1-0528-FP4-V2/kv_a_proj_with_mqa", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/kv_a_proj_with_mqa", + "nvidia/Kimi-K2.5-NVFP4/kv_a_proj_with_mqa", + "nvidia/Kimi-K2.6-NVFP4/kv_a_proj_with_mqa" + ] }, { "type": "gemm", "args": { - "m": 21845, - "n": 384, + "m": 2342, + "n": 576, "k": 7168, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 32 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 1, + 32 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "amd/DeepSeek-R1-0528-MXFP4-Preview/kv_a_proj_with_mqa", + "amd/DeepSeek-R1-0528-MXFP4-v2/kv_a_proj_with_mqa", + "amd/DeepSeek-R1-0528-MXFP4/kv_a_proj_with_mqa" + ] }, { "type": "gemm", "args": { - "m": 21845, - "n": 7168, - "k": 384, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 2342, + "n": 1536, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/Kimi-K2.5-MXFP4/q_a_proj", + "moonshotai/Kimi-K2.5/q_a_proj", + "nvidia/DeepSeek-R1-0528-FP4-V2/q_a_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/q_a_proj", + "nvidia/Kimi-K2.5-NVFP4/q_a_proj", + "nvidia/Kimi-K2.6-NVFP4/q_a_proj" + ] }, { "type": "gemm", "args": { - "m": 21, + "m": 2342, "n": 1536, "k": 7168, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 32 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 1, + 32 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "amd/DeepSeek-R1-0528-MXFP4-Preview/q_a_proj", + "amd/DeepSeek-R1-0528-MXFP4-v2/q_a_proj", + "amd/DeepSeek-R1-0528-MXFP4/q_a_proj" + ] }, { "type": "gemm", "args": { - "m": 21, - "n": 24576, - "k": 1536, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 2342, + "n": 1536, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w3" + ] }, { "type": "gemm", "args": { - "m": 21, - "n": 576, - "k": 7168, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 2342, + "n": 7168, + "k": 9216, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/down_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/down_proj", + "nvidia/Kimi-K2.5-NVFP4/down_proj" + ] }, { "type": "gemm", "args": { - "m": 21, - "n": 32768, - "k": 512, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 2342, + "n": 7168, + "k": 16384, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/o_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/o_proj" + ] }, { "type": "gemm", "args": { - "m": 21, + "m": 2342, "n": 7168, "k": 16384, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 32 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 1, + 32 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "amd/DeepSeek-R1-0528-MXFP4-Preview/o_proj", + "amd/DeepSeek-R1-0528-MXFP4-v2/o_proj", + "amd/DeepSeek-R1-0528-MXFP4/o_proj" + ] }, { "type": "gemm", "args": { - "m": 21, - "n": 384, - "k": 7168, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 2342, + "n": 7168, + "k": 16384, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] }, { "type": "gemm", "args": { - "m": 21, - "n": 7168, - "k": 384, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 2342, + "n": 9216, + "k": 7168, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/gate_up_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/gate_up_proj", + "nvidia/Kimi-K2.5-NVFP4/gate_up_proj" + ] }, { "type": "gemm", "args": { - "m": 43690, - "n": 1536, + "m": 2342, + "n": 9216, "k": 7168, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 32 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 1, + 32 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "amd/DeepSeek-R1-0528-MXFP4-Preview/gate_up_proj", + "amd/DeepSeek-R1-0528-MXFP4-v2/gate_up_proj", + "amd/DeepSeek-R1-0528-MXFP4/gate_up_proj", + "amd/Kimi-K2.5-MXFP4/gate_up_proj" + ] }, { "type": "gemm", "args": { - "m": 43690, + "m": 2342, "n": 24576, "k": 1536, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/q_b_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/q_b_proj" + ] }, { "type": "gemm", "args": { - "m": 43690, - "n": 576, - "k": 7168, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 2342, + "n": 24576, + "k": 1536, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 32 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 1, + 32 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "amd/DeepSeek-R1-0528-MXFP4-Preview/q_b_proj", + "amd/DeepSeek-R1-0528-MXFP4-v2/q_b_proj", + "amd/DeepSeek-R1-0528-MXFP4/q_b_proj" + ] }, { "type": "gemm", "args": { - "m": 43690, + "m": 2342, "n": 32768, "k": 512, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/kv_b_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/kv_b_proj" + ] }, { "type": "gemm", "args": { - "m": 43690, - "n": 7168, - "k": 16384, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 2342, + "n": 32768, + "k": 512, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 32 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 1, + 32 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "amd/DeepSeek-R1-0528-MXFP4-Preview/kv_b_proj", + "amd/DeepSeek-R1-0528-MXFP4-v2/kv_b_proj", + "amd/DeepSeek-R1-0528-MXFP4/kv_b_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 2484, + "n": 576, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/Kimi-K2.5-MXFP4/kv_a_proj_with_mqa", + "moonshotai/Kimi-K2.5/kv_a_proj_with_mqa", + "nvidia/DeepSeek-R1-0528-FP4-V2/kv_a_proj_with_mqa", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/kv_a_proj_with_mqa", + "nvidia/Kimi-K2.5-NVFP4/kv_a_proj_with_mqa", + "nvidia/Kimi-K2.6-NVFP4/kv_a_proj_with_mqa" + ] + }, + { + "type": "gemm", + "args": { + "m": 2484, + "n": 576, + "k": 7168, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 32 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 1, + 32 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "amd/DeepSeek-R1-0528-MXFP4-Preview/kv_a_proj_with_mqa", + "amd/DeepSeek-R1-0528-MXFP4-v2/kv_a_proj_with_mqa", + "amd/DeepSeek-R1-0528-MXFP4/kv_a_proj_with_mqa" + ] }, { "type": "gemm", "args": { - "m": 43690, - "n": 384, + "m": 2484, + "n": 1536, "k": 7168, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/Kimi-K2.5-MXFP4/q_a_proj", + "moonshotai/Kimi-K2.5/q_a_proj", + "nvidia/DeepSeek-R1-0528-FP4-V2/q_a_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/q_a_proj", + "nvidia/Kimi-K2.5-NVFP4/q_a_proj", + "nvidia/Kimi-K2.6-NVFP4/q_a_proj" + ] }, { "type": "gemm", "args": { - "m": 43690, - "n": 7168, - "k": 384, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 2484, + "n": 1536, + "k": 7168, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 32 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 1, + 32 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "amd/DeepSeek-R1-0528-MXFP4-Preview/q_a_proj", + "amd/DeepSeek-R1-0528-MXFP4-v2/q_a_proj", + "amd/DeepSeek-R1-0528-MXFP4/q_a_proj" + ] }, { "type": "gemm", "args": { - "m": 42, + "m": 2484, "n": 1536, "k": 7168, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w3" + ] }, { "type": "gemm", "args": { - "m": 42, - "n": 24576, - "k": 1536, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 2484, + "n": 4608, + "k": 7168, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/gate_up_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/gate_up_proj", + "nvidia/Kimi-K2.5-NVFP4/gate_up_proj" + ] }, { "type": "gemm", "args": { - "m": 42, - "n": 576, + "m": 2484, + "n": 4608, "k": 7168, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 32 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 1, + 32 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "amd/DeepSeek-R1-0528-MXFP4-Preview/gate_up_proj", + "amd/DeepSeek-R1-0528-MXFP4-v2/gate_up_proj", + "amd/DeepSeek-R1-0528-MXFP4/gate_up_proj", + "amd/Kimi-K2.5-MXFP4/gate_up_proj" + ] }, { "type": "gemm", "args": { - "m": 42, - "n": 32768, - "k": 512, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 2484, + "n": 7168, + "k": 4608, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/down_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/down_proj", + "nvidia/Kimi-K2.5-NVFP4/down_proj" + ] }, { "type": "gemm", "args": { - "m": 42, + "m": 2484, + "n": 7168, + "k": 4608, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 32 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 1, + 32 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "amd/DeepSeek-R1-0528-MXFP4-Preview/down_proj", + "amd/DeepSeek-R1-0528-MXFP4-v2/down_proj", + "amd/DeepSeek-R1-0528-MXFP4/down_proj", + "amd/Kimi-K2.5-MXFP4/down_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 2484, "n": 7168, "k": 16384, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/o_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/o_proj" + ] }, { "type": "gemm", "args": { - "m": 42, - "n": 384, - "k": 7168, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 2484, + "n": 7168, + "k": 16384, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 32 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 1, + 32 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "amd/DeepSeek-R1-0528-MXFP4-Preview/o_proj", + "amd/DeepSeek-R1-0528-MXFP4-v2/o_proj", + "amd/DeepSeek-R1-0528-MXFP4/o_proj" + ] }, { "type": "gemm", "args": { - "m": 42, + "m": 2484, "n": 7168, - "k": 384, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "k": 16384, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] }, { "type": "gemm", "args": { - "m": 87381, - "n": 1536, - "k": 7168, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 2484, + "n": 24576, + "k": 1536, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/q_b_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/q_b_proj" + ] }, { "type": "gemm", "args": { - "m": 87381, + "m": 2484, "n": 24576, "k": 1536, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 32 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 1, + 32 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "amd/DeepSeek-R1-0528-MXFP4-Preview/q_b_proj", + "amd/DeepSeek-R1-0528-MXFP4-v2/q_b_proj", + "amd/DeepSeek-R1-0528-MXFP4/q_b_proj" + ] }, { "type": "gemm", "args": { - "m": 87381, - "n": 576, - "k": 7168, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 2484, + "n": 32768, + "k": 512, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/kv_b_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/kv_b_proj" + ] }, { "type": "gemm", "args": { - "m": 87381, + "m": 2484, "n": 32768, "k": 512, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 32 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 1, + 32 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "amd/DeepSeek-R1-0528-MXFP4-Preview/kv_b_proj", + "amd/DeepSeek-R1-0528-MXFP4-v2/kv_b_proj", + "amd/DeepSeek-R1-0528-MXFP4/kv_b_proj" + ] }, { "type": "gemm", "args": { - "m": 87381, - "n": 7168, - "k": 16384, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 2560, + "n": 512, + "k": 2880, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/gpt-oss-120b-w-mxfp4-a-fp8/k_proj", + "amd/gpt-oss-120b-w-mxfp4-a-fp8/q_proj", + "amd/gpt-oss-120b-w-mxfp4-a-fp8/v_proj", + "openai/gpt-oss-120b/k_proj", + "openai/gpt-oss-120b/q_proj", + "openai/gpt-oss-120b/v_proj" + ] }, { "type": "gemm", "args": { - "m": 87381, - "n": 384, + "m": 2560, + "n": 576, "k": 7168, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/Kimi-K2.5-MXFP4/kv_a_proj_with_mqa", + "moonshotai/Kimi-K2.5/kv_a_proj_with_mqa", + "nvidia/DeepSeek-R1-0528-FP4-V2/kv_a_proj_with_mqa", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/kv_a_proj_with_mqa", + "nvidia/Kimi-K2.5-NVFP4/kv_a_proj_with_mqa", + "nvidia/Kimi-K2.6-NVFP4/kv_a_proj_with_mqa" + ] }, { "type": "gemm", "args": { - "m": 87381, - "n": 7168, - "k": 384, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 2560, + "n": 576, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/kv_a_proj_with_mqa" + ] }, { "type": "gemm", "args": { - "m": 85, + "m": 2560, "n": 1536, "k": 7168, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/Kimi-K2.5-MXFP4/q_a_proj", + "moonshotai/Kimi-K2.5/q_a_proj", + "nvidia/DeepSeek-R1-0528-FP4-V2/q_a_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/q_a_proj", + "nvidia/Kimi-K2.5-NVFP4/q_a_proj", + "nvidia/Kimi-K2.6-NVFP4/q_a_proj" + ] }, { "type": "gemm", "args": { - "m": 85, - "n": 24576, - "k": 1536, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 2560, + "n": 1536, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/gate_up_proj", + "deepseek-ai/DeepSeek-R1-0528/q_a_proj", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w3" + ] }, { "type": "gemm", "args": { - "m": 85, - "n": 576, - "k": 7168, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 2560, + "n": 2880, + "k": 4096, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/gpt-oss-120b-w-mxfp4-a-fp8/o_proj", + "openai/gpt-oss-120b/o_proj" + ] }, { "type": "gemm", "args": { - "m": 85, - "n": 32768, - "k": 512, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 2560, + "n": 4096, + "k": 2880, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/gpt-oss-120b-w-mxfp4-a-fp8/q_proj", + "openai/gpt-oss-120b/q_proj" + ] }, { "type": "gemm", "args": { - "m": 85, - "n": 7168, - "k": 16384, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 2560, + "n": 4608, + "k": 7168, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/gate_up_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/gate_up_proj", + "nvidia/Kimi-K2.5-NVFP4/gate_up_proj" + ] }, { "type": "gemm", "args": { - "m": 85, - "n": 384, + "m": 2560, + "n": 4608, "k": 7168, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 32 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 1, + 32 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "amd/DeepSeek-R1-0528-MXFP4-Preview/gate_up_proj", + "amd/DeepSeek-R1-0528-MXFP4-v2/gate_up_proj", + "amd/DeepSeek-R1-0528-MXFP4/gate_up_proj", + "amd/Kimi-K2.5-MXFP4/gate_up_proj" + ] }, { "type": "gemm", "args": { - "m": 85, - "n": 7168, - "k": 384, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 2560, + "n": 4608, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/gate_up_proj" + ] }, { "type": "gemm", "args": { - "m": 32768, + "m": 2560, "n": 7168, - "k": 576, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "k": 4608, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/down_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/down_proj", + "nvidia/Kimi-K2.5-NVFP4/down_proj" + ] }, { "type": "gemm", "args": { - "m": 32, + "m": 2560, "n": 7168, - "k": 576, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "k": 4608, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/down_proj" + ] }, { "type": "gemm", "args": { - "m": 65536, + "m": 2560, "n": 7168, - "k": 576, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "k": 9216, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/down_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/down_proj", + "nvidia/Kimi-K2.5-NVFP4/down_proj" + ] }, { "type": "gemm", "args": { - "m": 64, + "m": 2560, "n": 7168, - "k": 576, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "k": 16384, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/o_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/o_proj" + ] }, { "type": "gemm", "args": { - "m": 131072, + "m": 2560, "n": 7168, - "k": 576, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "k": 16384, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/o_proj", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] }, { "type": "gemm", "args": { - "m": 128, - "n": 7168, - "k": 576, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 2560, + "n": 9216, + "k": 7168, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/gate_up_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/gate_up_proj", + "nvidia/Kimi-K2.5-NVFP4/gate_up_proj" + ] }, { "type": "gemm", "args": { - "m": 196608, - "n": 1536, + "m": 2560, + "n": 9216, "k": 7168, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/gate_up_proj" + ] }, { "type": "gemm", "args": { - "m": 196608, + "m": 2560, "n": 24576, "k": 1536, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/q_b_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/q_b_proj" + ] }, { "type": "gemm", "args": { - "m": 196608, - "n": 576, - "k": 7168, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 2560, + "n": 24576, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/q_b_proj" + ] }, { "type": "gemm", "args": { - "m": 196608, + "m": 2560, "n": 32768, "k": 512, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/kv_b_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/kv_b_proj" + ] }, { "type": "gemm", "args": { - "m": 196608, - "n": 7168, - "k": 16384, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 2560, + "n": 32768, + "k": 512, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/kv_b_proj" + ] }, { "type": "gemm", "args": { - "m": 196608, - "n": 7168, - "k": 576, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 2816, + "n": 576, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/kv_a_proj_with_mqa" + ] }, { "type": "gemm", "args": { - "m": 192, + "m": 2816, "n": 1536, "k": 7168, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/gate_up_proj", + "deepseek-ai/DeepSeek-R1-0528/q_a_proj", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w3" + ] }, { "type": "gemm", "args": { - "m": 192, - "n": 24576, - "k": 1536, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 2816, + "n": 2304, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/gate_up_proj" + ] }, { "type": "gemm", "args": { - "m": 192, - "n": 576, - "k": 7168, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 2816, + "n": 7168, + "k": 2304, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/down_proj" + ] }, { "type": "gemm", "args": { - "m": 192, + "m": 2816, + "n": 7168, + "k": 16384, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/o_proj", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 2816, + "n": 24576, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/q_b_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 2816, "n": 32768, "k": 512, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/kv_b_proj" + ] }, { "type": "gemm", "args": { - "m": 192, - "n": 7168, - "k": 16384, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 3072, + "n": 512, + "k": 2880, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/gpt-oss-120b-w-mxfp4-a-fp8/k_proj", + "amd/gpt-oss-120b-w-mxfp4-a-fp8/q_proj", + "amd/gpt-oss-120b-w-mxfp4-a-fp8/v_proj", + "openai/gpt-oss-120b/k_proj", + "openai/gpt-oss-120b/q_proj", + "openai/gpt-oss-120b/v_proj" + ] }, { "type": "gemm", "args": { - "m": 192, - "n": 7168, - "k": 576, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 3072, + "n": 576, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/Kimi-K2.5-MXFP4/kv_a_proj_with_mqa", + "moonshotai/Kimi-K2.5/kv_a_proj_with_mqa", + "nvidia/DeepSeek-R1-0528-FP4-V2/kv_a_proj_with_mqa", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/kv_a_proj_with_mqa", + "nvidia/Kimi-K2.5-NVFP4/kv_a_proj_with_mqa", + "nvidia/Kimi-K2.6-NVFP4/kv_a_proj_with_mqa" + ] }, { "type": "gemm", "args": { - "m": 6291456, - "n": 2304, + "m": 3072, + "n": 576, "k": 7168, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/kv_a_proj_with_mqa" + ] }, { "type": "gemm", "args": { - "m": 6291456, - "n": 7168, - "k": 2304, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 3072, + "n": 1152, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/gate_up_proj" + ] }, { "type": "gemm", "args": { - "m": 16, - "n": 7168, - "k": 576, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 3072, + "n": 1536, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/Kimi-K2.5-MXFP4/q_a_proj", + "moonshotai/Kimi-K2.5/q_a_proj", + "nvidia/DeepSeek-R1-0528-FP4-V2/q_a_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/q_a_proj", + "nvidia/Kimi-K2.5-NVFP4/q_a_proj", + "nvidia/Kimi-K2.6-NVFP4/q_a_proj" + ] }, { "type": "gemm", "args": { - "m": 262144, - "n": 7168, - "k": 576, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 3072, + "n": 1536, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/gate_up_proj", + "deepseek-ai/DeepSeek-R1-0528/q_a_proj", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w3" + ] }, { "type": "gemm", "args": { - "m": 524288, - "n": 7168, - "k": 576, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 3072, + "n": 2304, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/gate_up_proj" + ] }, { "type": "gemm", "args": { - "m": 1572864, - "n": 1536, - "k": 7168, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 3072, + "n": 2880, + "k": 4096, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/gpt-oss-120b-w-mxfp4-a-fp8/o_proj", + "openai/gpt-oss-120b/o_proj" + ] }, { "type": "gemm", "args": { - "m": 1572864, - "n": 24576, + "m": 3072, + "n": 3072, "k": 1536, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "MiniMaxAI/MiniMax-M2.5/experts.down_proj", + "MiniMaxAI/MiniMax-M2.5/o_proj", + "deepseek-ai/DeepSeek-R1-0528/q_b_proj" + ] }, { "type": "gemm", "args": { - "m": 1572864, - "n": 576, - "k": 7168, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 3072, + "n": 4096, + "k": 512, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "Qwen/Qwen3.5-397B-A17B-FP8/o_proj", + "Qwen/Qwen3.5-397B-A17B-FP8/shared_expert.down_proj", + "deepseek-ai/DeepSeek-R1-0528/kv_b_proj" + ] }, { "type": "gemm", "args": { - "m": 1572864, - "n": 32768, - "k": 512, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 3072, + "n": 4096, + "k": 2880, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/gpt-oss-120b-w-mxfp4-a-fp8/q_proj", + "openai/gpt-oss-120b/q_proj" + ] }, { "type": "gemm", "args": { - "m": 1572864, - "n": 7168, - "k": 16384, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 3072, + "n": 4608, + "k": 7168, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/gate_up_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/gate_up_proj", + "nvidia/Kimi-K2.5-NVFP4/gate_up_proj" + ] }, { "type": "gemm", "args": { - "m": 1572864, - "n": 7168, - "k": 576, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 3072, + "n": 4608, + "k": 7168, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 32 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 1, + 32 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "amd/DeepSeek-R1-0528-MXFP4-Preview/gate_up_proj", + "amd/DeepSeek-R1-0528-MXFP4-v2/gate_up_proj", + "amd/DeepSeek-R1-0528-MXFP4/gate_up_proj", + "amd/Kimi-K2.5-MXFP4/gate_up_proj" + ] }, { "type": "gemm", "args": { - "m": 699050, - "n": 1536, + "m": 3072, + "n": 4608, "k": 7168, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/gate_up_proj" + ] }, { "type": "gemm", "args": { - "m": 699050, - "n": 24576, - "k": 1536, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 3072, + "n": 7168, + "k": 1152, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/down_proj" + ] }, { "type": "gemm", "args": { - "m": 699050, - "n": 576, - "k": 7168, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 3072, + "n": 7168, + "k": 2048, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/o_proj", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] }, { "type": "gemm", "args": { - "m": 699050, - "n": 32768, - "k": 512, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 3072, + "n": 7168, + "k": 2304, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/down_proj" + ] }, { "type": "gemm", "args": { - "m": 699050, + "m": 3072, "n": 7168, - "k": 16384, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "k": 4608, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/down_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/down_proj", + "nvidia/Kimi-K2.5-NVFP4/down_proj" + ] }, { "type": "gemm", "args": { - "m": 699050, - "n": 768, - "k": 7168, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 3072, + "n": 7168, + "k": 4608, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/down_proj" + ] }, { "type": "gemm", "args": { - "m": 699050, + "m": 3072, "n": 7168, - "k": 768, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "k": 9216, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/down_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/down_proj", + "nvidia/Kimi-K2.5-NVFP4/down_proj" + ] }, { "type": "gemm", "args": { - "m": 85, - "n": 768, - "k": 7168, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 3072, + "n": 7168, + "k": 16384, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/o_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/o_proj" + ] }, { "type": "gemm", "args": { - "m": 85, + "m": 3072, "n": 7168, - "k": 768, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "k": 16384, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/o_proj", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] }, { "type": "gemm", "args": { - "m": 1398101, - "n": 1536, + "m": 3072, + "n": 9216, + "k": 7168, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/gate_up_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/gate_up_proj", + "nvidia/Kimi-K2.5-NVFP4/gate_up_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 3072, + "n": 9216, "k": 7168, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/gate_up_proj" + ] }, { "type": "gemm", "args": { - "m": 1398101, + "m": 3072, "n": 24576, "k": 1536, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/q_b_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/q_b_proj" + ] }, { "type": "gemm", "args": { - "m": 1398101, - "n": 576, - "k": 7168, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 3072, + "n": 24576, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/q_b_proj" + ] }, { "type": "gemm", "args": { - "m": 1398101, + "m": 3072, "n": 32768, "k": 512, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/kv_b_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/kv_b_proj" + ] }, { "type": "gemm", "args": { - "m": 1398101, - "n": 7168, - "k": 16384, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 3072, + "n": 32768, + "k": 512, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/kv_b_proj" + ] }, { "type": "gemm", "args": { - "m": 1398101, - "n": 768, + "m": 3226, + "n": 576, "k": 7168, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/Kimi-K2.5-MXFP4/kv_a_proj_with_mqa", + "moonshotai/Kimi-K2.5/kv_a_proj_with_mqa", + "nvidia/DeepSeek-R1-0528-FP4-V2/kv_a_proj_with_mqa", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/kv_a_proj_with_mqa", + "nvidia/Kimi-K2.5-NVFP4/kv_a_proj_with_mqa", + "nvidia/Kimi-K2.6-NVFP4/kv_a_proj_with_mqa" + ] }, { "type": "gemm", "args": { - "m": 1398101, - "n": 7168, - "k": 768, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 3226, + "n": 576, + "k": 7168, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 32 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 1, + 32 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "amd/DeepSeek-R1-0528-MXFP4-Preview/kv_a_proj_with_mqa", + "amd/DeepSeek-R1-0528-MXFP4-v2/kv_a_proj_with_mqa", + "amd/DeepSeek-R1-0528-MXFP4/kv_a_proj_with_mqa" + ] }, { "type": "gemm", "args": { - "m": 170, + "m": 3226, "n": 1536, "k": 7168, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/Kimi-K2.5-MXFP4/q_a_proj", + "moonshotai/Kimi-K2.5/q_a_proj", + "nvidia/DeepSeek-R1-0528-FP4-V2/q_a_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/q_a_proj", + "nvidia/Kimi-K2.5-NVFP4/q_a_proj", + "nvidia/Kimi-K2.6-NVFP4/q_a_proj" + ] }, { "type": "gemm", "args": { - "m": 170, - "n": 24576, - "k": 1536, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 3226, + "n": 1536, + "k": 7168, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 32 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 1, + 32 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "amd/DeepSeek-R1-0528-MXFP4-Preview/q_a_proj", + "amd/DeepSeek-R1-0528-MXFP4-v2/q_a_proj", + "amd/DeepSeek-R1-0528-MXFP4/q_a_proj" + ] }, { "type": "gemm", "args": { - "m": 170, - "n": 576, + "m": 3226, + "n": 1536, "k": 7168, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w3" + ] }, { "type": "gemm", "args": { - "m": 170, - "n": 32768, - "k": 512, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 3226, + "n": 7168, + "k": 9216, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/down_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/down_proj", + "nvidia/Kimi-K2.5-NVFP4/down_proj" + ] }, { "type": "gemm", "args": { - "m": 170, + "m": 3226, "n": 7168, "k": 16384, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/o_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/o_proj" + ] }, { "type": "gemm", "args": { - "m": 170, - "n": 768, - "k": 7168, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 3226, + "n": 7168, + "k": 16384, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 32 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 1, + 32 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "amd/DeepSeek-R1-0528-MXFP4-Preview/o_proj", + "amd/DeepSeek-R1-0528-MXFP4-v2/o_proj", + "amd/DeepSeek-R1-0528-MXFP4/o_proj" + ] }, { "type": "gemm", "args": { - "m": 170, + "m": 3226, "n": 7168, - "k": 768, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "k": 16384, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] }, { "type": "gemm", "args": { - "m": 2097152, - "n": 768, + "m": 3226, + "n": 9216, "k": 7168, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/gate_up_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/gate_up_proj", + "nvidia/Kimi-K2.5-NVFP4/gate_up_proj" + ] }, { "type": "gemm", "args": { - "m": 2097152, - "n": 7168, - "k": 768, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 3226, + "n": 9216, + "k": 7168, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 32 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 1, + 32 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "amd/DeepSeek-R1-0528-MXFP4-Preview/gate_up_proj", + "amd/DeepSeek-R1-0528-MXFP4-v2/gate_up_proj", + "amd/DeepSeek-R1-0528-MXFP4/gate_up_proj", + "amd/Kimi-K2.5-MXFP4/gate_up_proj" + ] }, { "type": "gemm", "args": { - "m": 256, - "n": 768, - "k": 7168, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 3226, + "n": 24576, + "k": 1536, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/q_b_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/q_b_proj" + ] }, { "type": "gemm", "args": { - "m": 256, - "n": 7168, - "k": 768, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 3226, + "n": 24576, + "k": 1536, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 32 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 1, + 32 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "amd/DeepSeek-R1-0528-MXFP4-Preview/q_b_proj", + "amd/DeepSeek-R1-0528-MXFP4-v2/q_b_proj", + "amd/DeepSeek-R1-0528-MXFP4/q_b_proj" + ] }, { "type": "gemm", "args": { - "m": 917504, - "n": 1536, - "k": 7168, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 3226, + "n": 32768, + "k": 512, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/kv_b_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/kv_b_proj" + ] }, { "type": "gemm", "args": { - "m": 917504, - "n": 24576, - "k": 1536, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 3226, + "n": 32768, + "k": 512, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 32 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 1, + 32 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "amd/DeepSeek-R1-0528-MXFP4-Preview/kv_b_proj", + "amd/DeepSeek-R1-0528-MXFP4-v2/kv_b_proj", + "amd/DeepSeek-R1-0528-MXFP4/kv_b_proj" + ] }, { "type": "gemm", "args": { - "m": 917504, + "m": 3231, "n": 576, "k": 7168, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 917504, - "n": 32768, - "k": 512, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/Kimi-K2.5-MXFP4/kv_a_proj_with_mqa", + "moonshotai/Kimi-K2.5/kv_a_proj_with_mqa", + "nvidia/DeepSeek-R1-0528-FP4-V2/kv_a_proj_with_mqa", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/kv_a_proj_with_mqa", + "nvidia/Kimi-K2.5-NVFP4/kv_a_proj_with_mqa", + "nvidia/Kimi-K2.6-NVFP4/kv_a_proj_with_mqa" + ] }, { "type": "gemm", "args": { - "m": 917504, - "n": 7168, - "k": 16384, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 3231, + "n": 1536, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/Kimi-K2.5-MXFP4/q_a_proj", + "moonshotai/Kimi-K2.5/q_a_proj", + "nvidia/DeepSeek-R1-0528-FP4-V2/q_a_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/q_a_proj", + "nvidia/Kimi-K2.5-NVFP4/q_a_proj", + "nvidia/Kimi-K2.6-NVFP4/q_a_proj" + ] }, { "type": "gemm", "args": { - "m": 917504, + "m": 3231, "n": 2304, "k": 7168, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/gate_up_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/gate_up_proj", + "nvidia/Kimi-K2.5-NVFP4/gate_up_proj" + ] }, { "type": "gemm", "args": { - "m": 917504, + "m": 3231, "n": 7168, "k": 2304, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/down_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/down_proj", + "nvidia/Kimi-K2.5-NVFP4/down_proj" + ] }, { "type": "gemm", "args": { - "m": 896, - "n": 1536, - "k": 7168, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 3231, + "n": 7168, + "k": 16384, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/o_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/o_proj" + ] }, { "type": "gemm", "args": { - "m": 896, + "m": 3231, "n": 24576, "k": 1536, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/q_b_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/q_b_proj" + ] }, { "type": "gemm", "args": { - "m": 896, - "n": 576, - "k": 7168, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 3231, + "n": 32768, + "k": 512, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/kv_b_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/kv_b_proj" + ] }, { "type": "gemm", "args": { - "m": 896, - "n": 32768, - "k": 512, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 3456, + "n": 576, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/kv_a_proj_with_mqa" + ] }, { "type": "gemm", "args": { - "m": 896, - "n": 7168, - "k": 16384, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 3456, + "n": 1536, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/gate_up_proj", + "deepseek-ai/DeepSeek-R1-0528/q_a_proj", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w3" + ] }, { "type": "gemm", "args": { - "m": 896, + "m": 3456, "n": 2304, "k": 7168, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/gate_up_proj" + ] }, { "type": "gemm", "args": { - "m": 896, + "m": 3456, "n": 7168, "k": 2304, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/down_proj" + ] }, { "type": "gemm", "args": { - "m": 983040, - "n": 1536, - "k": 7168, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 3456, + "n": 7168, + "k": 16384, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/o_proj", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] }, { "type": "gemm", "args": { - "m": 983040, + "m": 3456, "n": 24576, "k": 1536, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/q_b_proj" + ] }, { "type": "gemm", "args": { - "m": 983040, - "n": 576, - "k": 7168, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 3456, + "n": 32768, + "k": 512, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/kv_b_proj" + ] }, { "type": "gemm", "args": { - "m": 983040, - "n": 32768, - "k": 512, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 3840, + "n": 576, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/Kimi-K2.5-MXFP4/kv_a_proj_with_mqa", + "moonshotai/Kimi-K2.5/kv_a_proj_with_mqa", + "nvidia/DeepSeek-R1-0528-FP4-V2/kv_a_proj_with_mqa", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/kv_a_proj_with_mqa", + "nvidia/Kimi-K2.5-NVFP4/kv_a_proj_with_mqa", + "nvidia/Kimi-K2.6-NVFP4/kv_a_proj_with_mqa" + ] }, { "type": "gemm", "args": { - "m": 983040, - "n": 7168, - "k": 16384, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 3840, + "n": 1152, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/gate_up_proj" + ] }, { "type": "gemm", "args": { - "m": 983040, - "n": 2304, + "m": 3840, + "n": 1536, "k": 7168, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/Kimi-K2.5-MXFP4/q_a_proj", + "moonshotai/Kimi-K2.5/q_a_proj", + "nvidia/DeepSeek-R1-0528-FP4-V2/q_a_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/q_a_proj", + "nvidia/Kimi-K2.5-NVFP4/q_a_proj", + "nvidia/Kimi-K2.6-NVFP4/q_a_proj" + ] }, { "type": "gemm", "args": { - "m": 983040, - "n": 7168, - "k": 2304, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 3840, + "n": 4608, + "k": 7168, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/gate_up_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/gate_up_proj", + "nvidia/Kimi-K2.5-NVFP4/gate_up_proj" + ] }, { "type": "gemm", "args": { - "m": 960, - "n": 1536, + "m": 3840, + "n": 4608, "k": 7168, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 32 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 1, + 32 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "amd/DeepSeek-R1-0528-MXFP4-Preview/gate_up_proj", + "amd/DeepSeek-R1-0528-MXFP4-v2/gate_up_proj", + "amd/DeepSeek-R1-0528-MXFP4/gate_up_proj", + "amd/Kimi-K2.5-MXFP4/gate_up_proj" + ] }, { "type": "gemm", "args": { - "m": 960, - "n": 24576, - "k": 1536, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 3840, + "n": 7168, + "k": 1152, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/down_proj" + ] }, { "type": "gemm", "args": { - "m": 960, - "n": 576, - "k": 7168, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 3840, + "n": 7168, + "k": 4608, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/down_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/down_proj", + "nvidia/Kimi-K2.5-NVFP4/down_proj" + ] }, { "type": "gemm", "args": { - "m": 960, - "n": 32768, - "k": 512, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 3840, + "n": 7168, + "k": 8192, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/Kimi-K2.5-MXFP4/o_proj", + "moonshotai/Kimi-K2.5/o_proj", + "nvidia/Kimi-K2.5-NVFP4/o_proj", + "nvidia/Kimi-K2.6-NVFP4/o_proj" + ] }, { "type": "gemm", "args": { - "m": 960, + "m": 3840, "n": 7168, "k": 16384, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/o_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/o_proj" + ] }, { "type": "gemm", "args": { - "m": 960, - "n": 2304, - "k": 7168, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 3840, + "n": 12288, + "k": 1536, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/Kimi-K2.5-MXFP4/q_b_proj", + "moonshotai/Kimi-K2.5/q_b_proj", + "nvidia/DeepSeek-R1-0528-FP4-V2/q_b_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/q_b_proj", + "nvidia/Kimi-K2.5-NVFP4/q_b_proj", + "nvidia/Kimi-K2.6-NVFP4/q_b_proj" + ] }, { "type": "gemm", "args": { - "m": 960, - "n": 7168, - "k": 2304, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 3840, + "n": 16384, + "k": 512, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/Kimi-K2.5-MXFP4/kv_b_proj", + "moonshotai/Kimi-K2.5/kv_b_proj", + "nvidia/DeepSeek-R1-0528-FP4-V2/kv_b_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/kv_b_proj", + "nvidia/Kimi-K2.5-NVFP4/kv_b_proj", + "nvidia/Kimi-K2.6-NVFP4/kv_b_proj" + ] }, { "type": "gemm", "args": { - "m": 4, - "n": 7168, - "k": 576, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 3840, + "n": 24576, + "k": 1536, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/q_b_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/q_b_proj" + ] }, { "type": "gemm", "args": { - "m": 512, - "n": 4608, - "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 3840, + "n": 32768, + "k": 512, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/kv_b_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/kv_b_proj" + ] }, { "type": "gemm", "args": { - "m": 512, - "n": 7168, - "k": 4608, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 4096, + "n": 64, + "k": 2880, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/gpt-oss-120b-w-mxfp4-a-fp8/k_proj", + "amd/gpt-oss-120b-w-mxfp4-a-fp8/v_proj", + "openai/gpt-oss-120b/k_proj", + "openai/gpt-oss-120b/v_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 4096, + "n": 128, + "k": 2880, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/gpt-oss-120b-w-mxfp4-a-fp8/k_proj", + "amd/gpt-oss-120b-w-mxfp4-a-fp8/v_proj", + "openai/gpt-oss-120b/k_proj", + "openai/gpt-oss-120b/v_proj" + ] }, { "type": "gemm", "args": { - "m": 16384, - "n": 7168, - "k": 576, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 4096, + "n": 128, + "k": 3072, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/MiniMax-M2.5-MXFP4/k_proj", + "amd/MiniMax-M2.5-MXFP4/v_proj", + "nvidia/MiniMax-M2.5-NVFP4/k_proj", + "nvidia/MiniMax-M2.5-NVFP4/v_proj" + ] }, { "type": "gemm", "args": { - "m": 16, - "n": 7168, - "k": 576, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 4096, + "n": 128, + "k": 3072, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "MiniMaxAI/MiniMax-M2.5/k_proj", + "MiniMaxAI/MiniMax-M2.5/v_proj" + ] }, { "type": "gemm", "args": { - "m": 65536, - "n": 7168, - "k": 576, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 4096, + "n": 128, + "k": 4096, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "Qwen/Qwen3.5-397B-A17B/shared_expert.gate_up_proj", + "amd/Qwen3.5-397B-A17B-MXFP4/shared_expert.gate_up_proj", + "nvidia/Qwen3.5-397B-A17B-NVFP4/shared_expert.gate_up_proj" + ] }, { "type": "gemm", "args": { - "m": 64, - "n": 7168, - "k": 576, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 4096, + "n": 128, + "k": 4096, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "Qwen/Qwen3.5-397B-A17B-FP8/shared_expert.gate_up_proj" + ] }, { "type": "gemm", "args": { - "m": 131072, - "n": 7168, - "k": 576, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 4096, + "n": 192, + "k": 3072, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/MiniMax-M2.5-NVFP4/experts.gate_up_proj" + ] }, { "type": "gemm", "args": { - "m": 128, - "n": 7168, - "k": 576, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 4096, + "n": 192, + "k": 3072, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 32 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 1, + 32 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "amd/MiniMax-M2.5-MXFP4/experts.gate_up_proj" + ] }, { "type": "gemm", "args": { - "m": 262144, - "n": 7168, - "k": 576, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 4096, + "n": 192, + "k": 3072, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "MiniMaxAI/MiniMax-M2.5/experts.gate_up_proj" + ] }, { "type": "gemm", "args": { - "m": 256, - "n": 7168, - "k": 576, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 4096, + "n": 256, + "k": 2880, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/gpt-oss-120b-w-mxfp4-a-fp8/k_proj", + "amd/gpt-oss-120b-w-mxfp4-a-fp8/q_proj", + "amd/gpt-oss-120b-w-mxfp4-a-fp8/v_proj", + "openai/gpt-oss-120b/k_proj", + "openai/gpt-oss-120b/q_proj", + "openai/gpt-oss-120b/v_proj" + ] }, { "type": "gemm", "args": { - "m": 43690, - "n": 1536, - "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 4096, + "n": 256, + "k": 3072, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/MiniMax-M2.5-MXFP4/k_proj", + "amd/MiniMax-M2.5-MXFP4/v_proj", + "nvidia/MiniMax-M2.5-NVFP4/k_proj", + "nvidia/MiniMax-M2.5-NVFP4/v_proj" + ] }, { "type": "gemm", "args": { - "m": 43690, - "n": 24576, - "k": 1536, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 4096, + "n": 256, + "k": 3072, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "MiniMaxAI/MiniMax-M2.5/k_proj", + "MiniMaxAI/MiniMax-M2.5/v_proj" + ] }, { "type": "gemm", "args": { - "m": 43690, - "n": 576, - "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 4096, + "n": 256, + "k": 4096, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "Qwen/Qwen3.5-397B-A17B/k_proj", + "Qwen/Qwen3.5-397B-A17B/shared_expert.gate_up_proj", + "Qwen/Qwen3.5-397B-A17B/v_proj", + "amd/Qwen3.5-397B-A17B-MXFP4/k_proj", + "amd/Qwen3.5-397B-A17B-MXFP4/shared_expert.gate_up_proj", + "amd/Qwen3.5-397B-A17B-MXFP4/v_proj", + "nvidia/Qwen3.5-397B-A17B-NVFP4/k_proj", + "nvidia/Qwen3.5-397B-A17B-NVFP4/shared_expert.gate_up_proj", + "nvidia/Qwen3.5-397B-A17B-NVFP4/v_proj" + ] }, { "type": "gemm", "args": { - "m": 43690, - "n": 32768, - "k": 512, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 4096, + "n": 256, + "k": 4096, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "Qwen/Qwen3.5-397B-A17B-FP8/k_proj", + "Qwen/Qwen3.5-397B-A17B-FP8/shared_expert.gate_up_proj", + "Qwen/Qwen3.5-397B-A17B-FP8/v_proj" + ] }, { "type": "gemm", "args": { - "m": 43690, - "n": 7168, - "k": 16384, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 4096, + "n": 384, + "k": 3072, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/MiniMax-M2.5-NVFP4/experts.gate_up_proj" + ] }, { "type": "gemm", "args": { - "m": 43690, + "m": 4096, "n": 384, - "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "k": 3072, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 32 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 1, + 32 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "amd/MiniMax-M2.5-MXFP4/experts.gate_up_proj" + ] }, { "type": "gemm", "args": { - "m": 43690, - "n": 7168, - "k": 384, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 4096, + "n": 384, + "k": 3072, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "MiniMaxAI/MiniMax-M2.5/experts.gate_up_proj", + "MiniMaxAI/MiniMax-M2.5/q_proj" + ] }, { "type": "gemm", "args": { - "m": 42, - "n": 1536, - "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 4096, + "n": 512, + "k": 2880, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/gpt-oss-120b-w-mxfp4-a-fp8/k_proj", + "amd/gpt-oss-120b-w-mxfp4-a-fp8/q_proj", + "amd/gpt-oss-120b-w-mxfp4-a-fp8/v_proj", + "openai/gpt-oss-120b/k_proj", + "openai/gpt-oss-120b/q_proj", + "openai/gpt-oss-120b/v_proj" + ] }, { "type": "gemm", "args": { - "m": 42, - "n": 24576, - "k": 1536, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 4096, + "n": 512, + "k": 3072, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/MiniMax-M2.5-MXFP4/k_proj", + "amd/MiniMax-M2.5-MXFP4/v_proj", + "nvidia/MiniMax-M2.5-NVFP4/k_proj", + "nvidia/MiniMax-M2.5-NVFP4/v_proj" + ] }, { "type": "gemm", "args": { - "m": 42, - "n": 576, - "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 4096, + "n": 512, + "k": 3072, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "MiniMaxAI/MiniMax-M2.5/k_proj", + "MiniMaxAI/MiniMax-M2.5/v_proj" + ] }, { "type": "gemm", "args": { - "m": 42, - "n": 32768, - "k": 512, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 4096, + "n": 512, + "k": 4096, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "Qwen/Qwen3.5-397B-A17B/k_proj", + "Qwen/Qwen3.5-397B-A17B/q_proj", + "Qwen/Qwen3.5-397B-A17B/shared_expert.gate_up_proj", + "Qwen/Qwen3.5-397B-A17B/v_proj", + "amd/Qwen3.5-397B-A17B-MXFP4/k_proj", + "amd/Qwen3.5-397B-A17B-MXFP4/q_proj", + "amd/Qwen3.5-397B-A17B-MXFP4/shared_expert.gate_up_proj", + "amd/Qwen3.5-397B-A17B-MXFP4/v_proj", + "nvidia/Qwen3.5-397B-A17B-NVFP4/k_proj", + "nvidia/Qwen3.5-397B-A17B-NVFP4/q_proj", + "nvidia/Qwen3.5-397B-A17B-NVFP4/shared_expert.gate_up_proj", + "nvidia/Qwen3.5-397B-A17B-NVFP4/v_proj" + ] }, { "type": "gemm", "args": { - "m": 42, - "n": 7168, - "k": 16384, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 4096, + "n": 512, + "k": 4096, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "Qwen/Qwen3.5-397B-A17B-FP8/k_proj", + "Qwen/Qwen3.5-397B-A17B-FP8/q_proj", + "Qwen/Qwen3.5-397B-A17B-FP8/shared_expert.gate_up_proj", + "Qwen/Qwen3.5-397B-A17B-FP8/v_proj", + "deepseek-ai/DeepSeek-V4.1-Flash/attn.wo_a" + ] }, { "type": "gemm", "args": { - "m": 42, - "n": 384, - "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 4096, + "n": 576, + "k": 6144, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/GLM-5.1-MXFP4/kv_a_proj_with_mqa", + "amd/GLM-5.2-MXFP4/kv_a_proj_with_mqa", + "nvidia/GLM-5-NVFP4/kv_a_proj_with_mqa", + "nvidia/GLM-5.1-NVFP4/kv_a_proj_with_mqa", + "nvidia/GLM-5.2-NVFP4/kv_a_proj_with_mqa" + ] }, { "type": "gemm", "args": { - "m": 42, - "n": 7168, - "k": 384, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 4096, + "n": 576, + "k": 6144, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "zai-org/GLM-5-FP8/kv_a_proj_with_mqa", + "zai-org/GLM-5.1-FP8/kv_a_proj_with_mqa", + "zai-org/GLM-5.2-FP8/kv_a_proj_with_mqa" + ] }, { "type": "gemm", "args": { - "m": 87381, - "n": 1536, + "m": 4096, + "n": 576, "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/Kimi-K2.5-MXFP4/kv_a_proj_with_mqa", + "moonshotai/Kimi-K2.5/kv_a_proj_with_mqa", + "nvidia/DeepSeek-R1-0528-FP4-V2/kv_a_proj_with_mqa", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/kv_a_proj_with_mqa", + "nvidia/Kimi-K2.5-NVFP4/kv_a_proj_with_mqa", + "nvidia/Kimi-K2.6-NVFP4/kv_a_proj_with_mqa" + ] }, { "type": "gemm", "args": { - "m": 87381, - "n": 24576, - "k": 1536, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 4096, + "n": 576, + "k": 7168, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 32 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 1, + 32 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "amd/DeepSeek-R1-0528-MXFP4-Preview/kv_a_proj_with_mqa", + "amd/DeepSeek-R1-0528-MXFP4-v2/kv_a_proj_with_mqa", + "amd/DeepSeek-R1-0528-MXFP4/kv_a_proj_with_mqa" + ] }, { "type": "gemm", "args": { - "m": 87381, + "m": 4096, "n": 576, "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/kv_a_proj_with_mqa" + ] }, { "type": "gemm", "args": { - "m": 87381, - "n": 32768, - "k": 512, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 4096, + "n": 768, + "k": 3072, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/MiniMax-M2.5-NVFP4/experts.gate_up_proj" + ] }, { "type": "gemm", "args": { - "m": 87381, - "n": 7168, - "k": 16384, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 4096, + "n": 768, + "k": 3072, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 32 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 1, + 32 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "amd/MiniMax-M2.5-MXFP4/experts.gate_up_proj" + ] }, { "type": "gemm", "args": { - "m": 87381, - "n": 384, - "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 4096, + "n": 768, + "k": 3072, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "MiniMaxAI/MiniMax-M2.5/experts.gate_up_proj", + "MiniMaxAI/MiniMax-M2.5/q_proj" + ] }, { "type": "gemm", "args": { - "m": 87381, - "n": 7168, - "k": 384, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 4096, + "n": 1024, + "k": 2880, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/gpt-oss-120b-w-mxfp4-a-fp8/q_proj", + "openai/gpt-oss-120b/q_proj" + ] }, { "type": "gemm", "args": { - "m": 85, - "n": 1536, - "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 4096, + "n": 1024, + "k": 3072, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/MiniMax-M2.5-MXFP4/k_proj", + "amd/MiniMax-M2.5-MXFP4/qkv_proj", + "amd/MiniMax-M2.5-MXFP4/v_proj", + "nvidia/MiniMax-M2.5-NVFP4/k_proj", + "nvidia/MiniMax-M2.5-NVFP4/qkv_proj", + "nvidia/MiniMax-M2.5-NVFP4/v_proj" + ] }, { "type": "gemm", "args": { - "m": 85, - "n": 24576, - "k": 1536, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 4096, + "n": 1024, + "k": 3072, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "MiniMaxAI/MiniMax-M2.5/k_proj", + "MiniMaxAI/MiniMax-M2.5/qkv_proj", + "MiniMaxAI/MiniMax-M2.5/v_proj" + ] }, { "type": "gemm", "args": { - "m": 85, - "n": 576, - "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 4096, + "n": 1024, + "k": 4096, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "Qwen/Qwen3.5-397B-A17B/q_proj", + "Qwen/Qwen3.5-397B-A17B/shared_expert.gate_up_proj", + "amd/Qwen3.5-397B-A17B-MXFP4/q_proj", + "amd/Qwen3.5-397B-A17B-MXFP4/shared_expert.gate_up_proj", + "nvidia/Qwen3.5-397B-A17B-NVFP4/q_proj", + "nvidia/Qwen3.5-397B-A17B-NVFP4/shared_expert.gate_up_proj" + ] }, { "type": "gemm", "args": { - "m": 85, - "n": 32768, - "k": 512, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 4096, + "n": 1024, + "k": 4096, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "Qwen/Qwen3.5-397B-A17B-FP8/q_proj", + "Qwen/Qwen3.5-397B-A17B-FP8/shared_expert.gate_up_proj", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_a", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_a", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_a" + ] }, { "type": "gemm", "args": { - "m": 85, - "n": 7168, - "k": 16384, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 4096, + "n": 1088, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] }, { "type": "gemm", "args": { - "m": 85, - "n": 384, + "m": 4096, + "n": 1152, "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/gate_up_proj" + ] }, { "type": "gemm", "args": { - "m": 85, - "n": 7168, - "k": 384, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 4096, + "n": 1536, + "k": 1536, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/Kimi-K2.5-MXFP4/q_b_proj", + "moonshotai/Kimi-K2.5/q_b_proj", + "nvidia/DeepSeek-R1-0528-FP4-V2/q_b_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/q_b_proj", + "nvidia/Kimi-K2.5-NVFP4/q_b_proj", + "nvidia/Kimi-K2.6-NVFP4/q_b_proj" + ] }, { "type": "gemm", "args": { - "m": 4194304, - "n": 6144, + "m": 4096, + "n": 1536, "k": 1536, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/q_b_proj" + ] }, { "type": "gemm", "args": { - "m": 4194304, - "n": 8192, - "k": 512, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 4096, + "n": 1536, + "k": 3072, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 32 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 1, + 32 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "amd/MiniMax-M2.5-MXFP4/experts.gate_up_proj" + ] }, { "type": "gemm", "args": { - "m": 4194304, - "n": 7168, - "k": 4096, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 4096, + "n": 1536, + "k": 3072, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "MiniMaxAI/MiniMax-M2.5/experts.gate_up_proj", + "MiniMaxAI/MiniMax-M2.5/q_proj" + ] }, { "type": "gemm", "args": { - "m": 4194304, - "n": 4608, - "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 4096, + "n": 1536, + "k": 6144, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/GLM-5-NVFP4/gate_up_proj" + ] }, { "type": "gemm", "args": { - "m": 4194304, - "n": 7168, - "k": 4608, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 4096, + "n": 1536, + "k": 6144, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "zai-org/GLM-5-FP8/gate_up_proj", + "zai-org/GLM-5.1-FP8/gate_up_proj", + "zai-org/GLM-5.2-FP8/gate_up_proj" + ] }, { "type": "gemm", "args": { - "m": 512, - "n": 6144, - "k": 1536, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 4096, + "n": 1536, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/Kimi-K2.5-MXFP4/q_a_proj", + "moonshotai/Kimi-K2.5/q_a_proj", + "nvidia/DeepSeek-R1-0528-FP4-V2/q_a_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/q_a_proj", + "nvidia/Kimi-K2.5-NVFP4/q_a_proj", + "nvidia/Kimi-K2.6-NVFP4/q_a_proj" + ] }, { "type": "gemm", "args": { - "m": 512, - "n": 8192, - "k": 512, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 4096, + "n": 1536, + "k": 7168, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 32 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 1, + 32 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "amd/DeepSeek-R1-0528-MXFP4-Preview/q_a_proj", + "amd/DeepSeek-R1-0528-MXFP4-v2/q_a_proj", + "amd/DeepSeek-R1-0528-MXFP4/q_a_proj" + ] }, { "type": "gemm", "args": { - "m": 512, - "n": 7168, - "k": 4096, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 4096, + "n": 1536, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/gate_up_proj", + "deepseek-ai/DeepSeek-R1-0528/q_a_proj", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w3" + ] }, { "type": "gemm", "args": { - "m": 16777216, + "m": 4096, "n": 1536, "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w3" + ] }, { "type": "gemm", "args": { - "m": 16777216, - "n": 6144, - "k": 1536, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 4096, + "n": 2048, + "k": 512, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/Kimi-K2.5-MXFP4/kv_b_proj", + "moonshotai/Kimi-K2.5/kv_b_proj", + "nvidia/DeepSeek-R1-0528-FP4-V2/kv_b_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/kv_b_proj", + "nvidia/Kimi-K2.5-NVFP4/kv_b_proj", + "nvidia/Kimi-K2.6-NVFP4/kv_b_proj" + ] }, { "type": "gemm", "args": { - "m": 16777216, - "n": 576, - "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 4096, + "n": 2048, + "k": 512, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/kv_b_proj" + ] }, { "type": "gemm", "args": { - "m": 16777216, - "n": 8192, - "k": 512, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 4096, + "n": 2048, + "k": 2048, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/GLM-5.1-MXFP4/q_b_proj", + "amd/GLM-5.2-MXFP4/q_b_proj", + "nvidia/GLM-5-NVFP4/q_b_proj", + "nvidia/GLM-5.1-NVFP4/q_b_proj", + "nvidia/GLM-5.2-NVFP4/q_b_proj" + ] }, { "type": "gemm", "args": { - "m": 16777216, - "n": 7168, - "k": 4096, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 4096, + "n": 2048, + "k": 2048, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "zai-org/GLM-5-FP8/q_b_proj", + "zai-org/GLM-5.1-FP8/q_b_proj", + "zai-org/GLM-5.2-FP8/q_b_proj" + ] }, { "type": "gemm", "args": { - "m": 16777216, - "n": 4608, - "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 4096, + "n": 2048, + "k": 2880, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/gpt-oss-120b-w-mxfp4-a-fp8/q_proj", + "openai/gpt-oss-120b/q_proj" + ] }, { "type": "gemm", "args": { - "m": 16777216, - "n": 7168, - "k": 4608, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 4096, + "n": 2048, + "k": 4096, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "Qwen/Qwen3.5-397B-A17B/q_proj", + "Qwen/Qwen3.5-397B-A17B/shared_expert.gate_up_proj", + "amd/Qwen3.5-397B-A17B-MXFP4/q_proj", + "amd/Qwen3.5-397B-A17B-MXFP4/shared_expert.gate_up_proj", + "nvidia/Qwen3.5-397B-A17B-NVFP4/q_proj", + "nvidia/Qwen3.5-397B-A17B-NVFP4/shared_expert.gate_up_proj" + ] }, { "type": "gemm", "args": { - "m": 2048, - "n": 6144, - "k": 1536, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 4096, + "n": 2048, + "k": 4096, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "Qwen/Qwen3.5-397B-A17B-FP8/q_proj", + "Qwen/Qwen3.5-397B-A17B-FP8/shared_expert.gate_up_proj" + ] }, { "type": "gemm", "args": { - "m": 2048, - "n": 8192, - "k": 512, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 4096, + "n": 2048, + "k": 6144, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/GLM-5.1-MXFP4/q_a_proj", + "amd/GLM-5.2-MXFP4/q_a_proj", + "amd/MiniMax-M3-MXFP4/q_proj", + "nvidia/GLM-5-NVFP4/q_a_proj", + "nvidia/GLM-5.1-NVFP4/q_a_proj", + "nvidia/GLM-5.2-NVFP4/q_a_proj" + ] }, { "type": "gemm", "args": { - "m": 2048, - "n": 7168, - "k": 4096, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 4096, + "n": 2048, + "k": 6144, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "zai-org/GLM-5-FP8/q_a_proj", + "zai-org/GLM-5.1-FP8/q_a_proj", + "zai-org/GLM-5.2-FP8/q_a_proj" + ] }, { "type": "gemm", "args": { - "m": 33554432, - "n": 1536, + "m": 4096, + "n": 2304, "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/gate_up_proj" + ] }, { "type": "gemm", "args": { - "m": 33554432, - "n": 6144, - "k": 1536, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 4096, + "n": 2880, + "k": 512, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/gpt-oss-120b-w-mxfp4-a-fp8/o_proj", + "openai/gpt-oss-120b/o_proj" + ] }, { "type": "gemm", "args": { - "m": 33554432, - "n": 576, - "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 4096, + "n": 2880, + "k": 1024, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/gpt-oss-120b-w-mxfp4-a-fp8/o_proj", + "openai/gpt-oss-120b/o_proj" + ] }, { "type": "gemm", "args": { - "m": 33554432, - "n": 8192, - "k": 512, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 4096, + "n": 2880, + "k": 2048, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/gpt-oss-120b-w-mxfp4-a-fp8/o_proj", + "openai/gpt-oss-120b/o_proj" + ] }, { "type": "gemm", "args": { - "m": 33554432, - "n": 7168, + "m": 4096, + "n": 2880, "k": 4096, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/gpt-oss-120b-w-mxfp4-a-fp8/o_proj", + "openai/gpt-oss-120b/o_proj" + ] }, { "type": "gemm", "args": { - "m": 33554432, - "n": 4608, - "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 4096, + "n": 3072, + "k": 192, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/MiniMax-M2.5-NVFP4/experts.down_proj" + ] }, { "type": "gemm", "args": { - "m": 33554432, - "n": 7168, - "k": 4608, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 4096, + "n": 3072, + "k": 192, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 32 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 1, + 32 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "amd/MiniMax-M2.5-MXFP4/experts.down_proj" + ] }, { "type": "gemm", "args": { - "m": 10, - "n": 384, - "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 4096, + "n": 3072, + "k": 192, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "MiniMaxAI/MiniMax-M2.5/experts.down_proj" + ] }, { "type": "gemm", "args": { - "m": 10, - "n": 7168, + "m": 4096, + "n": 3072, "k": 384, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/MiniMax-M2.5-NVFP4/experts.down_proj" + ] }, { "type": "gemm", "args": { - "m": 349525, - "n": 1536, - "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 349525, - "n": 24576, - "k": 1536, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 4096, + "n": 3072, + "k": 384, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 32 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 1, + 32 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "amd/MiniMax-M2.5-MXFP4/experts.down_proj" + ] }, { "type": "gemm", "args": { - "m": 349525, - "n": 576, - "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 4096, + "n": 3072, + "k": 384, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "MiniMaxAI/MiniMax-M2.5/experts.down_proj", + "MiniMaxAI/MiniMax-M2.5/o_proj" + ] }, { "type": "gemm", "args": { - "m": 349525, - "n": 32768, - "k": 512, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 4096, + "n": 3072, + "k": 768, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/MiniMax-M2.5-MXFP4/o_proj", + "nvidia/MiniMax-M2.5-NVFP4/o_proj" + ] }, { "type": "gemm", "args": { - "m": 349525, - "n": 7168, - "k": 16384, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 4096, + "n": 3072, + "k": 768, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 32 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 1, + 32 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "amd/MiniMax-M2.5-MXFP4/experts.down_proj" + ] }, { "type": "gemm", "args": { - "m": 349525, - "n": 384, - "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 4096, + "n": 3072, + "k": 768, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "MiniMaxAI/MiniMax-M2.5/experts.down_proj", + "MiniMaxAI/MiniMax-M2.5/o_proj" + ] }, { "type": "gemm", "args": { - "m": 349525, - "n": 7168, - "k": 384, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 4096, + "n": 3072, + "k": 1536, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/Kimi-K2.5-MXFP4/q_b_proj", + "amd/MiniMax-M2.5-MXFP4/o_proj", + "moonshotai/Kimi-K2.5/q_b_proj", + "nvidia/DeepSeek-R1-0528-FP4-V2/q_b_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/q_b_proj", + "nvidia/Kimi-K2.5-NVFP4/q_b_proj", + "nvidia/Kimi-K2.6-NVFP4/q_b_proj", + "nvidia/MiniMax-M2.5-NVFP4/o_proj" + ] }, { "type": "gemm", "args": { - "m": 699050, - "n": 1536, - "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 4096, + "n": 3072, + "k": 1536, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/MiniMax-M2.5-NVFP4/experts.down_proj" + ] }, { "type": "gemm", "args": { - "m": 699050, - "n": 24576, + "m": 4096, + "n": 3072, "k": 1536, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 32 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 1, + 32 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "amd/DeepSeek-R1-0528-MXFP4-Preview/q_b_proj", + "amd/DeepSeek-R1-0528-MXFP4-v2/q_b_proj", + "amd/DeepSeek-R1-0528-MXFP4/q_b_proj", + "amd/MiniMax-M2.5-MXFP4/experts.down_proj" + ] }, { "type": "gemm", "args": { - "m": 699050, - "n": 576, - "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 4096, + "n": 3072, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "MiniMaxAI/MiniMax-M2.5/experts.down_proj", + "MiniMaxAI/MiniMax-M2.5/o_proj", + "deepseek-ai/DeepSeek-R1-0528/q_b_proj" + ] }, { "type": "gemm", "args": { - "m": 699050, - "n": 32768, - "k": 512, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 4096, + "n": 3072, + "k": 3072, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/MiniMax-M2.5-MXFP4/o_proj", + "amd/MiniMax-M2.5-MXFP4/q_proj", + "nvidia/MiniMax-M2.5-NVFP4/o_proj", + "nvidia/MiniMax-M2.5-NVFP4/q_proj" + ] }, { "type": "gemm", "args": { - "m": 699050, - "n": 7168, - "k": 16384, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 4096, + "n": 3072, + "k": 3072, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/MiniMax-M2.5-NVFP4/experts.gate_up_proj" + ] }, { "type": "gemm", "args": { - "m": 699050, - "n": 384, - "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 4096, + "n": 3072, + "k": 3072, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 32 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 1, + 32 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "amd/MiniMax-M2.5-MXFP4/experts.gate_up_proj" + ] }, { "type": "gemm", "args": { - "m": 699050, - "n": 7168, - "k": 384, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 4096, + "n": 3072, + "k": 3072, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "MiniMaxAI/MiniMax-M2.5/experts.gate_up_proj", + "MiniMaxAI/MiniMax-M2.5/o_proj", + "MiniMaxAI/MiniMax-M2.5/q_proj" + ] }, { "type": "gemm", "args": { - "m": 524288, - "n": 7168, - "k": 576, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 4096, + "n": 3072, + "k": 6144, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/GLM-5.1-MXFP4/gate_up_proj", + "amd/GLM-5.2-MXFP4/gate_up_proj", + "amd/MiniMax-M2.5-MXFP4/o_proj", + "amd/MiniMax-M3-MXFP4/gate_up_proj", + "nvidia/GLM-5.1-NVFP4/gate_up_proj", + "nvidia/GLM-5.2-NVFP4/gate_up_proj", + "nvidia/MiniMax-M2.5-NVFP4/o_proj" + ] }, { "type": "gemm", "args": { - "m": 1651712, - "n": 1536, - "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 4096, + "n": 3072, + "k": 6144, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/GLM-5-NVFP4/gate_up_proj" + ] }, { "type": "gemm", "args": { - "m": 1651712, - "n": 24576, - "k": 1536, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 4096, + "n": 3072, + "k": 6144, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "MiniMaxAI/MiniMax-M2.5/o_proj", + "zai-org/GLM-5-FP8/gate_up_proj", + "zai-org/GLM-5.1-FP8/gate_up_proj", + "zai-org/GLM-5.2-FP8/gate_up_proj" + ] }, { "type": "gemm", "args": { - "m": 1651712, - "n": 576, - "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 4096, + "n": 3584, + "k": 512, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/GLM-5.1-MXFP4/kv_b_proj", + "amd/GLM-5.2-MXFP4/kv_b_proj", + "nvidia/GLM-5-NVFP4/kv_b_proj", + "nvidia/GLM-5.1-NVFP4/kv_b_proj", + "nvidia/GLM-5.2-NVFP4/kv_b_proj" + ] }, { "type": "gemm", "args": { - "m": 1651712, - "n": 32768, + "m": 4096, + "n": 3584, "k": 512, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "zai-org/GLM-5-FP8/kv_b_proj", + "zai-org/GLM-5.1-FP8/kv_b_proj", + "zai-org/GLM-5.2-FP8/kv_b_proj" + ] }, { "type": "gemm", "args": { - "m": 1651712, - "n": 7168, - "k": 16384, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 4096, + "n": 4096, + "k": 128, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "Qwen/Qwen3.5-397B-A17B/shared_expert.down_proj", + "amd/Qwen3.5-397B-A17B-MXFP4/shared_expert.down_proj", + "nvidia/Qwen3.5-397B-A17B-NVFP4/shared_expert.down_proj" + ] }, { "type": "gemm", "args": { - "m": 1651712, - "n": 9216, - "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 4096, + "n": 4096, + "k": 128, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "Qwen/Qwen3.5-397B-A17B-FP8/shared_expert.down_proj" + ] }, { "type": "gemm", "args": { - "m": 1651712, - "n": 7168, - "k": 9216, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 4096, + "n": 4096, + "k": 256, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "Qwen/Qwen3.5-397B-A17B/shared_expert.down_proj", + "amd/Qwen3.5-397B-A17B-MXFP4/shared_expert.down_proj", + "nvidia/Qwen3.5-397B-A17B-NVFP4/shared_expert.down_proj" + ] }, { "type": "gemm", "args": { - "m": 3226, - "n": 1536, - "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 4096, + "n": 4096, + "k": 256, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "Qwen/Qwen3.5-397B-A17B-FP8/shared_expert.down_proj" + ] }, { "type": "gemm", "args": { - "m": 3226, - "n": 24576, - "k": 1536, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 4096, + "n": 4096, + "k": 512, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "Qwen/Qwen3.5-397B-A17B/o_proj", + "Qwen/Qwen3.5-397B-A17B/shared_expert.down_proj", + "amd/Kimi-K2.5-MXFP4/kv_b_proj", + "amd/Qwen3.5-397B-A17B-MXFP4/o_proj", + "amd/Qwen3.5-397B-A17B-MXFP4/shared_expert.down_proj", + "moonshotai/Kimi-K2.5/kv_b_proj", + "nvidia/DeepSeek-R1-0528-FP4-V2/kv_b_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/kv_b_proj", + "nvidia/Kimi-K2.5-NVFP4/kv_b_proj", + "nvidia/Kimi-K2.6-NVFP4/kv_b_proj", + "nvidia/Qwen3.5-397B-A17B-NVFP4/o_proj", + "nvidia/Qwen3.5-397B-A17B-NVFP4/shared_expert.down_proj" + ] }, { "type": "gemm", "args": { - "m": 3226, - "n": 576, - "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 4096, + "n": 4096, + "k": 512, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 32 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 1, + 32 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "amd/DeepSeek-R1-0528-MXFP4-Preview/kv_b_proj", + "amd/DeepSeek-R1-0528-MXFP4-v2/kv_b_proj", + "amd/DeepSeek-R1-0528-MXFP4/kv_b_proj", + "amd/Qwen3.5-397B-A17B-MXFP4-AttnFP8-V2/shared_expert.down_proj" + ] }, { "type": "gemm", "args": { - "m": 3226, - "n": 32768, + "m": 4096, + "n": 4096, "k": 512, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "Qwen/Qwen3.5-397B-A17B-FP8/o_proj", + "Qwen/Qwen3.5-397B-A17B-FP8/shared_expert.down_proj", + "deepseek-ai/DeepSeek-R1-0528/kv_b_proj" + ] }, { "type": "gemm", "args": { - "m": 3226, - "n": 7168, - "k": 16384, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 4096, + "n": 4096, + "k": 1024, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "Qwen/Qwen3.5-397B-A17B/o_proj", + "Qwen/Qwen3.5-397B-A17B/shared_expert.down_proj", + "amd/Qwen3.5-397B-A17B-MXFP4/o_proj", + "amd/Qwen3.5-397B-A17B-MXFP4/shared_expert.down_proj", + "nvidia/Qwen3.5-397B-A17B-NVFP4/o_proj", + "nvidia/Qwen3.5-397B-A17B-NVFP4/shared_expert.down_proj" + ] }, { "type": "gemm", "args": { - "m": 3226, - "n": 9216, - "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 4096, + "n": 4096, + "k": 1024, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "Qwen/Qwen3.5-397B-A17B-FP8/o_proj", + "Qwen/Qwen3.5-397B-A17B-FP8/shared_expert.down_proj" + ] }, { "type": "gemm", "args": { - "m": 3226, - "n": 7168, - "k": 9216, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 4096, + "n": 4096, + "k": 2048, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "Qwen/Qwen3.5-397B-A17B/o_proj", + "amd/GLM-5.1-MXFP4/q_b_proj", + "amd/GLM-5.2-MXFP4/q_b_proj", + "amd/Qwen3.5-397B-A17B-MXFP4/o_proj", + "nvidia/GLM-5-NVFP4/q_b_proj", + "nvidia/GLM-5.1-NVFP4/q_b_proj", + "nvidia/GLM-5.2-NVFP4/q_b_proj", + "nvidia/Qwen3.5-397B-A17B-NVFP4/o_proj" + ] }, { "type": "gemm", "args": { - "m": 825856, - "n": 1536, - "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 4096, + "n": 4096, + "k": 2048, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "Qwen/Qwen3.5-397B-A17B-FP8/o_proj", + "zai-org/GLM-5-FP8/q_b_proj", + "zai-org/GLM-5.1-FP8/q_b_proj", + "zai-org/GLM-5.2-FP8/q_b_proj" + ] }, { "type": "gemm", "args": { - "m": 825856, - "n": 24576, - "k": 1536, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 4096, + "n": 4096, + "k": 2880, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/gpt-oss-120b-w-mxfp4-a-fp8/q_proj", + "openai/gpt-oss-120b/q_proj" + ] }, { "type": "gemm", "args": { - "m": 825856, - "n": 576, - "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 4096, + "n": 4096, + "k": 4096, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "Qwen/Qwen3.5-397B-A17B/o_proj", + "Qwen/Qwen3.5-397B-A17B/q_proj", + "amd/Qwen3.5-397B-A17B-MXFP4/o_proj", + "amd/Qwen3.5-397B-A17B-MXFP4/q_proj", + "nvidia/Qwen3.5-397B-A17B-NVFP4/o_proj", + "nvidia/Qwen3.5-397B-A17B-NVFP4/q_proj" + ] }, { "type": "gemm", "args": { - "m": 825856, - "n": 32768, - "k": 512, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 4096, + "n": 4096, + "k": 4096, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "Qwen/Qwen3.5-397B-A17B-FP8/o_proj", + "Qwen/Qwen3.5-397B-A17B-FP8/q_proj" + ] }, { "type": "gemm", "args": { - "m": 825856, - "n": 7168, - "k": 16384, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 4096, + "n": 4608, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "moonshotai/Kimi-K2.5/gate_up_proj", + "nvidia/Kimi-K2.6-NVFP4/gate_up_proj" + ] }, { "type": "gemm", "args": { - "m": 825856, + "m": 4096, "n": 4608, "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/gate_up_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/gate_up_proj", + "nvidia/Kimi-K2.5-NVFP4/gate_up_proj" + ] }, { "type": "gemm", "args": { - "m": 825856, - "n": 7168, - "k": 4608, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 4096, + "n": 4608, + "k": 7168, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 32 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 1, + 32 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "amd/DeepSeek-R1-0528-MXFP4-Preview/gate_up_proj", + "amd/DeepSeek-R1-0528-MXFP4-v2/gate_up_proj", + "amd/DeepSeek-R1-0528-MXFP4/gate_up_proj", + "amd/Kimi-K2.5-MXFP4/gate_up_proj" + ] }, { "type": "gemm", "args": { - "m": 1613, - "n": 1536, + "m": 4096, + "n": 4608, "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/gate_up_proj" + ] }, { "type": "gemm", "args": { - "m": 1613, - "n": 24576, + "m": 4096, + "n": 6144, "k": 1536, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/GLM-5.1-MXFP4/down_proj", + "amd/GLM-5.2-MXFP4/down_proj", + "amd/Kimi-K2.5-MXFP4/q_b_proj", + "amd/MiniMax-M3-MXFP4/down_proj", + "moonshotai/Kimi-K2.5/q_b_proj", + "nvidia/DeepSeek-R1-0528-FP4-V2/q_b_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/q_b_proj", + "nvidia/GLM-5.1-NVFP4/down_proj", + "nvidia/GLM-5.2-NVFP4/down_proj", + "nvidia/Kimi-K2.5-NVFP4/q_b_proj", + "nvidia/Kimi-K2.6-NVFP4/q_b_proj" + ] }, { "type": "gemm", "args": { - "m": 1613, - "n": 576, - "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 4096, + "n": 6144, + "k": 1536, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/GLM-5-NVFP4/down_proj" + ] }, { "type": "gemm", "args": { - "m": 1613, - "n": 32768, - "k": 512, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 4096, + "n": 6144, + "k": 1536, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 32 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 1, + 32 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "amd/DeepSeek-R1-0528-MXFP4-Preview/q_b_proj", + "amd/DeepSeek-R1-0528-MXFP4-v2/q_b_proj", + "amd/DeepSeek-R1-0528-MXFP4/q_b_proj" + ] }, { "type": "gemm", "args": { - "m": 1613, - "n": 7168, - "k": 16384, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 4096, + "n": 6144, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/q_b_proj", + "zai-org/GLM-5-FP8/down_proj", + "zai-org/GLM-5.1-FP8/down_proj", + "zai-org/GLM-5.2-FP8/down_proj" + ] }, { "type": "gemm", "args": { - "m": 1613, - "n": 4608, - "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 4096, + "n": 6144, + "k": 2048, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/GLM-5.1-MXFP4/o_proj", + "amd/GLM-5.2-MXFP4/o_proj", + "amd/MiniMax-M3-MXFP4/o_proj", + "nvidia/GLM-5-NVFP4/o_proj", + "nvidia/GLM-5.1-NVFP4/o_proj", + "nvidia/GLM-5.2-NVFP4/o_proj" + ] }, { "type": "gemm", "args": { - "m": 1613, - "n": 7168, - "k": 4608, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 4096, + "n": 6144, + "k": 2048, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "zai-org/GLM-5-FP8/o_proj", + "zai-org/GLM-5.1-FP8/o_proj", + "zai-org/GLM-5.2-FP8/o_proj" + ] }, { "type": "gemm", "args": { - "m": 170496, - "n": 9216, - "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 4096, + "n": 6144, + "k": 3072, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/GLM-5.1-MXFP4/down_proj", + "amd/GLM-5.2-MXFP4/down_proj", + "amd/MiniMax-M2.5-MXFP4/q_proj", + "amd/MiniMax-M3-MXFP4/down_proj", + "nvidia/GLM-5.1-NVFP4/down_proj", + "nvidia/GLM-5.2-NVFP4/down_proj", + "nvidia/MiniMax-M2.5-NVFP4/q_proj" + ] }, { "type": "gemm", "args": { - "m": 170496, - "n": 7168, - "k": 9216, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 4096, + "n": 6144, + "k": 3072, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/GLM-5-NVFP4/down_proj" + ] }, { "type": "gemm", "args": { - "m": 333, - "n": 9216, - "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 4096, + "n": 6144, + "k": 3072, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "MiniMaxAI/MiniMax-M2.5/q_proj", + "zai-org/GLM-5-FP8/down_proj", + "zai-org/GLM-5.1-FP8/down_proj", + "zai-org/GLM-5.2-FP8/down_proj" + ] }, { "type": "gemm", "args": { - "m": 333, - "n": 7168, - "k": 9216, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 4096, + "n": 6144, + "k": 4096, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/GLM-5.1-MXFP4/o_proj", + "amd/GLM-5.2-MXFP4/o_proj", + "amd/MiniMax-M3-MXFP4/o_proj", + "nvidia/GLM-5-NVFP4/o_proj", + "nvidia/GLM-5.1-NVFP4/o_proj", + "nvidia/GLM-5.2-NVFP4/o_proj" + ] }, { "type": "gemm", "args": { - "m": 10656, - "n": 1536, - "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 4096, + "n": 6144, + "k": 4096, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "zai-org/GLM-5-FP8/o_proj", + "zai-org/GLM-5.1-FP8/o_proj", + "zai-org/GLM-5.2-FP8/o_proj" + ] }, { "type": "gemm", "args": { - "m": 10656, - "n": 24576, - "k": 1536, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 4096, + "n": 6144, + "k": 6144, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/GLM-5.1-MXFP4/down_proj", + "amd/GLM-5.1-MXFP4/gate_up_proj", + "amd/GLM-5.2-MXFP4/down_proj", + "amd/GLM-5.2-MXFP4/gate_up_proj", + "amd/MiniMax-M3-MXFP4/gate_up_proj", + "nvidia/GLM-5.1-NVFP4/down_proj", + "nvidia/GLM-5.1-NVFP4/gate_up_proj", + "nvidia/GLM-5.2-NVFP4/down_proj", + "nvidia/GLM-5.2-NVFP4/gate_up_proj" + ] }, { "type": "gemm", "args": { - "m": 10656, - "n": 576, - "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 4096, + "n": 6144, + "k": 6144, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/GLM-5-NVFP4/down_proj", + "nvidia/GLM-5-NVFP4/gate_up_proj" + ] }, { "type": "gemm", "args": { - "m": 10656, - "n": 32768, - "k": 512, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 4096, + "n": 6144, + "k": 6144, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "zai-org/GLM-5-FP8/down_proj", + "zai-org/GLM-5-FP8/gate_up_proj", + "zai-org/GLM-5.1-FP8/down_proj", + "zai-org/GLM-5.1-FP8/gate_up_proj", + "zai-org/GLM-5.2-FP8/down_proj", + "zai-org/GLM-5.2-FP8/gate_up_proj" + ] }, { "type": "gemm", "args": { - "m": 10656, - "n": 7168, - "k": 16384, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 4096, + "n": 6144, + "k": 8192, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/GLM-5.1-MXFP4/o_proj", + "amd/GLM-5.2-MXFP4/o_proj", + "amd/MiniMax-M3-MXFP4/o_proj", + "nvidia/GLM-5-NVFP4/o_proj", + "nvidia/GLM-5.1-NVFP4/o_proj", + "nvidia/GLM-5.2-NVFP4/o_proj" + ] }, { "type": "gemm", "args": { - "m": 10656, + "m": 4096, "n": 7168, - "k": 576, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "k": 512, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/GLM-5.1-MXFP4/kv_b_proj", + "amd/GLM-5.2-MXFP4/kv_b_proj", + "amd/Kimi-K2.5-MXFP4/o_proj", + "moonshotai/Kimi-K2.5/o_proj", + "nvidia/GLM-5-NVFP4/kv_b_proj", + "nvidia/GLM-5.1-NVFP4/kv_b_proj", + "nvidia/GLM-5.2-NVFP4/kv_b_proj", + "nvidia/Kimi-K2.5-NVFP4/o_proj", + "nvidia/Kimi-K2.6-NVFP4/o_proj" + ] }, { "type": "gemm", "args": { - "m": 6144, - "n": 9216, - "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 4096, + "n": 7168, + "k": 512, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "zai-org/GLM-5-FP8/kv_b_proj", + "zai-org/GLM-5.1-FP8/kv_b_proj", + "zai-org/GLM-5.2-FP8/kv_b_proj" + ] }, { "type": "gemm", "args": { - "m": 6144, + "m": 4096, "n": 7168, - "k": 9216, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "k": 1024, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/Kimi-K2.5-MXFP4/o_proj", + "moonshotai/Kimi-K2.5/o_proj", + "nvidia/Kimi-K2.5-NVFP4/o_proj", + "nvidia/Kimi-K2.6-NVFP4/o_proj" + ] }, { "type": "gemm", "args": { - "m": 12288, - "n": 9216, - "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 4096, + "n": 7168, + "k": 1024, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/o_proj", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] }, { "type": "gemm", "args": { - "m": 12288, + "m": 4096, "n": 7168, - "k": 9216, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "k": 1152, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/down_proj" + ] }, { "type": "gemm", "args": { - "m": 24, - "n": 9216, - "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 4096, + "n": 7168, + "k": 2048, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/Kimi-K2.5-MXFP4/o_proj", + "moonshotai/Kimi-K2.5/o_proj", + "nvidia/Kimi-K2.5-NVFP4/o_proj", + "nvidia/Kimi-K2.6-NVFP4/o_proj" + ] }, { "type": "gemm", "args": { - "m": 24, + "m": 4096, "n": 7168, - "k": 9216, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "k": 2048, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/o_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/o_proj" + ] }, { "type": "gemm", "args": { - "m": 24576, - "n": 9216, - "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 4096, + "n": 7168, + "k": 2048, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 32 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 1, + 32 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "amd/DeepSeek-R1-0528-MXFP4-Preview/o_proj", + "amd/DeepSeek-R1-0528-MXFP4-v2/o_proj", + "amd/DeepSeek-R1-0528-MXFP4/o_proj" + ] }, { "type": "gemm", "args": { - "m": 24576, + "m": 4096, "n": 7168, - "k": 9216, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "k": 2048, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/o_proj", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] }, { "type": "gemm", "args": { - "m": 48, - "n": 9216, - "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 4096, + "n": 7168, + "k": 2304, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "moonshotai/Kimi-K2.5/down_proj", + "nvidia/Kimi-K2.6-NVFP4/down_proj" + ] }, { "type": "gemm", "args": { - "m": 48, + "m": 4096, "n": 7168, - "k": 9216, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "k": 2304, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/down_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/down_proj", + "nvidia/Kimi-K2.5-NVFP4/down_proj" + ] }, { "type": "gemm", "args": { - "m": 1153536, - "n": 9216, - "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 4096, + "n": 7168, + "k": 2304, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 32 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 1, + 32 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "amd/DeepSeek-R1-0528-MXFP4-Preview/down_proj", + "amd/DeepSeek-R1-0528-MXFP4-v2/down_proj", + "amd/DeepSeek-R1-0528-MXFP4/down_proj", + "amd/Kimi-K2.5-MXFP4/down_proj" + ] }, { "type": "gemm", "args": { - "m": 1153536, + "m": 4096, "n": 7168, - "k": 9216, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "k": 2304, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/down_proj" + ] }, { "type": "gemm", "args": { - "m": 2253, - "n": 1536, - "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 4096, + "n": 7168, + "k": 4096, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/o_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/o_proj" + ] }, { "type": "gemm", "args": { - "m": 2253, - "n": 24576, - "k": 1536, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 4096, + "n": 7168, + "k": 4096, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 32 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 1, + 32 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "amd/DeepSeek-R1-0528-MXFP4-Preview/o_proj", + "amd/DeepSeek-R1-0528-MXFP4-v2/o_proj", + "amd/DeepSeek-R1-0528-MXFP4/o_proj" + ] }, { "type": "gemm", "args": { - "m": 2253, - "n": 576, - "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 4096, + "n": 7168, + "k": 4096, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/o_proj", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] }, { "type": "gemm", "args": { - "m": 2253, - "n": 32768, - "k": 512, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 4096, + "n": 7168, + "k": 4608, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "moonshotai/Kimi-K2.5/down_proj", + "nvidia/Kimi-K2.6-NVFP4/down_proj" + ] }, { "type": "gemm", "args": { - "m": 2253, + "m": 4096, "n": 7168, - "k": 16384, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "k": 4608, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/down_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/down_proj", + "nvidia/Kimi-K2.5-NVFP4/down_proj" + ] }, { "type": "gemm", "args": { - "m": 2253, - "n": 9216, - "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 4096, + "n": 7168, + "k": 4608, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 32 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 1, + 32 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "amd/DeepSeek-R1-0528-MXFP4-Preview/down_proj", + "amd/DeepSeek-R1-0528-MXFP4-v2/down_proj", + "amd/DeepSeek-R1-0528-MXFP4/down_proj", + "amd/Kimi-K2.5-MXFP4/down_proj" + ] }, { "type": "gemm", "args": { - "m": 2253, + "m": 4096, "n": 7168, - "k": 9216, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "k": 4608, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/down_proj" + ] }, { "type": "gemm", "args": { - "m": 144192, - "n": 1536, - "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 4096, + "n": 7168, + "k": 8192, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/Kimi-K2.5-MXFP4/o_proj", + "moonshotai/Kimi-K2.5/o_proj", + "nvidia/Kimi-K2.5-NVFP4/o_proj", + "nvidia/Kimi-K2.6-NVFP4/o_proj" + ] }, { "type": "gemm", "args": { - "m": 144192, - "n": 24576, - "k": 1536, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 4096, + "n": 7168, + "k": 8192, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] }, { "type": "gemm", "args": { - "m": 144192, - "n": 576, - "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 4096, + "n": 7168, + "k": 9216, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/down_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/down_proj", + "nvidia/Kimi-K2.5-NVFP4/down_proj" + ] }, { "type": "gemm", "args": { - "m": 144192, - "n": 32768, - "k": 512, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 4096, + "n": 7168, + "k": 16384, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/o_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/o_proj" + ] }, { "type": "gemm", "args": { - "m": 144192, + "m": 4096, "n": 7168, "k": 16384, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 32 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 1, + 32 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "amd/DeepSeek-R1-0528-MXFP4-Preview/o_proj", + "amd/DeepSeek-R1-0528-MXFP4-v2/o_proj", + "amd/DeepSeek-R1-0528-MXFP4/o_proj" + ] }, { "type": "gemm", "args": { - "m": 144192, - "n": 1152, - "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 4096, + "n": 7168, + "k": 16384, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/o_proj", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] }, { "type": "gemm", "args": { - "m": 144192, + "m": 4096, "n": 7168, - "k": 1152, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "k": 16384, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] }, { "type": "gemm", "args": { - "m": 281, - "n": 1536, - "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 4096, + "n": 7168, + "k": 65536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] }, { "type": "gemm", "args": { - "m": 281, - "n": 24576, - "k": 1536, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 4096, + "n": 8192, + "k": 512, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/Kimi-K2.5-MXFP4/kv_b_proj", + "moonshotai/Kimi-K2.5/kv_b_proj", + "nvidia/DeepSeek-R1-0528-FP4-V2/kv_b_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/kv_b_proj", + "nvidia/Kimi-K2.5-NVFP4/kv_b_proj", + "nvidia/Kimi-K2.6-NVFP4/kv_b_proj" + ] }, { "type": "gemm", "args": { - "m": 281, - "n": 576, - "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 4096, + "n": 8192, + "k": 512, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 32 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 1, + 32 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "amd/DeepSeek-R1-0528-MXFP4-Preview/kv_b_proj", + "amd/DeepSeek-R1-0528-MXFP4-v2/kv_b_proj", + "amd/DeepSeek-R1-0528-MXFP4/kv_b_proj" + ] }, { "type": "gemm", "args": { - "m": 281, - "n": 32768, + "m": 4096, + "n": 8192, "k": 512, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/kv_b_proj" + ] }, { "type": "gemm", "args": { - "m": 281, - "n": 7168, - "k": 16384, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 4096, + "n": 8192, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_b" + ] }, { "type": "gemm", "args": { - "m": 281, - "n": 1152, - "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 4096, + "n": 8192, + "k": 2048, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/GLM-5.1-MXFP4/q_b_proj", + "amd/GLM-5.2-MXFP4/q_b_proj", + "nvidia/GLM-5-NVFP4/q_b_proj", + "nvidia/GLM-5.1-NVFP4/q_b_proj", + "nvidia/GLM-5.2-NVFP4/q_b_proj" + ] }, { "type": "gemm", "args": { - "m": 281, - "n": 7168, - "k": 1152, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 4096, + "n": 9216, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "moonshotai/Kimi-K2.5/gate_up_proj", + "nvidia/Kimi-K2.6-NVFP4/gate_up_proj" + ] }, { "type": "gemm", "args": { - "m": 629248, + "m": 4096, "n": 9216, "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/gate_up_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/gate_up_proj", + "nvidia/Kimi-K2.5-NVFP4/gate_up_proj" + ] }, { "type": "gemm", "args": { - "m": 629248, - "n": 7168, - "k": 9216, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 4096, + "n": 9216, + "k": 7168, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 32 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 1, + 32 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "amd/DeepSeek-R1-0528-MXFP4-Preview/gate_up_proj", + "amd/DeepSeek-R1-0528-MXFP4-v2/gate_up_proj", + "amd/DeepSeek-R1-0528-MXFP4/gate_up_proj", + "amd/Kimi-K2.5-MXFP4/gate_up_proj" + ] }, { "type": "gemm", "args": { - "m": 1229, - "n": 1536, + "m": 4096, + "n": 9216, "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/gate_up_proj" + ] }, { "type": "gemm", "args": { - "m": 1229, - "n": 24576, + "m": 4096, + "n": 12288, "k": 1536, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/Kimi-K2.5-MXFP4/q_b_proj", + "moonshotai/Kimi-K2.5/q_b_proj", + "nvidia/DeepSeek-R1-0528-FP4-V2/q_b_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/q_b_proj", + "nvidia/Kimi-K2.5-NVFP4/q_b_proj", + "nvidia/Kimi-K2.6-NVFP4/q_b_proj" + ] }, { "type": "gemm", "args": { - "m": 1229, - "n": 576, - "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 4096, + "n": 14336, + "k": 512, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/GLM-5.1-MXFP4/kv_b_proj", + "amd/GLM-5.2-MXFP4/kv_b_proj", + "nvidia/GLM-5-NVFP4/kv_b_proj", + "nvidia/GLM-5.1-NVFP4/kv_b_proj", + "nvidia/GLM-5.2-NVFP4/kv_b_proj" + ] }, { "type": "gemm", "args": { - "m": 1229, - "n": 32768, - "k": 512, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 4096, + "n": 15360, + "k": 1024, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] }, { "type": "gemm", "args": { - "m": 1229, - "n": 7168, - "k": 16384, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 4096, + "n": 16384, + "k": 512, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/Kimi-K2.5-MXFP4/kv_b_proj", + "moonshotai/Kimi-K2.5/kv_b_proj", + "nvidia/DeepSeek-R1-0528-FP4-V2/kv_b_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/kv_b_proj", + "nvidia/Kimi-K2.5-NVFP4/kv_b_proj", + "nvidia/Kimi-K2.6-NVFP4/kv_b_proj" + ] }, { "type": "gemm", "args": { - "m": 1229, - "n": 9216, - "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 4096, + "n": 16384, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_b" + ] }, { "type": "gemm", "args": { - "m": 1229, - "n": 7168, - "k": 9216, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 4096, + "n": 24576, + "k": 1536, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/q_b_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/q_b_proj" + ] }, { "type": "gemm", "args": { - "m": 39328, - "n": 1536, - "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 4096, + "n": 24576, + "k": 1536, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 32 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 1, + 32 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "amd/DeepSeek-R1-0528-MXFP4-Preview/q_b_proj", + "amd/DeepSeek-R1-0528-MXFP4-v2/q_b_proj", + "amd/DeepSeek-R1-0528-MXFP4/q_b_proj" + ] }, { "type": "gemm", "args": { - "m": 39328, + "m": 4096, "n": 24576, "k": 1536, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/q_b_proj" + ] }, { "type": "gemm", "args": { - "m": 39328, - "n": 576, - "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 4096, + "n": 30720, + "k": 1024, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] }, { "type": "gemm", "args": { - "m": 39328, + "m": 4096, "n": 32768, "k": 512, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/kv_b_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/kv_b_proj" + ] }, { "type": "gemm", "args": { - "m": 39328, - "n": 7168, - "k": 16384, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 4096, + "n": 32768, + "k": 512, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 32 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 1, + 32 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "amd/DeepSeek-R1-0528-MXFP4-Preview/kv_b_proj", + "amd/DeepSeek-R1-0528-MXFP4-v2/kv_b_proj", + "amd/DeepSeek-R1-0528-MXFP4/kv_b_proj" + ] }, { "type": "gemm", "args": { - "m": 39328, - "n": 7168, - "k": 576, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 4096, + "n": 32768, + "k": 512, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/kv_b_proj" + ] }, { "type": "gemm", "args": { - "m": 76, - "n": 24576, + "m": 4096, + "n": 65536, "k": 1536, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_b" + ] }, { "type": "gemm", "args": { - "m": 76, - "n": 32768, - "k": 512, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 4096, + "n": 122880, + "k": 1024, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] }, { "type": "gemm", "args": { - "m": 76, - "n": 7168, - "k": 16384, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 4608, + "n": 576, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/kv_a_proj_with_mqa" + ] }, { "type": "gemm", "args": { - "m": 76, - "n": 7168, - "k": 576, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 4608, + "n": 1152, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/gate_up_proj" + ] }, { "type": "gemm", "args": { - "m": 98304, - "n": 9216, + "m": 4608, + "n": 1536, "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/gate_up_proj", + "deepseek-ai/DeepSeek-R1-0528/q_a_proj", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w3" + ] }, { "type": "gemm", "args": { - "m": 98304, - "n": 7168, - "k": 9216, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 4608, + "n": 2304, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/gate_up_proj" + ] }, { "type": "gemm", "args": { - "m": 96, - "n": 24576, - "k": 1536, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 4608, + "n": 7168, + "k": 1152, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/down_proj" + ] }, { "type": "gemm", "args": { - "m": 96, - "n": 32768, - "k": 512, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 4608, + "n": 7168, + "k": 2304, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/down_proj" + ] }, { "type": "gemm", "args": { - "m": 96, + "m": 4608, "n": 7168, "k": 16384, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/o_proj", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] }, { "type": "gemm", "args": { - "m": 96, - "n": 9216, - "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 4608, + "n": 24576, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/q_b_proj" + ] }, { "type": "gemm", "args": { - "m": 96, - "n": 7168, - "k": 9216, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 4608, + "n": 32768, + "k": 512, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/kv_b_proj" + ] }, { "type": "gemm", "args": { - "m": 614, - "n": 9216, + "m": 5120, + "n": 576, "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/Kimi-K2.5-MXFP4/kv_a_proj_with_mqa", + "moonshotai/Kimi-K2.5/kv_a_proj_with_mqa", + "nvidia/DeepSeek-R1-0528-FP4-V2/kv_a_proj_with_mqa", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/kv_a_proj_with_mqa", + "nvidia/Kimi-K2.5-NVFP4/kv_a_proj_with_mqa", + "nvidia/Kimi-K2.6-NVFP4/kv_a_proj_with_mqa" + ] }, { "type": "gemm", "args": { - "m": 614, - "n": 7168, - "k": 9216, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 5120, + "n": 1536, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/Kimi-K2.5-MXFP4/q_a_proj", + "moonshotai/Kimi-K2.5/q_a_proj", + "nvidia/DeepSeek-R1-0528-FP4-V2/q_a_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/q_a_proj", + "nvidia/Kimi-K2.5-NVFP4/q_a_proj", + "nvidia/Kimi-K2.6-NVFP4/q_a_proj" + ] }, { "type": "gemm", "args": { - "m": 2202112, - "n": 9216, + "m": 5120, + "n": 2304, "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/gate_up_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/gate_up_proj", + "nvidia/Kimi-K2.5-NVFP4/gate_up_proj" + ] }, { "type": "gemm", "args": { - "m": 2202112, - "n": 7168, - "k": 9216, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 5120, + "n": 3072, + "k": 1536, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/Kimi-K2.5-MXFP4/q_b_proj", + "amd/MiniMax-M2.5-MXFP4/o_proj", + "moonshotai/Kimi-K2.5/q_b_proj", + "nvidia/DeepSeek-R1-0528-FP4-V2/q_b_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/q_b_proj", + "nvidia/Kimi-K2.5-NVFP4/q_b_proj", + "nvidia/Kimi-K2.6-NVFP4/q_b_proj", + "nvidia/MiniMax-M2.5-NVFP4/o_proj" + ] }, { "type": "gemm", "args": { - "m": 2150, - "n": 9216, - "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 5120, + "n": 3072, + "k": 1536, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/MiniMax-M2.5-NVFP4/experts.down_proj" + ] }, { "type": "gemm", "args": { - "m": 2150, - "n": 7168, - "k": 9216, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 5120, + "n": 3072, + "k": 1536, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 32 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 1, + 32 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "amd/DeepSeek-R1-0528-MXFP4-Preview/q_b_proj", + "amd/DeepSeek-R1-0528-MXFP4-v2/q_b_proj", + "amd/DeepSeek-R1-0528-MXFP4/q_b_proj", + "amd/MiniMax-M2.5-MXFP4/experts.down_proj" + ] }, { "type": "gemm", "args": { - "m": 1126, - "n": 9216, - "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 5120, + "n": 4096, + "k": 512, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "Qwen/Qwen3.5-397B-A17B/o_proj", + "Qwen/Qwen3.5-397B-A17B/shared_expert.down_proj", + "amd/Kimi-K2.5-MXFP4/kv_b_proj", + "amd/Qwen3.5-397B-A17B-MXFP4/o_proj", + "amd/Qwen3.5-397B-A17B-MXFP4/shared_expert.down_proj", + "moonshotai/Kimi-K2.5/kv_b_proj", + "nvidia/DeepSeek-R1-0528-FP4-V2/kv_b_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/kv_b_proj", + "nvidia/Kimi-K2.5-NVFP4/kv_b_proj", + "nvidia/Kimi-K2.6-NVFP4/kv_b_proj", + "nvidia/Qwen3.5-397B-A17B-NVFP4/o_proj", + "nvidia/Qwen3.5-397B-A17B-NVFP4/shared_expert.down_proj" + ] }, { "type": "gemm", "args": { - "m": 1126, - "n": 7168, - "k": 9216, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 5120, + "n": 4096, + "k": 512, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 32 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 1, + 32 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "amd/DeepSeek-R1-0528-MXFP4-Preview/kv_b_proj", + "amd/DeepSeek-R1-0528-MXFP4-v2/kv_b_proj", + "amd/DeepSeek-R1-0528-MXFP4/kv_b_proj", + "amd/Qwen3.5-397B-A17B-MXFP4-AttnFP8-V2/shared_expert.down_proj" + ] }, { "type": "gemm", "args": { - "m": 135168, - "n": 9216, + "m": 5120, + "n": 4608, "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 135168, - "n": 7168, - "k": 9216, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/gate_up_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/gate_up_proj", + "nvidia/Kimi-K2.5-NVFP4/gate_up_proj" + ] }, { "type": "gemm", "args": { - "m": 33, - "n": 9216, + "m": 5120, + "n": 4608, "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 32 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 1, + 32 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "amd/DeepSeek-R1-0528-MXFP4-Preview/gate_up_proj", + "amd/DeepSeek-R1-0528-MXFP4-v2/gate_up_proj", + "amd/DeepSeek-R1-0528-MXFP4/gate_up_proj", + "amd/Kimi-K2.5-MXFP4/gate_up_proj" + ] }, { "type": "gemm", "args": { - "m": 33, - "n": 7168, - "k": 9216, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 5120, + "n": 6144, + "k": 1536, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/GLM-5.1-MXFP4/down_proj", + "amd/GLM-5.2-MXFP4/down_proj", + "amd/Kimi-K2.5-MXFP4/q_b_proj", + "amd/MiniMax-M3-MXFP4/down_proj", + "moonshotai/Kimi-K2.5/q_b_proj", + "nvidia/DeepSeek-R1-0528-FP4-V2/q_b_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/q_b_proj", + "nvidia/GLM-5.1-NVFP4/down_proj", + "nvidia/GLM-5.2-NVFP4/down_proj", + "nvidia/Kimi-K2.5-NVFP4/q_b_proj", + "nvidia/Kimi-K2.6-NVFP4/q_b_proj" + ] }, { "type": "gemm", "args": { - "m": 270336, - "n": 1536, - "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 5120, + "n": 6144, + "k": 1536, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/GLM-5-NVFP4/down_proj" + ] }, { "type": "gemm", "args": { - "m": 270336, - "n": 3072, + "m": 5120, + "n": 6144, "k": 1536, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 32 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 1, + 32 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "amd/DeepSeek-R1-0528-MXFP4-Preview/q_b_proj", + "amd/DeepSeek-R1-0528-MXFP4-v2/q_b_proj", + "amd/DeepSeek-R1-0528-MXFP4/q_b_proj" + ] }, { "type": "gemm", "args": { - "m": 270336, - "n": 576, - "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 5120, + "n": 7168, + "k": 2048, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/Kimi-K2.5-MXFP4/o_proj", + "moonshotai/Kimi-K2.5/o_proj", + "nvidia/Kimi-K2.5-NVFP4/o_proj", + "nvidia/Kimi-K2.6-NVFP4/o_proj" + ] }, { "type": "gemm", "args": { - "m": 270336, - "n": 4096, - "k": 512, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 5120, + "n": 7168, + "k": 2048, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/o_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/o_proj" + ] }, { "type": "gemm", "args": { - "m": 270336, + "m": 5120, "n": 7168, "k": 2048, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 32 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 1, + 32 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "amd/DeepSeek-R1-0528-MXFP4-Preview/o_proj", + "amd/DeepSeek-R1-0528-MXFP4-v2/o_proj", + "amd/DeepSeek-R1-0528-MXFP4/o_proj" + ] }, { "type": "gemm", "args": { - "m": 270336, - "n": 2304, - "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 5120, + "n": 7168, + "k": 2304, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/down_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/down_proj", + "nvidia/Kimi-K2.5-NVFP4/down_proj" + ] }, { "type": "gemm", "args": { - "m": 270336, + "m": 5120, "n": 7168, "k": 2304, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 32 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 1, + 32 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "amd/DeepSeek-R1-0528-MXFP4-Preview/down_proj", + "amd/DeepSeek-R1-0528-MXFP4-v2/down_proj", + "amd/DeepSeek-R1-0528-MXFP4/down_proj", + "amd/Kimi-K2.5-MXFP4/down_proj" + ] }, { "type": "gemm", "args": { - "m": 66, - "n": 1536, - "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 5120, + "n": 7168, + "k": 4096, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/o_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/o_proj" + ] }, { "type": "gemm", "args": { - "m": 66, - "n": 3072, - "k": 1536, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 5120, + "n": 7168, + "k": 4096, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 32 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 1, + 32 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "amd/DeepSeek-R1-0528-MXFP4-Preview/o_proj", + "amd/DeepSeek-R1-0528-MXFP4-v2/o_proj", + "amd/DeepSeek-R1-0528-MXFP4/o_proj" + ] }, { "type": "gemm", "args": { - "m": 66, - "n": 576, - "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 5120, + "n": 7168, + "k": 4608, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/down_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/down_proj", + "nvidia/Kimi-K2.5-NVFP4/down_proj" + ] }, { "type": "gemm", "args": { - "m": 66, - "n": 4096, - "k": 512, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 5120, + "n": 7168, + "k": 4608, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 32 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 1, + 32 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "amd/DeepSeek-R1-0528-MXFP4-Preview/down_proj", + "amd/DeepSeek-R1-0528-MXFP4-v2/down_proj", + "amd/DeepSeek-R1-0528-MXFP4/down_proj", + "amd/Kimi-K2.5-MXFP4/down_proj" + ] }, { "type": "gemm", "args": { - "m": 66, + "m": 5120, "n": 7168, - "k": 2048, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "k": 9216, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/down_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/down_proj", + "nvidia/Kimi-K2.5-NVFP4/down_proj" + ] }, { "type": "gemm", "args": { - "m": 66, - "n": 2304, - "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 5120, + "n": 7168, + "k": 16384, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/o_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/o_proj" + ] }, { "type": "gemm", "args": { - "m": 66, - "n": 7168, - "k": 2304, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 5120, + "n": 8192, + "k": 512, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/Kimi-K2.5-MXFP4/kv_b_proj", + "moonshotai/Kimi-K2.5/kv_b_proj", + "nvidia/DeepSeek-R1-0528-FP4-V2/kv_b_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/kv_b_proj", + "nvidia/Kimi-K2.5-NVFP4/kv_b_proj", + "nvidia/Kimi-K2.6-NVFP4/kv_b_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 5120, + "n": 8192, + "k": 512, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 32 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 1, + 32 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "amd/DeepSeek-R1-0528-MXFP4-Preview/kv_b_proj", + "amd/DeepSeek-R1-0528-MXFP4-v2/kv_b_proj", + "amd/DeepSeek-R1-0528-MXFP4/kv_b_proj" + ] }, { "type": "gemm", "args": { - "m": 20480, + "m": 5120, "n": 9216, "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/gate_up_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/gate_up_proj", + "nvidia/Kimi-K2.5-NVFP4/gate_up_proj" + ] }, { "type": "gemm", "args": { - "m": 20480, - "n": 7168, - "k": 9216, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 5120, + "n": 9216, + "k": 7168, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 32 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 1, + 32 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "amd/DeepSeek-R1-0528-MXFP4-Preview/gate_up_proj", + "amd/DeepSeek-R1-0528-MXFP4-v2/gate_up_proj", + "amd/DeepSeek-R1-0528-MXFP4/gate_up_proj", + "amd/Kimi-K2.5-MXFP4/gate_up_proj" + ] }, { "type": "gemm", "args": { - "m": 737280, - "n": 1536, - "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 5120, + "n": 24576, + "k": 1536, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/q_b_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/q_b_proj" + ] }, { "type": "gemm", "args": { - "m": 737280, - "n": 24576, - "k": 1536, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 5120, + "n": 32768, + "k": 512, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/kv_b_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/kv_b_proj" + ] }, { "type": "gemm", "args": { - "m": 737280, - "n": 576, + "m": 5376, + "n": 1088, "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] }, { "type": "gemm", "args": { - "m": 737280, - "n": 32768, - "k": 512, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 5376, + "n": 1536, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/Kimi-K2.5-MXFP4/q_a_proj", + "moonshotai/Kimi-K2.5/q_a_proj", + "nvidia/DeepSeek-R1-0528-FP4-V2/q_a_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/q_a_proj", + "nvidia/Kimi-K2.5-NVFP4/q_a_proj", + "nvidia/Kimi-K2.6-NVFP4/q_a_proj" + ] }, { "type": "gemm", "args": { - "m": 737280, - "n": 7168, - "k": 16384, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 5376, + "n": 1536, + "k": 7168, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 32 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 1, + 32 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "amd/DeepSeek-R1-0528-MXFP4-Preview/q_a_proj", + "amd/DeepSeek-R1-0528-MXFP4-v2/q_a_proj", + "amd/DeepSeek-R1-0528-MXFP4/q_a_proj" + ] }, { "type": "gemm", "args": { - "m": 737280, - "n": 9216, + "m": 5376, + "n": 1536, "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w3" + ] }, { "type": "gemm", "args": { - "m": 737280, + "m": 5376, "n": 7168, - "k": 9216, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "k": 65536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] }, { "type": "gemm", "args": { - "m": 180, - "n": 24576, + "m": 5376, + "n": 65536, "k": 1536, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_b" + ] }, { "type": "gemm", "args": { - "m": 180, - "n": 32768, - "k": 512, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 5376, + "n": 122880, + "k": 1024, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] }, { "type": "gemm", "args": { - "m": 180, - "n": 7168, - "k": 16384, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 5430, + "n": 576, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/Kimi-K2.5-MXFP4/kv_a_proj_with_mqa", + "moonshotai/Kimi-K2.5/kv_a_proj_with_mqa", + "nvidia/DeepSeek-R1-0528-FP4-V2/kv_a_proj_with_mqa", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/kv_a_proj_with_mqa", + "nvidia/Kimi-K2.5-NVFP4/kv_a_proj_with_mqa", + "nvidia/Kimi-K2.6-NVFP4/kv_a_proj_with_mqa" + ] }, { "type": "gemm", "args": { - "m": 180, - "n": 9216, + "m": 5430, + "n": 576, "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 32 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 1, + 32 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "amd/DeepSeek-R1-0528-MXFP4-Preview/kv_a_proj_with_mqa", + "amd/DeepSeek-R1-0528-MXFP4-v2/kv_a_proj_with_mqa", + "amd/DeepSeek-R1-0528-MXFP4/kv_a_proj_with_mqa" + ] }, { "type": "gemm", "args": { - "m": 180, - "n": 7168, - "k": 9216, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 5430, + "n": 1536, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/Kimi-K2.5-MXFP4/q_a_proj", + "moonshotai/Kimi-K2.5/q_a_proj", + "nvidia/DeepSeek-R1-0528-FP4-V2/q_a_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/q_a_proj", + "nvidia/Kimi-K2.5-NVFP4/q_a_proj", + "nvidia/Kimi-K2.6-NVFP4/q_a_proj" + ] }, { "type": "gemm", "args": { - "m": 1261568, + "m": 5430, "n": 1536, "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 32 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 1, + 32 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "amd/DeepSeek-R1-0528-MXFP4-Preview/q_a_proj", + "amd/DeepSeek-R1-0528-MXFP4-v2/q_a_proj", + "amd/DeepSeek-R1-0528-MXFP4/q_a_proj" + ] }, { "type": "gemm", "args": { - "m": 1261568, - "n": 24576, - "k": 1536, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 5430, + "n": 1536, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w3" + ] }, { "type": "gemm", "args": { - "m": 1261568, - "n": 576, + "m": 5430, + "n": 4608, "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/gate_up_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/gate_up_proj", + "nvidia/Kimi-K2.5-NVFP4/gate_up_proj" + ] }, { "type": "gemm", "args": { - "m": 1261568, - "n": 32768, - "k": 512, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 5430, + "n": 4608, + "k": 7168, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 32 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 1, + 32 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "amd/DeepSeek-R1-0528-MXFP4-Preview/gate_up_proj", + "amd/DeepSeek-R1-0528-MXFP4-v2/gate_up_proj", + "amd/DeepSeek-R1-0528-MXFP4/gate_up_proj", + "amd/Kimi-K2.5-MXFP4/gate_up_proj" + ] }, { "type": "gemm", "args": { - "m": 1261568, + "m": 5430, "n": 7168, - "k": 16384, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "k": 4608, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/down_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/down_proj", + "nvidia/Kimi-K2.5-NVFP4/down_proj" + ] }, { "type": "gemm", "args": { - "m": 1261568, - "n": 9216, - "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 5430, + "n": 7168, + "k": 4608, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 32 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 1, + 32 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "amd/DeepSeek-R1-0528-MXFP4-Preview/down_proj", + "amd/DeepSeek-R1-0528-MXFP4-v2/down_proj", + "amd/DeepSeek-R1-0528-MXFP4/down_proj", + "amd/Kimi-K2.5-MXFP4/down_proj" + ] }, { "type": "gemm", "args": { - "m": 1261568, + "m": 5430, "n": 7168, - "k": 9216, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "k": 16384, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/o_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/o_proj" + ] }, { "type": "gemm", "args": { - "m": 308, - "n": 1536, - "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 5430, + "n": 7168, + "k": 16384, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 32 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 1, + 32 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "amd/DeepSeek-R1-0528-MXFP4-Preview/o_proj", + "amd/DeepSeek-R1-0528-MXFP4-v2/o_proj", + "amd/DeepSeek-R1-0528-MXFP4/o_proj" + ] }, { "type": "gemm", "args": { - "m": 308, + "m": 5430, + "n": 7168, + "k": 16384, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 5430, "n": 24576, "k": 1536, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/q_b_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/q_b_proj" + ] }, { "type": "gemm", "args": { - "m": 308, - "n": 576, - "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 5430, + "n": 24576, + "k": 1536, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 32 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 1, + 32 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "amd/DeepSeek-R1-0528-MXFP4-Preview/q_b_proj", + "amd/DeepSeek-R1-0528-MXFP4-v2/q_b_proj", + "amd/DeepSeek-R1-0528-MXFP4/q_b_proj" + ] }, { "type": "gemm", "args": { - "m": 308, + "m": 5430, "n": 32768, "k": 512, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/kv_b_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/kv_b_proj" + ] }, { "type": "gemm", "args": { - "m": 308, - "n": 7168, - "k": 16384, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 5430, + "n": 32768, + "k": 512, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 32 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 1, + 32 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "amd/DeepSeek-R1-0528-MXFP4-Preview/kv_b_proj", + "amd/DeepSeek-R1-0528-MXFP4-v2/kv_b_proj", + "amd/DeepSeek-R1-0528-MXFP4/kv_b_proj" + ] }, { "type": "gemm", "args": { - "m": 308, - "n": 9216, + "m": 5632, + "n": 576, "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 308, - "n": 7168, - "k": 9216, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/Kimi-K2.5-MXFP4/kv_a_proj_with_mqa", + "moonshotai/Kimi-K2.5/kv_a_proj_with_mqa", + "nvidia/DeepSeek-R1-0528-FP4-V2/kv_a_proj_with_mqa", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/kv_a_proj_with_mqa", + "nvidia/Kimi-K2.5-NVFP4/kv_a_proj_with_mqa", + "nvidia/Kimi-K2.6-NVFP4/kv_a_proj_with_mqa" + ] }, { "type": "gemm", "args": { - "m": 78848, + "m": 5632, "n": 1536, "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 78848, - "n": 24576, - "k": 1536, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/Kimi-K2.5-MXFP4/q_a_proj", + "moonshotai/Kimi-K2.5/q_a_proj", + "nvidia/DeepSeek-R1-0528-FP4-V2/q_a_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/q_a_proj", + "nvidia/Kimi-K2.5-NVFP4/q_a_proj", + "nvidia/Kimi-K2.6-NVFP4/q_a_proj" + ] }, { "type": "gemm", "args": { - "m": 78848, - "n": 576, + "m": 5632, + "n": 4608, "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/gate_up_proj" + ] }, { "type": "gemm", "args": { - "m": 78848, - "n": 32768, - "k": 512, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 5632, + "n": 7168, + "k": 4608, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/down_proj" + ] }, { "type": "gemm", "args": { - "m": 78848, + "m": 5632, "n": 7168, - "k": 16384, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "k": 9216, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/down_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/down_proj", + "nvidia/Kimi-K2.5-NVFP4/down_proj" + ] }, { "type": "gemm", "args": { - "m": 78848, + "m": 5632, "n": 7168, - "k": 576, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "k": 16384, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/o_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/o_proj" + ] }, { "type": "gemm", "args": { - "m": 19, - "n": 7168, - "k": 576, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 5632, + "n": 9216, + "k": 7168, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/gate_up_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/gate_up_proj", + "nvidia/Kimi-K2.5-NVFP4/gate_up_proj" + ] }, { "type": "gemm", "args": { - "m": 9228288, - "n": 1536, + "m": 5632, + "n": 9216, "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 32 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 1, + 32 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "amd/DeepSeek-R1-0528-MXFP4-Preview/gate_up_proj", + "amd/DeepSeek-R1-0528-MXFP4-v2/gate_up_proj", + "amd/DeepSeek-R1-0528-MXFP4/gate_up_proj", + "amd/Kimi-K2.5-MXFP4/gate_up_proj" + ] }, { "type": "gemm", "args": { - "m": 9228288, + "m": 5632, "n": 24576, "k": 1536, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/q_b_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/q_b_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 5632, + "n": 32768, + "k": 512, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/kv_b_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/kv_b_proj" + ] }, { "type": "gemm", "args": { - "m": 9228288, + "m": 5760, "n": 576, "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/Kimi-K2.5-MXFP4/kv_a_proj_with_mqa", + "moonshotai/Kimi-K2.5/kv_a_proj_with_mqa", + "nvidia/DeepSeek-R1-0528-FP4-V2/kv_a_proj_with_mqa", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/kv_a_proj_with_mqa", + "nvidia/Kimi-K2.5-NVFP4/kv_a_proj_with_mqa", + "nvidia/Kimi-K2.6-NVFP4/kv_a_proj_with_mqa" + ] }, { "type": "gemm", "args": { - "m": 9228288, - "n": 32768, - "k": 512, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 5760, + "n": 576, + "k": 7168, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 32 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 1, + 32 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "amd/DeepSeek-R1-0528-MXFP4-Preview/kv_a_proj_with_mqa", + "amd/DeepSeek-R1-0528-MXFP4-v2/kv_a_proj_with_mqa", + "amd/DeepSeek-R1-0528-MXFP4/kv_a_proj_with_mqa" + ] }, { "type": "gemm", "args": { - "m": 9228288, - "n": 7168, - "k": 16384, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 5760, + "n": 576, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/kv_a_proj_with_mqa" + ] }, { "type": "gemm", "args": { - "m": 9228288, - "n": 9216, + "m": 5760, + "n": 1536, "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/Kimi-K2.5-MXFP4/q_a_proj", + "moonshotai/Kimi-K2.5/q_a_proj", + "nvidia/DeepSeek-R1-0528-FP4-V2/q_a_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/q_a_proj", + "nvidia/Kimi-K2.5-NVFP4/q_a_proj", + "nvidia/Kimi-K2.6-NVFP4/q_a_proj" + ] }, { "type": "gemm", "args": { - "m": 9228288, - "n": 7168, - "k": 9216, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 5760, + "n": 1536, + "k": 7168, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 32 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 1, + 32 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "amd/DeepSeek-R1-0528-MXFP4-Preview/q_a_proj", + "amd/DeepSeek-R1-0528-MXFP4-v2/q_a_proj", + "amd/DeepSeek-R1-0528-MXFP4/q_a_proj" + ] }, { "type": "gemm", "args": { - "m": 2307072, + "m": 5760, "n": 1536, "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/gate_up_proj", + "deepseek-ai/DeepSeek-R1-0528/q_a_proj", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w3" + ] }, { "type": "gemm", "args": { - "m": 2307072, - "n": 24576, - "k": 1536, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 5760, + "n": 1536, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w3" + ] }, { "type": "gemm", "args": { - "m": 2307072, - "n": 576, - "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 5760, + "n": 7168, + "k": 576, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-NVFP4-v2/down_proj" + ] }, { "type": "gemm", "args": { - "m": 2307072, - "n": 32768, - "k": 512, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 5760, + "n": 7168, + "k": 576, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/down_proj" + ] }, { "type": "gemm", "args": { - "m": 2307072, + "m": 5760, "n": 7168, "k": 16384, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/o_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/o_proj" + ] }, { "type": "gemm", "args": { - "m": 2307072, - "n": 2304, - "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 5760, + "n": 7168, + "k": 16384, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 32 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 1, + 32 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "amd/DeepSeek-R1-0528-MXFP4-Preview/o_proj", + "amd/DeepSeek-R1-0528-MXFP4-v2/o_proj", + "amd/DeepSeek-R1-0528-MXFP4/o_proj" + ] }, { "type": "gemm", "args": { - "m": 2307072, + "m": 5760, "n": 7168, - "k": 2304, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "k": 16384, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/o_proj", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] }, { "type": "gemm", "args": { - "m": 563, - "n": 2304, - "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 5760, + "n": 7168, + "k": 16384, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] }, { "type": "gemm", "args": { - "m": 563, - "n": 7168, - "k": 2304, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 5760, + "n": 24576, + "k": 1536, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/q_b_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/q_b_proj" + ] }, { "type": "gemm", "args": { - "m": 2727936, - "n": 1536, - "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 5760, + "n": 24576, + "k": 1536, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 32 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 1, + 32 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "amd/DeepSeek-R1-0528-MXFP4-Preview/q_b_proj", + "amd/DeepSeek-R1-0528-MXFP4-v2/q_b_proj", + "amd/DeepSeek-R1-0528-MXFP4/q_b_proj" + ] }, { "type": "gemm", "args": { - "m": 2727936, + "m": 5760, "n": 24576, "k": 1536, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/q_b_proj" + ] }, { "type": "gemm", "args": { - "m": 2727936, - "n": 576, - "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 5760, + "n": 32768, + "k": 512, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/kv_b_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/kv_b_proj" + ] }, { "type": "gemm", "args": { - "m": 2727936, + "m": 5760, "n": 32768, "k": 512, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 32 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 1, + 32 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "amd/DeepSeek-R1-0528-MXFP4-Preview/kv_b_proj", + "amd/DeepSeek-R1-0528-MXFP4-v2/kv_b_proj", + "amd/DeepSeek-R1-0528-MXFP4/kv_b_proj" + ] }, { "type": "gemm", "args": { - "m": 2727936, - "n": 7168, - "k": 16384, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 5760, + "n": 32768, + "k": 512, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/kv_b_proj" + ] }, { "type": "gemm", "args": { - "m": 2727936, - "n": 9216, + "m": 6144, + "n": 576, "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/Kimi-K2.5-MXFP4/kv_a_proj_with_mqa", + "moonshotai/Kimi-K2.5/kv_a_proj_with_mqa", + "nvidia/DeepSeek-R1-0528-FP4-V2/kv_a_proj_with_mqa", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/kv_a_proj_with_mqa", + "nvidia/Kimi-K2.5-NVFP4/kv_a_proj_with_mqa", + "nvidia/Kimi-K2.6-NVFP4/kv_a_proj_with_mqa" + ] }, { "type": "gemm", "args": { - "m": 2727936, - "n": 7168, - "k": 9216, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 6144, + "n": 576, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/kv_a_proj_with_mqa" + ] }, { "type": "gemm", "args": { - "m": 666, - "n": 1536, + "m": 6144, + "n": 1152, "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/gate_up_proj" + ] }, { "type": "gemm", "args": { - "m": 666, - "n": 24576, + "m": 6144, + "n": 1536, "k": 1536, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/q_b_proj" + ] }, { "type": "gemm", "args": { - "m": 666, - "n": 576, + "m": 6144, + "n": 1536, "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/Kimi-K2.5-MXFP4/q_a_proj", + "moonshotai/Kimi-K2.5/q_a_proj", + "nvidia/DeepSeek-R1-0528-FP4-V2/q_a_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/q_a_proj", + "nvidia/Kimi-K2.5-NVFP4/q_a_proj", + "nvidia/Kimi-K2.6-NVFP4/q_a_proj" + ] }, { "type": "gemm", "args": { - "m": 666, - "n": 32768, - "k": 512, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 6144, + "n": 1536, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/gate_up_proj", + "deepseek-ai/DeepSeek-R1-0528/q_a_proj", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w3" + ] }, { "type": "gemm", "args": { - "m": 666, - "n": 7168, - "k": 16384, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 6144, + "n": 2048, + "k": 512, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/kv_b_proj" + ] }, { "type": "gemm", "args": { - "m": 666, - "n": 9216, + "m": 6144, + "n": 2304, "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/gate_up_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/gate_up_proj", + "nvidia/Kimi-K2.5-NVFP4/gate_up_proj" + ] }, { "type": "gemm", "args": { - "m": 666, - "n": 7168, - "k": 9216, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 6144, + "n": 2304, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/gate_up_proj" + ] }, { "type": "gemm", "args": { - "m": 340992, - "n": 1536, - "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 6144, + "n": 3072, + "k": 1536, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/Kimi-K2.5-MXFP4/q_b_proj", + "amd/MiniMax-M2.5-MXFP4/o_proj", + "moonshotai/Kimi-K2.5/q_b_proj", + "nvidia/DeepSeek-R1-0528-FP4-V2/q_b_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/q_b_proj", + "nvidia/Kimi-K2.5-NVFP4/q_b_proj", + "nvidia/Kimi-K2.6-NVFP4/q_b_proj", + "nvidia/MiniMax-M2.5-NVFP4/o_proj" + ] }, { "type": "gemm", "args": { - "m": 340992, - "n": 24576, + "m": 6144, + "n": 3072, "k": 1536, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/MiniMax-M2.5-NVFP4/experts.down_proj" + ] }, { "type": "gemm", "args": { - "m": 340992, - "n": 576, - "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 6144, + "n": 3072, + "k": 1536, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 32 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 1, + 32 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "amd/DeepSeek-R1-0528-MXFP4-Preview/q_b_proj", + "amd/DeepSeek-R1-0528-MXFP4-v2/q_b_proj", + "amd/DeepSeek-R1-0528-MXFP4/q_b_proj", + "amd/MiniMax-M2.5-MXFP4/experts.down_proj" + ] }, { "type": "gemm", "args": { - "m": 340992, - "n": 32768, + "m": 6144, + "n": 4096, "k": 512, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "Qwen/Qwen3.5-397B-A17B/o_proj", + "Qwen/Qwen3.5-397B-A17B/shared_expert.down_proj", + "amd/Kimi-K2.5-MXFP4/kv_b_proj", + "amd/Qwen3.5-397B-A17B-MXFP4/o_proj", + "amd/Qwen3.5-397B-A17B-MXFP4/shared_expert.down_proj", + "moonshotai/Kimi-K2.5/kv_b_proj", + "nvidia/DeepSeek-R1-0528-FP4-V2/kv_b_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/kv_b_proj", + "nvidia/Kimi-K2.5-NVFP4/kv_b_proj", + "nvidia/Kimi-K2.6-NVFP4/kv_b_proj", + "nvidia/Qwen3.5-397B-A17B-NVFP4/o_proj", + "nvidia/Qwen3.5-397B-A17B-NVFP4/shared_expert.down_proj" + ] }, { "type": "gemm", "args": { - "m": 340992, - "n": 7168, - "k": 16384, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 6144, + "n": 4096, + "k": 512, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 32 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 1, + 32 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "amd/DeepSeek-R1-0528-MXFP4-Preview/kv_b_proj", + "amd/DeepSeek-R1-0528-MXFP4-v2/kv_b_proj", + "amd/DeepSeek-R1-0528-MXFP4/kv_b_proj", + "amd/Qwen3.5-397B-A17B-MXFP4-AttnFP8-V2/shared_expert.down_proj" + ] }, { "type": "gemm", "args": { - "m": 340992, - "n": 1152, + "m": 6144, + "n": 4608, "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/gate_up_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 6144, + "n": 7168, + "k": 1024, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/o_proj", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] }, { "type": "gemm", "args": { - "m": 340992, + "m": 6144, "n": 7168, "k": 1152, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/down_proj" + ] }, { "type": "gemm", "args": { - "m": 83, - "n": 1152, - "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 6144, + "n": 7168, + "k": 2048, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/Kimi-K2.5-MXFP4/o_proj", + "moonshotai/Kimi-K2.5/o_proj", + "nvidia/Kimi-K2.5-NVFP4/o_proj", + "nvidia/Kimi-K2.6-NVFP4/o_proj" + ] }, { "type": "gemm", "args": { - "m": 83, + "m": 6144, "n": 7168, - "k": 1152, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "k": 2048, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/o_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/o_proj" + ] }, { "type": "gemm", "args": { - "m": 4616192, - "n": 1536, - "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 6144, + "n": 7168, + "k": 2048, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 32 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 1, + 32 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "amd/DeepSeek-R1-0528-MXFP4-Preview/o_proj", + "amd/DeepSeek-R1-0528-MXFP4-v2/o_proj", + "amd/DeepSeek-R1-0528-MXFP4/o_proj" + ] }, { "type": "gemm", "args": { - "m": 4616192, - "n": 24576, - "k": 1536, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 6144, + "n": 7168, + "k": 2304, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/down_proj" + ] }, { "type": "gemm", "args": { - "m": 4616192, - "n": 576, - "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 6144, + "n": 7168, + "k": 4608, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/down_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/down_proj", + "nvidia/Kimi-K2.5-NVFP4/down_proj" + ] }, { "type": "gemm", "args": { - "m": 4616192, - "n": 32768, - "k": 512, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 6144, + "n": 7168, + "k": 4608, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/down_proj" + ] }, { "type": "gemm", "args": { - "m": 4616192, + "m": 6144, "n": 7168, - "k": 16384, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "k": 9216, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/down_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/down_proj", + "nvidia/Kimi-K2.5-NVFP4/down_proj" + ] }, { "type": "gemm", "args": { - "m": 4616192, - "n": 9216, - "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 6144, + "n": 7168, + "k": 16384, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/o_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/o_proj" + ] }, { "type": "gemm", "args": { - "m": 4616192, + "m": 6144, "n": 7168, - "k": 9216, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "k": 16384, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/o_proj", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] }, { "type": "gemm", "args": { - "m": 1127, - "n": 1536, + "m": 6144, + "n": 9216, "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/gate_up_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/gate_up_proj", + "nvidia/Kimi-K2.5-NVFP4/gate_up_proj" + ] }, { "type": "gemm", "args": { - "m": 1127, + "m": 6144, + "n": 9216, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/gate_up_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 6144, "n": 24576, "k": 1536, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/q_b_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/q_b_proj" + ] }, { "type": "gemm", "args": { - "m": 1127, - "n": 576, - "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 6144, + "n": 24576, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/q_b_proj" + ] }, { "type": "gemm", "args": { - "m": 1127, + "m": 6144, "n": 32768, "k": 512, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/kv_b_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/kv_b_proj" + ] }, { "type": "gemm", "args": { - "m": 1127, - "n": 7168, - "k": 16384, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 6144, + "n": 32768, + "k": 512, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/kv_b_proj" + ] }, { "type": "gemm", "args": { - "m": 1127, - "n": 9216, + "m": 6400, + "n": 576, "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 1127, - "n": 7168, - "k": 9216, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/Kimi-K2.5-MXFP4/kv_a_proj_with_mqa", + "moonshotai/Kimi-K2.5/kv_a_proj_with_mqa", + "nvidia/DeepSeek-R1-0528-FP4-V2/kv_a_proj_with_mqa", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/kv_a_proj_with_mqa", + "nvidia/Kimi-K2.5-NVFP4/kv_a_proj_with_mqa", + "nvidia/Kimi-K2.6-NVFP4/kv_a_proj_with_mqa" + ] }, { "type": "gemm", "args": { - "m": 577024, + "m": 6400, "n": 1536, "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/Kimi-K2.5-MXFP4/q_a_proj", + "moonshotai/Kimi-K2.5/q_a_proj", + "nvidia/DeepSeek-R1-0528-FP4-V2/q_a_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/q_a_proj", + "nvidia/Kimi-K2.5-NVFP4/q_a_proj", + "nvidia/Kimi-K2.6-NVFP4/q_a_proj" + ] }, { "type": "gemm", "args": { - "m": 577024, - "n": 24576, - "k": 1536, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 6400, + "n": 4608, + "k": 7168, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/gate_up_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/gate_up_proj", + "nvidia/Kimi-K2.5-NVFP4/gate_up_proj" + ] }, { "type": "gemm", "args": { - "m": 577024, - "n": 576, + "m": 6400, + "n": 4608, "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 32 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 1, + 32 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "amd/DeepSeek-R1-0528-MXFP4-Preview/gate_up_proj", + "amd/DeepSeek-R1-0528-MXFP4-v2/gate_up_proj", + "amd/DeepSeek-R1-0528-MXFP4/gate_up_proj", + "amd/Kimi-K2.5-MXFP4/gate_up_proj" + ] }, { "type": "gemm", "args": { - "m": 577024, - "n": 32768, - "k": 512, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 6400, + "n": 7168, + "k": 4608, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/down_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/down_proj", + "nvidia/Kimi-K2.5-NVFP4/down_proj" + ] }, { "type": "gemm", "args": { - "m": 577024, + "m": 6400, "n": 7168, "k": 16384, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/o_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/o_proj" + ] }, { "type": "gemm", "args": { - "m": 577024, - "n": 1152, - "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 6400, + "n": 24576, + "k": 1536, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/q_b_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/q_b_proj" + ] }, { "type": "gemm", "args": { - "m": 577024, - "n": 7168, - "k": 1152, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 6400, + "n": 32768, + "k": 512, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/kv_b_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/kv_b_proj" + ] }, { "type": "gemm", "args": { - "m": 294912, - "n": 1536, + "m": 7488, + "n": 576, "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/kv_a_proj_with_mqa" + ] }, { "type": "gemm", "args": { - "m": 294912, - "n": 24576, - "k": 1536, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 7488, + "n": 1152, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/gate_up_proj" + ] }, { "type": "gemm", "args": { - "m": 294912, - "n": 576, + "m": 7488, + "n": 1536, "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/gate_up_proj", + "deepseek-ai/DeepSeek-R1-0528/q_a_proj", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w3" + ] }, { "type": "gemm", "args": { - "m": 294912, - "n": 32768, - "k": 512, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 7488, + "n": 7168, + "k": 1152, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/down_proj" + ] }, { "type": "gemm", "args": { - "m": 294912, + "m": 7488, "n": 7168, "k": 16384, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/o_proj", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] }, { "type": "gemm", "args": { - "m": 294912, - "n": 9216, - "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 7488, + "n": 24576, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/q_b_proj" + ] }, { "type": "gemm", "args": { - "m": 294912, - "n": 7168, - "k": 9216, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 7488, + "n": 32768, + "k": 512, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/kv_b_proj" + ] }, { "type": "gemm", "args": { - "m": 36, - "n": 1536, + "m": 7680, + "n": 576, "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/Kimi-K2.5-MXFP4/kv_a_proj_with_mqa", + "moonshotai/Kimi-K2.5/kv_a_proj_with_mqa", + "nvidia/DeepSeek-R1-0528-FP4-V2/kv_a_proj_with_mqa", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/kv_a_proj_with_mqa", + "nvidia/Kimi-K2.5-NVFP4/kv_a_proj_with_mqa", + "nvidia/Kimi-K2.6-NVFP4/kv_a_proj_with_mqa" + ] }, { "type": "gemm", "args": { - "m": 36, - "n": 24576, - "k": 1536, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 7680, + "n": 1536, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/Kimi-K2.5-MXFP4/q_a_proj", + "moonshotai/Kimi-K2.5/q_a_proj", + "nvidia/DeepSeek-R1-0528-FP4-V2/q_a_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/q_a_proj", + "nvidia/Kimi-K2.5-NVFP4/q_a_proj", + "nvidia/Kimi-K2.6-NVFP4/q_a_proj" + ] }, { "type": "gemm", "args": { - "m": 36, - "n": 576, + "m": 7680, + "n": 4608, "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/gate_up_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/gate_up_proj", + "nvidia/Kimi-K2.5-NVFP4/gate_up_proj" + ] }, { "type": "gemm", "args": { - "m": 36, - "n": 32768, - "k": 512, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 7680, + "n": 4608, + "k": 7168, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 32 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 1, + 32 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "amd/DeepSeek-R1-0528-MXFP4-Preview/gate_up_proj", + "amd/DeepSeek-R1-0528-MXFP4-v2/gate_up_proj", + "amd/DeepSeek-R1-0528-MXFP4/gate_up_proj", + "amd/Kimi-K2.5-MXFP4/gate_up_proj" + ] }, { "type": "gemm", "args": { - "m": 36, + "m": 7680, "n": 7168, - "k": 16384, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "k": 4608, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/down_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/down_proj", + "nvidia/Kimi-K2.5-NVFP4/down_proj" + ] }, { "type": "gemm", "args": { - "m": 36, - "n": 9216, - "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 7680, + "n": 7168, + "k": 8192, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/Kimi-K2.5-MXFP4/o_proj", + "moonshotai/Kimi-K2.5/o_proj", + "nvidia/Kimi-K2.5-NVFP4/o_proj", + "nvidia/Kimi-K2.6-NVFP4/o_proj" + ] }, { "type": "gemm", "args": { - "m": 36, + "m": 7680, "n": 7168, "k": 9216, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/down_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/down_proj", + "nvidia/Kimi-K2.5-NVFP4/down_proj" + ] }, { "type": "gemm", "args": { - "m": 589824, - "n": 1536, + "m": 7680, + "n": 7168, + "k": 16384, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/o_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/o_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 7680, + "n": 9216, "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/gate_up_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/gate_up_proj", + "nvidia/Kimi-K2.5-NVFP4/gate_up_proj" + ] }, { "type": "gemm", "args": { - "m": 589824, - "n": 3072, + "m": 7680, + "n": 12288, "k": 1536, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/Kimi-K2.5-MXFP4/q_b_proj", + "moonshotai/Kimi-K2.5/q_b_proj", + "nvidia/DeepSeek-R1-0528-FP4-V2/q_b_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/q_b_proj", + "nvidia/Kimi-K2.5-NVFP4/q_b_proj", + "nvidia/Kimi-K2.6-NVFP4/q_b_proj" + ] }, { "type": "gemm", "args": { - "m": 589824, - "n": 576, - "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 7680, + "n": 16384, + "k": 512, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/Kimi-K2.5-MXFP4/kv_b_proj", + "moonshotai/Kimi-K2.5/kv_b_proj", + "nvidia/DeepSeek-R1-0528-FP4-V2/kv_b_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/kv_b_proj", + "nvidia/Kimi-K2.5-NVFP4/kv_b_proj", + "nvidia/Kimi-K2.6-NVFP4/kv_b_proj" + ] }, { "type": "gemm", "args": { - "m": 589824, - "n": 4096, + "m": 7680, + "n": 24576, + "k": 1536, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/q_b_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/q_b_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 7680, + "n": 32768, "k": 512, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/kv_b_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/kv_b_proj" + ] }, { "type": "gemm", "args": { - "m": 589824, - "n": 7168, - "k": 2048, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 8065, + "n": 576, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/Kimi-K2.5-MXFP4/kv_a_proj_with_mqa", + "moonshotai/Kimi-K2.5/kv_a_proj_with_mqa", + "nvidia/DeepSeek-R1-0528-FP4-V2/kv_a_proj_with_mqa", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/kv_a_proj_with_mqa", + "nvidia/Kimi-K2.5-NVFP4/kv_a_proj_with_mqa", + "nvidia/Kimi-K2.6-NVFP4/kv_a_proj_with_mqa" + ] }, { "type": "gemm", "args": { - "m": 589824, - "n": 2304, + "m": 8065, + "n": 576, "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 32 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 1, + 32 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "amd/DeepSeek-R1-0528-MXFP4-Preview/kv_a_proj_with_mqa", + "amd/DeepSeek-R1-0528-MXFP4-v2/kv_a_proj_with_mqa", + "amd/DeepSeek-R1-0528-MXFP4/kv_a_proj_with_mqa" + ] }, { "type": "gemm", "args": { - "m": 589824, - "n": 7168, - "k": 2304, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 8065, + "n": 1536, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/Kimi-K2.5-MXFP4/q_a_proj", + "moonshotai/Kimi-K2.5/q_a_proj", + "nvidia/DeepSeek-R1-0528-FP4-V2/q_a_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/q_a_proj", + "nvidia/Kimi-K2.5-NVFP4/q_a_proj", + "nvidia/Kimi-K2.6-NVFP4/q_a_proj" + ] }, { "type": "gemm", "args": { - "m": 72, + "m": 8065, "n": 1536, "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 32 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 1, + 32 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "amd/DeepSeek-R1-0528-MXFP4-Preview/q_a_proj", + "amd/DeepSeek-R1-0528-MXFP4-v2/q_a_proj", + "amd/DeepSeek-R1-0528-MXFP4/q_a_proj" + ] }, { "type": "gemm", "args": { - "m": 72, - "n": 3072, - "k": 1536, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 8065, + "n": 1536, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w3" + ] }, { "type": "gemm", "args": { - "m": 72, - "n": 576, + "m": 8065, + "n": 4608, "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/gate_up_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/gate_up_proj", + "nvidia/Kimi-K2.5-NVFP4/gate_up_proj" + ] }, { "type": "gemm", "args": { - "m": 72, - "n": 4096, - "k": 512, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 8065, + "n": 4608, + "k": 7168, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 32 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 1, + 32 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "amd/DeepSeek-R1-0528-MXFP4-Preview/gate_up_proj", + "amd/DeepSeek-R1-0528-MXFP4-v2/gate_up_proj", + "amd/DeepSeek-R1-0528-MXFP4/gate_up_proj", + "amd/Kimi-K2.5-MXFP4/gate_up_proj" + ] }, { "type": "gemm", "args": { - "m": 72, + "m": 8065, "n": 7168, - "k": 2048, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "k": 4608, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/down_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/down_proj", + "nvidia/Kimi-K2.5-NVFP4/down_proj" + ] }, { "type": "gemm", "args": { - "m": 72, - "n": 2304, - "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 8065, + "n": 7168, + "k": 4608, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 32 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 1, + 32 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "amd/DeepSeek-R1-0528-MXFP4-Preview/down_proj", + "amd/DeepSeek-R1-0528-MXFP4-v2/down_proj", + "amd/DeepSeek-R1-0528-MXFP4/down_proj", + "amd/Kimi-K2.5-MXFP4/down_proj" + ] }, { "type": "gemm", "args": { - "m": 72, + "m": 8065, "n": 7168, - "k": 2304, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "k": 16384, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/o_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/o_proj" + ] }, { "type": "gemm", "args": { - "m": 122880, - "n": 9216, - "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 8065, + "n": 7168, + "k": 16384, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 32 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 1, + 32 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "amd/DeepSeek-R1-0528-MXFP4-Preview/o_proj", + "amd/DeepSeek-R1-0528-MXFP4-v2/o_proj", + "amd/DeepSeek-R1-0528-MXFP4/o_proj" + ] }, { "type": "gemm", "args": { - "m": 122880, + "m": 8065, "n": 7168, - "k": 9216, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "k": 16384, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] }, { "type": "gemm", "args": { - "m": 15, - "n": 9216, - "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 8065, + "n": 24576, + "k": 1536, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/q_b_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/q_b_proj" + ] }, { "type": "gemm", "args": { - "m": 15, - "n": 7168, - "k": 9216, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 8065, + "n": 24576, + "k": 1536, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 32 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 1, + 32 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "amd/DeepSeek-R1-0528-MXFP4-Preview/q_b_proj", + "amd/DeepSeek-R1-0528-MXFP4-v2/q_b_proj", + "amd/DeepSeek-R1-0528-MXFP4/q_b_proj" + ] }, { "type": "gemm", "args": { - "m": 40960, - "n": 6144, - "k": 1536, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 8065, + "n": 32768, + "k": 512, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/kv_b_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/kv_b_proj" + ] }, { "type": "gemm", "args": { - "m": 40960, - "n": 8192, + "m": 8065, + "n": 32768, "k": 512, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 32 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 1, + 32 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "amd/DeepSeek-R1-0528-MXFP4-Preview/kv_b_proj", + "amd/DeepSeek-R1-0528-MXFP4-v2/kv_b_proj", + "amd/DeepSeek-R1-0528-MXFP4/kv_b_proj" + ] }, { "type": "gemm", "args": { - "m": 40960, - "n": 7168, - "k": 4096, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 8192, + "n": 64, + "k": 2880, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/gpt-oss-120b-w-mxfp4-a-fp8/k_proj", + "amd/gpt-oss-120b-w-mxfp4-a-fp8/v_proj", + "openai/gpt-oss-120b/k_proj", + "openai/gpt-oss-120b/v_proj" + ] }, { "type": "gemm", "args": { - "m": 40960, - "n": 4608, - "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 8192, + "n": 128, + "k": 2880, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/gpt-oss-120b-w-mxfp4-a-fp8/k_proj", + "amd/gpt-oss-120b-w-mxfp4-a-fp8/v_proj", + "openai/gpt-oss-120b/k_proj", + "openai/gpt-oss-120b/v_proj" + ] }, { "type": "gemm", "args": { - "m": 40960, - "n": 7168, - "k": 4608, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 8192, + "n": 128, + "k": 3072, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/MiniMax-M2.5-MXFP4/k_proj", + "amd/MiniMax-M2.5-MXFP4/v_proj", + "nvidia/MiniMax-M2.5-NVFP4/k_proj", + "nvidia/MiniMax-M2.5-NVFP4/v_proj" + ] }, { "type": "gemm", "args": { - "m": 122880, - "n": 6144, - "k": 1536, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 8192, + "n": 128, + "k": 3072, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "MiniMaxAI/MiniMax-M2.5/k_proj", + "MiniMaxAI/MiniMax-M2.5/v_proj" + ] }, { "type": "gemm", "args": { - "m": 122880, - "n": 8192, - "k": 512, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 8192, + "n": 128, + "k": 4096, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "Qwen/Qwen3.5-397B-A17B/shared_expert.gate_up_proj", + "amd/Qwen3.5-397B-A17B-MXFP4/shared_expert.gate_up_proj", + "nvidia/Qwen3.5-397B-A17B-NVFP4/shared_expert.gate_up_proj" + ] }, { "type": "gemm", "args": { - "m": 122880, - "n": 7168, + "m": 8192, + "n": 128, "k": 4096, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "Qwen/Qwen3.5-397B-A17B-FP8/shared_expert.gate_up_proj" + ] }, { "type": "gemm", "args": { - "m": 245760, - "n": 6144, - "k": 1536, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 8192, + "n": 192, + "k": 3072, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/MiniMax-M2.5-NVFP4/experts.gate_up_proj" + ] }, { "type": "gemm", "args": { - "m": 245760, - "n": 8192, - "k": 512, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 8192, + "n": 192, + "k": 3072, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 32 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 1, + 32 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "amd/MiniMax-M2.5-MXFP4/experts.gate_up_proj" + ] }, { "type": "gemm", "args": { - "m": 245760, - "n": 7168, - "k": 4096, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 8192, + "n": 192, + "k": 3072, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "MiniMaxAI/MiniMax-M2.5/experts.gate_up_proj" + ] }, { "type": "gemm", "args": { - "m": 245760, - "n": 4608, - "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 8192, + "n": 256, + "k": 2880, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/gpt-oss-120b-w-mxfp4-a-fp8/k_proj", + "amd/gpt-oss-120b-w-mxfp4-a-fp8/q_proj", + "amd/gpt-oss-120b-w-mxfp4-a-fp8/v_proj", + "openai/gpt-oss-120b/k_proj", + "openai/gpt-oss-120b/q_proj", + "openai/gpt-oss-120b/v_proj" + ] }, { "type": "gemm", "args": { - "m": 245760, - "n": 7168, - "k": 4608, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 8192, + "n": 256, + "k": 3072, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/MiniMax-M2.5-MXFP4/k_proj", + "amd/MiniMax-M2.5-MXFP4/v_proj", + "nvidia/MiniMax-M2.5-NVFP4/k_proj", + "nvidia/MiniMax-M2.5-NVFP4/v_proj" + ] }, { "type": "gemm", "args": { - "m": 30, - "n": 6144, - "k": 1536, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 8192, + "n": 256, + "k": 3072, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "MiniMaxAI/MiniMax-M2.5/k_proj", + "MiniMaxAI/MiniMax-M2.5/v_proj" + ] }, { "type": "gemm", "args": { - "m": 30, - "n": 8192, - "k": 512, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 8192, + "n": 256, + "k": 4096, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "Qwen/Qwen3.5-397B-A17B/k_proj", + "Qwen/Qwen3.5-397B-A17B/shared_expert.gate_up_proj", + "Qwen/Qwen3.5-397B-A17B/v_proj", + "amd/Qwen3.5-397B-A17B-MXFP4/k_proj", + "amd/Qwen3.5-397B-A17B-MXFP4/shared_expert.gate_up_proj", + "amd/Qwen3.5-397B-A17B-MXFP4/v_proj", + "nvidia/Qwen3.5-397B-A17B-NVFP4/k_proj", + "nvidia/Qwen3.5-397B-A17B-NVFP4/shared_expert.gate_up_proj", + "nvidia/Qwen3.5-397B-A17B-NVFP4/v_proj" + ] }, { "type": "gemm", "args": { - "m": 30, - "n": 7168, + "m": 8192, + "n": 256, "k": 4096, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "Qwen/Qwen3.5-397B-A17B-FP8/k_proj", + "Qwen/Qwen3.5-397B-A17B-FP8/shared_expert.gate_up_proj", + "Qwen/Qwen3.5-397B-A17B-FP8/v_proj" + ] }, { "type": "gemm", "args": { - "m": 5033984, - "n": 1536, - "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 8192, + "n": 384, + "k": 3072, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/MiniMax-M2.5-NVFP4/experts.gate_up_proj" + ] }, { "type": "gemm", "args": { - "m": 5033984, - "n": 24576, - "k": 1536, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 8192, + "n": 384, + "k": 3072, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 32 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 1, + 32 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "amd/MiniMax-M2.5-MXFP4/experts.gate_up_proj" + ] }, { "type": "gemm", "args": { - "m": 5033984, - "n": 576, - "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 8192, + "n": 384, + "k": 3072, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "MiniMaxAI/MiniMax-M2.5/experts.gate_up_proj", + "MiniMaxAI/MiniMax-M2.5/q_proj" + ] }, { "type": "gemm", "args": { - "m": 5033984, - "n": 32768, - "k": 512, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 8192, + "n": 512, + "k": 2880, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/gpt-oss-120b-w-mxfp4-a-fp8/k_proj", + "amd/gpt-oss-120b-w-mxfp4-a-fp8/q_proj", + "amd/gpt-oss-120b-w-mxfp4-a-fp8/v_proj", + "openai/gpt-oss-120b/k_proj", + "openai/gpt-oss-120b/q_proj", + "openai/gpt-oss-120b/v_proj" + ] }, { "type": "gemm", "args": { - "m": 5033984, - "n": 7168, - "k": 16384, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 8192, + "n": 512, + "k": 3072, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/MiniMax-M2.5-MXFP4/k_proj", + "amd/MiniMax-M2.5-MXFP4/v_proj", + "nvidia/MiniMax-M2.5-NVFP4/k_proj", + "nvidia/MiniMax-M2.5-NVFP4/v_proj" + ] }, { "type": "gemm", "args": { - "m": 5033984, - "n": 9216, - "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 8192, + "n": 512, + "k": 3072, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "MiniMaxAI/MiniMax-M2.5/k_proj", + "MiniMaxAI/MiniMax-M2.5/v_proj" + ] }, { "type": "gemm", "args": { - "m": 5033984, - "n": 7168, - "k": 9216, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 8192, + "n": 512, + "k": 4096, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "Qwen/Qwen3.5-397B-A17B/k_proj", + "Qwen/Qwen3.5-397B-A17B/q_proj", + "Qwen/Qwen3.5-397B-A17B/shared_expert.gate_up_proj", + "Qwen/Qwen3.5-397B-A17B/v_proj", + "amd/Qwen3.5-397B-A17B-MXFP4/k_proj", + "amd/Qwen3.5-397B-A17B-MXFP4/q_proj", + "amd/Qwen3.5-397B-A17B-MXFP4/shared_expert.gate_up_proj", + "amd/Qwen3.5-397B-A17B-MXFP4/v_proj", + "nvidia/Qwen3.5-397B-A17B-NVFP4/k_proj", + "nvidia/Qwen3.5-397B-A17B-NVFP4/q_proj", + "nvidia/Qwen3.5-397B-A17B-NVFP4/shared_expert.gate_up_proj", + "nvidia/Qwen3.5-397B-A17B-NVFP4/v_proj" + ] }, { "type": "gemm", "args": { - "m": 76, - "n": 1152, - "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 8192, + "n": 512, + "k": 4096, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "Qwen/Qwen3.5-397B-A17B-FP8/k_proj", + "Qwen/Qwen3.5-397B-A17B-FP8/q_proj", + "Qwen/Qwen3.5-397B-A17B-FP8/shared_expert.gate_up_proj", + "Qwen/Qwen3.5-397B-A17B-FP8/v_proj", + "deepseek-ai/DeepSeek-V4.1-Flash/attn.wo_a" + ] }, { "type": "gemm", "args": { - "m": 76, - "n": 7168, - "k": 1152, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 8192, + "n": 576, + "k": 6144, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/GLM-5.1-MXFP4/kv_a_proj_with_mqa", + "amd/GLM-5.2-MXFP4/kv_a_proj_with_mqa", + "nvidia/GLM-5-NVFP4/kv_a_proj_with_mqa", + "nvidia/GLM-5.1-NVFP4/kv_a_proj_with_mqa", + "nvidia/GLM-5.2-NVFP4/kv_a_proj_with_mqa" + ] }, { "type": "gemm", "args": { - "m": 13221888, - "n": 1536, - "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 8192, + "n": 576, + "k": 6144, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "zai-org/GLM-5-FP8/kv_a_proj_with_mqa", + "zai-org/GLM-5.1-FP8/kv_a_proj_with_mqa", + "zai-org/GLM-5.2-FP8/kv_a_proj_with_mqa" + ] }, { "type": "gemm", "args": { - "m": 13221888, - "n": 24576, - "k": 1536, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 8192, + "n": 576, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/Kimi-K2.5-MXFP4/kv_a_proj_with_mqa", + "moonshotai/Kimi-K2.5/kv_a_proj_with_mqa", + "nvidia/DeepSeek-R1-0528-FP4-V2/kv_a_proj_with_mqa", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/kv_a_proj_with_mqa", + "nvidia/Kimi-K2.5-NVFP4/kv_a_proj_with_mqa", + "nvidia/Kimi-K2.6-NVFP4/kv_a_proj_with_mqa" + ] }, { "type": "gemm", "args": { - "m": 13221888, + "m": 8192, "n": 576, "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 32 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 1, + 32 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "amd/DeepSeek-R1-0528-MXFP4-Preview/kv_a_proj_with_mqa", + "amd/DeepSeek-R1-0528-MXFP4-v2/kv_a_proj_with_mqa", + "amd/DeepSeek-R1-0528-MXFP4/kv_a_proj_with_mqa" + ] }, { "type": "gemm", "args": { - "m": 13221888, - "n": 32768, - "k": 512, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 8192, + "n": 576, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/kv_a_proj_with_mqa" + ] }, { "type": "gemm", "args": { - "m": 13221888, - "n": 7168, - "k": 16384, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 8192, + "n": 768, + "k": 3072, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/MiniMax-M2.5-NVFP4/experts.gate_up_proj" + ] }, { "type": "gemm", "args": { - "m": 13221888, - "n": 9216, - "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 8192, + "n": 768, + "k": 3072, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 32 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 1, + 32 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "amd/MiniMax-M2.5-MXFP4/experts.gate_up_proj" + ] }, { "type": "gemm", "args": { - "m": 13221888, - "n": 7168, - "k": 9216, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 8192, + "n": 768, + "k": 3072, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "MiniMaxAI/MiniMax-M2.5/experts.gate_up_proj", + "MiniMaxAI/MiniMax-M2.5/q_proj" + ] }, { "type": "gemm", "args": { - "m": 1614, - "n": 1536, - "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 8192, + "n": 1024, + "k": 2880, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/gpt-oss-120b-w-mxfp4-a-fp8/q_proj", + "openai/gpt-oss-120b/q_proj" + ] }, { "type": "gemm", "args": { - "m": 1614, - "n": 24576, - "k": 1536, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 8192, + "n": 1024, + "k": 3072, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/MiniMax-M2.5-MXFP4/k_proj", + "amd/MiniMax-M2.5-MXFP4/qkv_proj", + "amd/MiniMax-M2.5-MXFP4/v_proj", + "nvidia/MiniMax-M2.5-NVFP4/k_proj", + "nvidia/MiniMax-M2.5-NVFP4/qkv_proj", + "nvidia/MiniMax-M2.5-NVFP4/v_proj" + ] }, { "type": "gemm", "args": { - "m": 1614, - "n": 576, - "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 8192, + "n": 1024, + "k": 3072, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "MiniMaxAI/MiniMax-M2.5/k_proj", + "MiniMaxAI/MiniMax-M2.5/qkv_proj", + "MiniMaxAI/MiniMax-M2.5/v_proj" + ] }, { "type": "gemm", "args": { - "m": 1614, - "n": 32768, - "k": 512, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 8192, + "n": 1024, + "k": 4096, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "Qwen/Qwen3.5-397B-A17B/q_proj", + "Qwen/Qwen3.5-397B-A17B/shared_expert.gate_up_proj", + "amd/Qwen3.5-397B-A17B-MXFP4/q_proj", + "amd/Qwen3.5-397B-A17B-MXFP4/shared_expert.gate_up_proj", + "nvidia/Qwen3.5-397B-A17B-NVFP4/q_proj", + "nvidia/Qwen3.5-397B-A17B-NVFP4/shared_expert.gate_up_proj" + ] }, { "type": "gemm", "args": { - "m": 1614, - "n": 7168, - "k": 16384, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 8192, + "n": 1024, + "k": 4096, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "Qwen/Qwen3.5-397B-A17B-FP8/q_proj", + "Qwen/Qwen3.5-397B-A17B-FP8/shared_expert.gate_up_proj", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_a", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_a", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_a" + ] }, { "type": "gemm", "args": { - "m": 1614, - "n": 9216, + "m": 8192, + "n": 1088, "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] }, { "type": "gemm", "args": { - "m": 1614, - "n": 7168, - "k": 9216, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 8192, + "n": 1152, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/gate_up_proj" + ] }, { "type": "gemm", "args": { - "m": 6610944, + "m": 8192, "n": 1536, - "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "k": 1536, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/Kimi-K2.5-MXFP4/q_b_proj", + "moonshotai/Kimi-K2.5/q_b_proj", + "nvidia/DeepSeek-R1-0528-FP4-V2/q_b_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/q_b_proj", + "nvidia/Kimi-K2.5-NVFP4/q_b_proj", + "nvidia/Kimi-K2.6-NVFP4/q_b_proj" + ] }, { "type": "gemm", "args": { - "m": 6610944, - "n": 24576, + "m": 8192, + "n": 1536, "k": 1536, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/q_b_proj" + ] }, { "type": "gemm", "args": { - "m": 6610944, - "n": 576, - "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 8192, + "n": 1536, + "k": 3072, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 32 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 1, + 32 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "amd/MiniMax-M2.5-MXFP4/experts.gate_up_proj" + ] }, { "type": "gemm", "args": { - "m": 6610944, - "n": 32768, - "k": 512, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 8192, + "n": 1536, + "k": 3072, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "MiniMaxAI/MiniMax-M2.5/experts.gate_up_proj", + "MiniMaxAI/MiniMax-M2.5/q_proj" + ] }, { "type": "gemm", "args": { - "m": 6610944, - "n": 7168, - "k": 16384, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 8192, + "n": 1536, + "k": 6144, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "zai-org/GLM-5-FP8/gate_up_proj", + "zai-org/GLM-5.1-FP8/gate_up_proj", + "zai-org/GLM-5.2-FP8/gate_up_proj" + ] }, { "type": "gemm", "args": { - "m": 6610944, - "n": 4608, + "m": 8192, + "n": 1536, "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/Kimi-K2.5-MXFP4/q_a_proj", + "moonshotai/Kimi-K2.5/q_a_proj", + "nvidia/DeepSeek-R1-0528-FP4-V2/q_a_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/q_a_proj", + "nvidia/Kimi-K2.5-NVFP4/q_a_proj", + "nvidia/Kimi-K2.6-NVFP4/q_a_proj" + ] }, { "type": "gemm", "args": { - "m": 6610944, - "n": 7168, - "k": 4608, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 8192, + "n": 1536, + "k": 7168, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 32 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 1, + 32 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "amd/DeepSeek-R1-0528-MXFP4-Preview/q_a_proj", + "amd/DeepSeek-R1-0528-MXFP4-v2/q_a_proj", + "amd/DeepSeek-R1-0528-MXFP4/q_a_proj" + ] }, { "type": "gemm", "args": { - "m": 807, + "m": 8192, "n": 1536, "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/gate_up_proj", + "deepseek-ai/DeepSeek-R1-0528/q_a_proj", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w3" + ] }, { "type": "gemm", "args": { - "m": 807, - "n": 24576, - "k": 1536, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 8192, + "n": 1536, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w3" + ] }, { "type": "gemm", "args": { - "m": 807, - "n": 576, - "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 8192, + "n": 2048, + "k": 512, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/Kimi-K2.5-MXFP4/kv_b_proj", + "moonshotai/Kimi-K2.5/kv_b_proj", + "nvidia/DeepSeek-R1-0528-FP4-V2/kv_b_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/kv_b_proj", + "nvidia/Kimi-K2.5-NVFP4/kv_b_proj", + "nvidia/Kimi-K2.6-NVFP4/kv_b_proj" + ] }, { "type": "gemm", "args": { - "m": 807, - "n": 32768, + "m": 8192, + "n": 2048, "k": 512, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/kv_b_proj" + ] }, { "type": "gemm", "args": { - "m": 807, - "n": 7168, - "k": 16384, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 8192, + "n": 2048, + "k": 2048, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "zai-org/GLM-5-FP8/q_b_proj", + "zai-org/GLM-5.1-FP8/q_b_proj", + "zai-org/GLM-5.2-FP8/q_b_proj" + ] }, { "type": "gemm", "args": { - "m": 807, - "n": 4608, - "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 8192, + "n": 2048, + "k": 2880, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/gpt-oss-120b-w-mxfp4-a-fp8/q_proj", + "openai/gpt-oss-120b/q_proj" + ] }, { "type": "gemm", "args": { - "m": 807, - "n": 7168, - "k": 4608, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 8192, + "n": 2048, + "k": 4096, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "Qwen/Qwen3.5-397B-A17B/q_proj", + "Qwen/Qwen3.5-397B-A17B/shared_expert.gate_up_proj", + "amd/Qwen3.5-397B-A17B-MXFP4/q_proj", + "amd/Qwen3.5-397B-A17B-MXFP4/shared_expert.gate_up_proj", + "nvidia/Qwen3.5-397B-A17B-NVFP4/q_proj", + "nvidia/Qwen3.5-397B-A17B-NVFP4/shared_expert.gate_up_proj" + ] }, { "type": "gemm", "args": { - "m": 10922, - "n": 1536, - "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 8192, + "n": 2048, + "k": 4096, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "Qwen/Qwen3.5-397B-A17B-FP8/q_proj", + "Qwen/Qwen3.5-397B-A17B-FP8/shared_expert.gate_up_proj" + ] }, { "type": "gemm", "args": { - "m": 10922, - "n": 24576, - "k": 1536, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 8192, + "n": 2048, + "k": 6144, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/GLM-5.1-MXFP4/q_a_proj", + "amd/GLM-5.2-MXFP4/q_a_proj", + "amd/MiniMax-M3-MXFP4/q_proj", + "nvidia/GLM-5-NVFP4/q_a_proj", + "nvidia/GLM-5.1-NVFP4/q_a_proj", + "nvidia/GLM-5.2-NVFP4/q_a_proj" + ] }, { "type": "gemm", "args": { - "m": 10922, - "n": 576, + "m": 8192, + "n": 2048, + "k": 6144, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "zai-org/GLM-5-FP8/q_a_proj", + "zai-org/GLM-5.1-FP8/q_a_proj", + "zai-org/GLM-5.2-FP8/q_a_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 8192, + "n": 2304, "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/gate_up_proj" + ] }, { "type": "gemm", "args": { - "m": 10922, - "n": 32768, + "m": 8192, + "n": 2880, "k": 512, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/gpt-oss-120b-w-mxfp4-a-fp8/o_proj", + "openai/gpt-oss-120b/o_proj" + ] }, { "type": "gemm", "args": { - "m": 10922, - "n": 7168, - "k": 16384, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 8192, + "n": 2880, + "k": 1024, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/gpt-oss-120b-w-mxfp4-a-fp8/o_proj", + "openai/gpt-oss-120b/o_proj" + ] }, { "type": "gemm", "args": { - "m": 10922, - "n": 384, - "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 8192, + "n": 2880, + "k": 2048, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/gpt-oss-120b-w-mxfp4-a-fp8/o_proj", + "openai/gpt-oss-120b/o_proj" + ] }, { "type": "gemm", "args": { - "m": 10922, - "n": 7168, - "k": 384, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 8192, + "n": 2880, + "k": 4096, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/gpt-oss-120b-w-mxfp4-a-fp8/o_proj", + "openai/gpt-oss-120b/o_proj" + ] }, { "type": "gemm", "args": { - "m": 174762, - "n": 1536, - "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 8192, + "n": 3072, + "k": 192, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/MiniMax-M2.5-NVFP4/experts.down_proj" + ] }, { "type": "gemm", "args": { - "m": 174762, - "n": 24576, - "k": 1536, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 8192, + "n": 3072, + "k": 192, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 32 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 1, + 32 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "amd/MiniMax-M2.5-MXFP4/experts.down_proj" + ] }, { "type": "gemm", "args": { - "m": 174762, - "n": 576, - "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 8192, + "n": 3072, + "k": 192, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "MiniMaxAI/MiniMax-M2.5/experts.down_proj" + ] }, { "type": "gemm", "args": { - "m": 174762, - "n": 32768, - "k": 512, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 8192, + "n": 3072, + "k": 384, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/MiniMax-M2.5-NVFP4/experts.down_proj" + ] }, { "type": "gemm", "args": { - "m": 174762, - "n": 7168, - "k": 16384, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 8192, + "n": 3072, + "k": 384, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 32 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 1, + 32 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "amd/MiniMax-M2.5-MXFP4/experts.down_proj" + ] }, { "type": "gemm", "args": { - "m": 174762, - "n": 384, - "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 8192, + "n": 3072, + "k": 384, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "MiniMaxAI/MiniMax-M2.5/experts.down_proj", + "MiniMaxAI/MiniMax-M2.5/o_proj" + ] }, { "type": "gemm", "args": { - "m": 174762, - "n": 7168, - "k": 384, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 8192, + "n": 3072, + "k": 768, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/MiniMax-M2.5-MXFP4/o_proj", + "nvidia/MiniMax-M2.5-NVFP4/o_proj" + ] }, { "type": "gemm", "args": { - "m": 170, - "n": 384, - "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 8192, + "n": 3072, + "k": 768, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 32 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 1, + 32 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "amd/MiniMax-M2.5-MXFP4/experts.down_proj" + ] }, { "type": "gemm", "args": { - "m": 170, - "n": 7168, - "k": 384, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 8192, + "n": 3072, + "k": 768, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "MiniMaxAI/MiniMax-M2.5/experts.down_proj", + "MiniMaxAI/MiniMax-M2.5/o_proj" + ] }, { "type": "gemm", "args": { - "m": 2048, - "n": 4608, - "k": 7168, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 8192, + "n": 3072, + "k": 1536, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/Kimi-K2.5-MXFP4/q_b_proj", + "amd/MiniMax-M2.5-MXFP4/o_proj", + "moonshotai/Kimi-K2.5/q_b_proj", + "nvidia/DeepSeek-R1-0528-FP4-V2/q_b_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/q_b_proj", + "nvidia/Kimi-K2.5-NVFP4/q_b_proj", + "nvidia/Kimi-K2.6-NVFP4/q_b_proj", + "nvidia/MiniMax-M2.5-NVFP4/o_proj" + ] }, { "type": "gemm", "args": { - "m": 2048, - "n": 7168, - "k": 4608, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 8192, + "n": 3072, + "k": 1536, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/MiniMax-M2.5-NVFP4/experts.down_proj" + ] }, { "type": "gemm", "args": { - "m": 6144, - "n": 4608, - "k": 7168, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 8192, + "n": 3072, + "k": 1536, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 32 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 1, + 32 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "amd/DeepSeek-R1-0528-MXFP4-Preview/q_b_proj", + "amd/DeepSeek-R1-0528-MXFP4-v2/q_b_proj", + "amd/DeepSeek-R1-0528-MXFP4/q_b_proj", + "amd/MiniMax-M2.5-MXFP4/experts.down_proj" + ] }, { "type": "gemm", "args": { - "m": 6144, - "n": 7168, - "k": 4608, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 8192, + "n": 3072, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "MiniMaxAI/MiniMax-M2.5/experts.down_proj", + "MiniMaxAI/MiniMax-M2.5/o_proj", + "deepseek-ai/DeepSeek-R1-0528/q_b_proj" + ] }, { "type": "gemm", "args": { - "m": 12, - "n": 4608, - "k": 7168, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 8192, + "n": 3072, + "k": 3072, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/MiniMax-M2.5-MXFP4/o_proj", + "amd/MiniMax-M2.5-MXFP4/q_proj", + "nvidia/MiniMax-M2.5-NVFP4/o_proj", + "nvidia/MiniMax-M2.5-NVFP4/q_proj" + ] }, { "type": "gemm", "args": { - "m": 12, - "n": 7168, - "k": 4608, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 8192, + "n": 3072, + "k": 3072, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/MiniMax-M2.5-NVFP4/experts.gate_up_proj" + ] }, { "type": "gemm", "args": { - "m": 24576, - "n": 1536, - "k": 7168, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 8192, + "n": 3072, + "k": 3072, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 32 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 1, + 32 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "amd/MiniMax-M2.5-MXFP4/experts.gate_up_proj" + ] }, { "type": "gemm", "args": { - "m": 24576, + "m": 8192, "n": 3072, - "k": 1536, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "k": 3072, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "MiniMaxAI/MiniMax-M2.5/experts.gate_up_proj", + "MiniMaxAI/MiniMax-M2.5/o_proj", + "MiniMaxAI/MiniMax-M2.5/q_proj" + ] }, { "type": "gemm", "args": { - "m": 24576, - "n": 576, - "k": 7168, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 8192, + "n": 3072, + "k": 6144, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/GLM-5.1-MXFP4/gate_up_proj", + "amd/GLM-5.2-MXFP4/gate_up_proj", + "amd/MiniMax-M2.5-MXFP4/o_proj", + "amd/MiniMax-M3-MXFP4/gate_up_proj", + "nvidia/GLM-5.1-NVFP4/gate_up_proj", + "nvidia/GLM-5.2-NVFP4/gate_up_proj", + "nvidia/MiniMax-M2.5-NVFP4/o_proj" + ] }, { "type": "gemm", "args": { - "m": 24576, - "n": 4096, - "k": 512, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 8192, + "n": 3072, + "k": 6144, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "MiniMaxAI/MiniMax-M2.5/o_proj", + "zai-org/GLM-5-FP8/gate_up_proj", + "zai-org/GLM-5.1-FP8/gate_up_proj", + "zai-org/GLM-5.2-FP8/gate_up_proj" + ] }, { "type": "gemm", "args": { - "m": 24576, - "n": 7168, - "k": 2048, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 8192, + "n": 3584, + "k": 512, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "zai-org/GLM-5-FP8/kv_b_proj", + "zai-org/GLM-5.1-FP8/kv_b_proj", + "zai-org/GLM-5.2-FP8/kv_b_proj" + ] }, { "type": "gemm", "args": { - "m": 24576, - "n": 2304, - "k": 7168, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 8192, + "n": 4096, + "k": 128, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "Qwen/Qwen3.5-397B-A17B/shared_expert.down_proj", + "amd/Qwen3.5-397B-A17B-MXFP4/shared_expert.down_proj", + "nvidia/Qwen3.5-397B-A17B-NVFP4/shared_expert.down_proj" + ] }, { "type": "gemm", "args": { - "m": 24576, - "n": 7168, - "k": 2304, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 8192, + "n": 4096, + "k": 128, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "Qwen/Qwen3.5-397B-A17B-FP8/shared_expert.down_proj" + ] }, { "type": "gemm", "args": { - "m": 48, - "n": 1536, - "k": 7168, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 8192, + "n": 4096, + "k": 256, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "Qwen/Qwen3.5-397B-A17B/shared_expert.down_proj", + "amd/Qwen3.5-397B-A17B-MXFP4/shared_expert.down_proj", + "nvidia/Qwen3.5-397B-A17B-NVFP4/shared_expert.down_proj" + ] }, { "type": "gemm", "args": { - "m": 48, - "n": 3072, - "k": 1536, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 8192, + "n": 4096, + "k": 256, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "Qwen/Qwen3.5-397B-A17B-FP8/shared_expert.down_proj" + ] }, { "type": "gemm", "args": { - "m": 48, - "n": 576, - "k": 7168, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 8192, + "n": 4096, + "k": 512, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "Qwen/Qwen3.5-397B-A17B/o_proj", + "Qwen/Qwen3.5-397B-A17B/shared_expert.down_proj", + "amd/Kimi-K2.5-MXFP4/kv_b_proj", + "amd/Qwen3.5-397B-A17B-MXFP4/o_proj", + "amd/Qwen3.5-397B-A17B-MXFP4/shared_expert.down_proj", + "moonshotai/Kimi-K2.5/kv_b_proj", + "nvidia/DeepSeek-R1-0528-FP4-V2/kv_b_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/kv_b_proj", + "nvidia/Kimi-K2.5-NVFP4/kv_b_proj", + "nvidia/Kimi-K2.6-NVFP4/kv_b_proj", + "nvidia/Qwen3.5-397B-A17B-NVFP4/o_proj", + "nvidia/Qwen3.5-397B-A17B-NVFP4/shared_expert.down_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 8192, + "n": 4096, + "k": 512, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 32 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 1, + 32 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "amd/DeepSeek-R1-0528-MXFP4-Preview/kv_b_proj", + "amd/DeepSeek-R1-0528-MXFP4-v2/kv_b_proj", + "amd/DeepSeek-R1-0528-MXFP4/kv_b_proj", + "amd/Qwen3.5-397B-A17B-MXFP4-AttnFP8-V2/shared_expert.down_proj" + ] }, { "type": "gemm", "args": { - "m": 48, + "m": 8192, "n": 4096, "k": 512, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "Qwen/Qwen3.5-397B-A17B-FP8/o_proj", + "Qwen/Qwen3.5-397B-A17B-FP8/shared_expert.down_proj", + "deepseek-ai/DeepSeek-R1-0528/kv_b_proj" + ] }, { "type": "gemm", "args": { - "m": 48, - "n": 7168, - "k": 2048, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 8192, + "n": 4096, + "k": 1024, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "Qwen/Qwen3.5-397B-A17B/o_proj", + "Qwen/Qwen3.5-397B-A17B/shared_expert.down_proj", + "amd/Qwen3.5-397B-A17B-MXFP4/o_proj", + "amd/Qwen3.5-397B-A17B-MXFP4/shared_expert.down_proj", + "nvidia/Qwen3.5-397B-A17B-NVFP4/o_proj", + "nvidia/Qwen3.5-397B-A17B-NVFP4/shared_expert.down_proj" + ] }, { "type": "gemm", "args": { - "m": 48, - "n": 2304, - "k": 7168, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 8192, + "n": 4096, + "k": 1024, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "Qwen/Qwen3.5-397B-A17B-FP8/o_proj", + "Qwen/Qwen3.5-397B-A17B-FP8/shared_expert.down_proj" + ] }, { "type": "gemm", "args": { - "m": 48, - "n": 7168, - "k": 2304, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 8192, + "n": 4096, + "k": 2048, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "Qwen/Qwen3.5-397B-A17B/o_proj", + "amd/GLM-5.1-MXFP4/q_b_proj", + "amd/GLM-5.2-MXFP4/q_b_proj", + "amd/Qwen3.5-397B-A17B-MXFP4/o_proj", + "nvidia/GLM-5-NVFP4/q_b_proj", + "nvidia/GLM-5.1-NVFP4/q_b_proj", + "nvidia/GLM-5.2-NVFP4/q_b_proj", + "nvidia/Qwen3.5-397B-A17B-NVFP4/o_proj" + ] }, { "type": "gemm", "args": { - "m": 46080, - "n": 1536, - "k": 7168, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 8192, + "n": 4096, + "k": 2048, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "Qwen/Qwen3.5-397B-A17B-FP8/o_proj", + "zai-org/GLM-5-FP8/q_b_proj", + "zai-org/GLM-5.1-FP8/q_b_proj", + "zai-org/GLM-5.2-FP8/q_b_proj" + ] }, { "type": "gemm", "args": { - "m": 46080, - "n": 24576, - "k": 1536, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 8192, + "n": 4096, + "k": 2880, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/gpt-oss-120b-w-mxfp4-a-fp8/q_proj", + "openai/gpt-oss-120b/q_proj" + ] }, { "type": "gemm", "args": { - "m": 46080, - "n": 576, - "k": 7168, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 8192, + "n": 4096, + "k": 4096, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "Qwen/Qwen3.5-397B-A17B/o_proj", + "Qwen/Qwen3.5-397B-A17B/q_proj", + "amd/Qwen3.5-397B-A17B-MXFP4/o_proj", + "amd/Qwen3.5-397B-A17B-MXFP4/q_proj", + "nvidia/Qwen3.5-397B-A17B-NVFP4/o_proj", + "nvidia/Qwen3.5-397B-A17B-NVFP4/q_proj" + ] }, { "type": "gemm", "args": { - "m": 46080, - "n": 32768, - "k": 512, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 8192, + "n": 4096, + "k": 4096, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "Qwen/Qwen3.5-397B-A17B-FP8/o_proj", + "Qwen/Qwen3.5-397B-A17B-FP8/q_proj" + ] }, { "type": "gemm", "args": { - "m": 46080, - "n": 7168, - "k": 16384, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 8192, + "n": 4608, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "moonshotai/Kimi-K2.5/gate_up_proj", + "nvidia/Kimi-K2.6-NVFP4/gate_up_proj" + ] }, { "type": "gemm", "args": { - "m": 46080, + "m": 8192, "n": 4608, "k": 7168, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/gate_up_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/gate_up_proj", + "nvidia/Kimi-K2.5-NVFP4/gate_up_proj" + ] }, { "type": "gemm", "args": { - "m": 46080, - "n": 7168, - "k": 4608, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 8192, + "n": 4608, + "k": 7168, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 32 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 1, + 32 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "amd/DeepSeek-R1-0528-MXFP4-Preview/gate_up_proj", + "amd/DeepSeek-R1-0528-MXFP4-v2/gate_up_proj", + "amd/DeepSeek-R1-0528-MXFP4/gate_up_proj", + "amd/Kimi-K2.5-MXFP4/gate_up_proj" + ] }, { "type": "gemm", "args": { - "m": 90, - "n": 1536, + "m": 8192, + "n": 4608, "k": 7168, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/gate_up_proj" + ] }, { "type": "gemm", "args": { - "m": 90, - "n": 24576, + "m": 8192, + "n": 6144, "k": 1536, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/GLM-5.1-MXFP4/down_proj", + "amd/GLM-5.2-MXFP4/down_proj", + "amd/Kimi-K2.5-MXFP4/q_b_proj", + "amd/MiniMax-M3-MXFP4/down_proj", + "moonshotai/Kimi-K2.5/q_b_proj", + "nvidia/DeepSeek-R1-0528-FP4-V2/q_b_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/q_b_proj", + "nvidia/GLM-5.1-NVFP4/down_proj", + "nvidia/GLM-5.2-NVFP4/down_proj", + "nvidia/Kimi-K2.5-NVFP4/q_b_proj", + "nvidia/Kimi-K2.6-NVFP4/q_b_proj" + ] }, { "type": "gemm", "args": { - "m": 90, - "n": 576, - "k": 7168, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 8192, + "n": 6144, + "k": 1536, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 32 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 1, + 32 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "amd/DeepSeek-R1-0528-MXFP4-Preview/q_b_proj", + "amd/DeepSeek-R1-0528-MXFP4-v2/q_b_proj", + "amd/DeepSeek-R1-0528-MXFP4/q_b_proj" + ] }, { "type": "gemm", "args": { - "m": 90, - "n": 32768, - "k": 512, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 8192, + "n": 6144, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/q_b_proj", + "zai-org/GLM-5-FP8/down_proj", + "zai-org/GLM-5.1-FP8/down_proj", + "zai-org/GLM-5.2-FP8/down_proj" + ] }, { "type": "gemm", "args": { - "m": 90, - "n": 7168, - "k": 16384, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 8192, + "n": 6144, + "k": 2048, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "zai-org/GLM-5-FP8/o_proj", + "zai-org/GLM-5.1-FP8/o_proj", + "zai-org/GLM-5.2-FP8/o_proj" + ] }, { "type": "gemm", "args": { - "m": 90, - "n": 4608, - "k": 7168, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 8192, + "n": 6144, + "k": 3072, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/GLM-5.1-MXFP4/down_proj", + "amd/GLM-5.2-MXFP4/down_proj", + "amd/MiniMax-M2.5-MXFP4/q_proj", + "amd/MiniMax-M3-MXFP4/down_proj", + "nvidia/GLM-5.1-NVFP4/down_proj", + "nvidia/GLM-5.2-NVFP4/down_proj", + "nvidia/MiniMax-M2.5-NVFP4/q_proj" + ] }, { "type": "gemm", "args": { - "m": 90, - "n": 7168, - "k": 4608, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 8192, + "n": 6144, + "k": 3072, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/GLM-5-NVFP4/down_proj" + ] }, { "type": "gemm", "args": { - "m": 5760, - "n": 1536, - "k": 7168, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 8192, + "n": 6144, + "k": 3072, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "MiniMaxAI/MiniMax-M2.5/q_proj", + "zai-org/GLM-5-FP8/down_proj", + "zai-org/GLM-5.1-FP8/down_proj", + "zai-org/GLM-5.2-FP8/down_proj" + ] }, { "type": "gemm", "args": { - "m": 5760, - "n": 24576, - "k": 1536, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 8192, + "n": 6144, + "k": 4096, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/GLM-5.1-MXFP4/o_proj", + "amd/GLM-5.2-MXFP4/o_proj", + "amd/MiniMax-M3-MXFP4/o_proj", + "nvidia/GLM-5-NVFP4/o_proj", + "nvidia/GLM-5.1-NVFP4/o_proj", + "nvidia/GLM-5.2-NVFP4/o_proj" + ] }, { "type": "gemm", "args": { - "m": 5760, - "n": 576, - "k": 7168, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 8192, + "n": 6144, + "k": 4096, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "zai-org/GLM-5-FP8/o_proj", + "zai-org/GLM-5.1-FP8/o_proj", + "zai-org/GLM-5.2-FP8/o_proj" + ] }, { "type": "gemm", "args": { - "m": 5760, - "n": 32768, - "k": 512, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 8192, + "n": 6144, + "k": 6144, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/GLM-5.1-MXFP4/down_proj", + "amd/GLM-5.1-MXFP4/gate_up_proj", + "amd/GLM-5.2-MXFP4/down_proj", + "amd/GLM-5.2-MXFP4/gate_up_proj", + "amd/MiniMax-M3-MXFP4/gate_up_proj", + "nvidia/GLM-5.1-NVFP4/down_proj", + "nvidia/GLM-5.1-NVFP4/gate_up_proj", + "nvidia/GLM-5.2-NVFP4/down_proj", + "nvidia/GLM-5.2-NVFP4/gate_up_proj" + ] }, { "type": "gemm", "args": { - "m": 5760, - "n": 7168, - "k": 16384, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 8192, + "n": 6144, + "k": 6144, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/GLM-5-NVFP4/down_proj", + "nvidia/GLM-5-NVFP4/gate_up_proj" + ] }, { "type": "gemm", "args": { - "m": 5760, - "n": 7168, - "k": 576, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 8192, + "n": 6144, + "k": 6144, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "zai-org/GLM-5-FP8/down_proj", + "zai-org/GLM-5-FP8/gate_up_proj", + "zai-org/GLM-5.1-FP8/down_proj", + "zai-org/GLM-5.1-FP8/gate_up_proj", + "zai-org/GLM-5.2-FP8/down_proj", + "zai-org/GLM-5.2-FP8/gate_up_proj" + ] }, { "type": "gemm", "args": { - "m": 11, - "n": 1536, - "k": 7168, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 8192, + "n": 6144, + "k": 8192, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/GLM-5.1-MXFP4/o_proj", + "amd/GLM-5.2-MXFP4/o_proj", + "amd/MiniMax-M3-MXFP4/o_proj", + "nvidia/GLM-5-NVFP4/o_proj", + "nvidia/GLM-5.1-NVFP4/o_proj", + "nvidia/GLM-5.2-NVFP4/o_proj" + ] }, { "type": "gemm", "args": { - "m": 11, - "n": 24576, - "k": 1536, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 8192, + "n": 7168, + "k": 512, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/GLM-5.1-MXFP4/kv_b_proj", + "amd/GLM-5.2-MXFP4/kv_b_proj", + "amd/Kimi-K2.5-MXFP4/o_proj", + "moonshotai/Kimi-K2.5/o_proj", + "nvidia/GLM-5-NVFP4/kv_b_proj", + "nvidia/GLM-5.1-NVFP4/kv_b_proj", + "nvidia/GLM-5.2-NVFP4/kv_b_proj", + "nvidia/Kimi-K2.5-NVFP4/o_proj", + "nvidia/Kimi-K2.6-NVFP4/o_proj" + ] }, { "type": "gemm", "args": { - "m": 11, - "n": 576, - "k": 7168, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 8192, + "n": 7168, + "k": 512, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "zai-org/GLM-5-FP8/kv_b_proj", + "zai-org/GLM-5.1-FP8/kv_b_proj", + "zai-org/GLM-5.2-FP8/kv_b_proj" + ] }, { "type": "gemm", "args": { - "m": 11, - "n": 32768, - "k": 512, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 8192, + "n": 7168, + "k": 576, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/down_proj" + ] }, { "type": "gemm", "args": { - "m": 11, + "m": 8192, "n": 7168, - "k": 16384, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "k": 1024, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/Kimi-K2.5-MXFP4/o_proj", + "moonshotai/Kimi-K2.5/o_proj", + "nvidia/Kimi-K2.5-NVFP4/o_proj", + "nvidia/Kimi-K2.6-NVFP4/o_proj" + ] }, { "type": "gemm", "args": { - "m": 11, + "m": 8192, "n": 7168, - "k": 576, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "k": 1024, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/o_proj", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] }, { "type": "gemm", "args": { - "m": 144384, - "n": 1536, - "k": 7168, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 8192, + "n": 7168, + "k": 1152, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/down_proj" + ] }, { "type": "gemm", "args": { - "m": 144384, - "n": 24576, - "k": 1536, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 8192, + "n": 7168, + "k": 2048, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/Kimi-K2.5-MXFP4/o_proj", + "moonshotai/Kimi-K2.5/o_proj", + "nvidia/Kimi-K2.5-NVFP4/o_proj", + "nvidia/Kimi-K2.6-NVFP4/o_proj" + ] }, { "type": "gemm", "args": { - "m": 144384, - "n": 576, - "k": 7168, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 8192, + "n": 7168, + "k": 2048, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/o_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/o_proj" + ] }, { "type": "gemm", "args": { - "m": 144384, - "n": 32768, - "k": 512, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 8192, + "n": 7168, + "k": 2048, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 32 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 1, + 32 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "amd/DeepSeek-R1-0528-MXFP4-Preview/o_proj", + "amd/DeepSeek-R1-0528-MXFP4-v2/o_proj", + "amd/DeepSeek-R1-0528-MXFP4/o_proj" + ] }, { "type": "gemm", "args": { - "m": 144384, + "m": 8192, "n": 7168, - "k": 16384, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "k": 2048, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/o_proj", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] }, { "type": "gemm", "args": { - "m": 144384, - "n": 4608, - "k": 7168, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 8192, + "n": 7168, + "k": 2304, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "moonshotai/Kimi-K2.5/down_proj", + "nvidia/Kimi-K2.6-NVFP4/down_proj" + ] }, { "type": "gemm", "args": { - "m": 144384, + "m": 8192, "n": 7168, - "k": 4608, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "k": 2304, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/down_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/down_proj", + "nvidia/Kimi-K2.5-NVFP4/down_proj" + ] }, { "type": "gemm", "args": { - "m": 282, - "n": 1536, - "k": 7168, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 8192, + "n": 7168, + "k": 2304, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 32 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 1, + 32 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "amd/DeepSeek-R1-0528-MXFP4-Preview/down_proj", + "amd/DeepSeek-R1-0528-MXFP4-v2/down_proj", + "amd/DeepSeek-R1-0528-MXFP4/down_proj", + "amd/Kimi-K2.5-MXFP4/down_proj" + ] }, { "type": "gemm", "args": { - "m": 282, - "n": 24576, - "k": 1536, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 8192, + "n": 7168, + "k": 2304, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/down_proj" + ] }, { "type": "gemm", "args": { - "m": 282, - "n": 576, - "k": 7168, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 8192, + "n": 7168, + "k": 4096, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/o_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/o_proj" + ] }, { "type": "gemm", "args": { - "m": 282, - "n": 32768, - "k": 512, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 8192, + "n": 7168, + "k": 4096, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 32 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 1, + 32 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "amd/DeepSeek-R1-0528-MXFP4-Preview/o_proj", + "amd/DeepSeek-R1-0528-MXFP4-v2/o_proj", + "amd/DeepSeek-R1-0528-MXFP4/o_proj" + ] }, { "type": "gemm", "args": { - "m": 282, + "m": 8192, "n": 7168, - "k": 16384, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "k": 4096, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/o_proj", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] }, { "type": "gemm", "args": { - "m": 282, - "n": 4608, - "k": 7168, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 8192, + "n": 7168, + "k": 4608, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "moonshotai/Kimi-K2.5/down_proj", + "nvidia/Kimi-K2.6-NVFP4/down_proj" + ] }, { "type": "gemm", "args": { - "m": 282, + "m": 8192, "n": 7168, "k": 4608, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/down_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/down_proj", + "nvidia/Kimi-K2.5-NVFP4/down_proj" + ] }, { "type": "gemm", "args": { - "m": 18048, - "n": 1536, - "k": 7168, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 8192, + "n": 7168, + "k": 4608, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 32 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 1, + 32 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "amd/DeepSeek-R1-0528-MXFP4-Preview/down_proj", + "amd/DeepSeek-R1-0528-MXFP4-v2/down_proj", + "amd/DeepSeek-R1-0528-MXFP4/down_proj", + "amd/Kimi-K2.5-MXFP4/down_proj" + ] }, { "type": "gemm", "args": { - "m": 18048, - "n": 24576, - "k": 1536, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 8192, + "n": 7168, + "k": 4608, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/down_proj" + ] }, { "type": "gemm", "args": { - "m": 18048, - "n": 576, - "k": 7168, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 8192, + "n": 7168, + "k": 8192, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/Kimi-K2.5-MXFP4/o_proj", + "moonshotai/Kimi-K2.5/o_proj", + "nvidia/Kimi-K2.5-NVFP4/o_proj", + "nvidia/Kimi-K2.6-NVFP4/o_proj" + ] }, { "type": "gemm", "args": { - "m": 18048, - "n": 32768, - "k": 512, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 8192, + "n": 7168, + "k": 8192, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] }, { "type": "gemm", "args": { - "m": 18048, + "m": 8192, "n": 7168, "k": 16384, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/o_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/o_proj" + ] }, { "type": "gemm", "args": { - "m": 18048, + "m": 8192, "n": 7168, - "k": 576, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "k": 16384, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 32 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 1, + 32 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "amd/DeepSeek-R1-0528-MXFP4-Preview/o_proj", + "amd/DeepSeek-R1-0528-MXFP4-v2/o_proj", + "amd/DeepSeek-R1-0528-MXFP4/o_proj" + ] }, { "type": "gemm", "args": { - "m": 170496, - "n": 4608, - "k": 7168, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 8192, + "n": 7168, + "k": 16384, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/o_proj", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] }, { "type": "gemm", "args": { - "m": 170496, + "m": 8192, "n": 7168, - "k": 4608, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "k": 16384, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] }, { "type": "gemm", "args": { - "m": 333, - "n": 1536, - "k": 7168, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 8192, + "n": 7168, + "k": 65536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] }, { "type": "gemm", "args": { - "m": 333, - "n": 24576, - "k": 1536, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 8192, + "n": 8192, + "k": 512, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/Kimi-K2.5-MXFP4/kv_b_proj", + "moonshotai/Kimi-K2.5/kv_b_proj", + "nvidia/DeepSeek-R1-0528-FP4-V2/kv_b_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/kv_b_proj", + "nvidia/Kimi-K2.5-NVFP4/kv_b_proj", + "nvidia/Kimi-K2.6-NVFP4/kv_b_proj" + ] }, { "type": "gemm", "args": { - "m": 333, - "n": 576, - "k": 7168, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 8192, + "n": 8192, + "k": 512, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 32 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 1, + 32 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "amd/DeepSeek-R1-0528-MXFP4-Preview/kv_b_proj", + "amd/DeepSeek-R1-0528-MXFP4-v2/kv_b_proj", + "amd/DeepSeek-R1-0528-MXFP4/kv_b_proj" + ] }, { "type": "gemm", "args": { - "m": 333, - "n": 32768, + "m": 8192, + "n": 8192, "k": 512, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/kv_b_proj" + ] }, { "type": "gemm", "args": { - "m": 333, - "n": 7168, - "k": 16384, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 8192, + "n": 8192, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_b" + ] }, { "type": "gemm", "args": { - "m": 333, - "n": 4608, - "k": 7168, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 8192, + "n": 8192, + "k": 2048, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/GLM-5.1-MXFP4/q_b_proj", + "amd/GLM-5.2-MXFP4/q_b_proj", + "nvidia/GLM-5-NVFP4/q_b_proj", + "nvidia/GLM-5.1-NVFP4/q_b_proj", + "nvidia/GLM-5.2-NVFP4/q_b_proj" + ] }, { "type": "gemm", "args": { - "m": 333, - "n": 7168, - "k": 4608, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 8192, + "n": 8192, + "k": 8192, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] }, { "type": "gemm", "args": { - "m": 42624, - "n": 1536, - "k": 7168, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 8192, + "n": 8192, + "k": 8192, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [] }, { "type": "gemm", "args": { - "m": 42624, - "n": 24576, - "k": 1536, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 8192, + "n": 8192, + "k": 8192, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 32 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 1, + 32 + ] + } + }, + "out": "bf16" + }, + "sources": [] }, { "type": "gemm", "args": { - "m": 42624, - "n": 576, - "k": 7168, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 8192, + "n": 8192, + "k": 8192, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] }, { "type": "gemm", "args": { - "m": 42624, - "n": 32768, - "k": 512, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 8192, + "n": 12288, + "k": 1536, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/Kimi-K2.5-MXFP4/q_b_proj", + "moonshotai/Kimi-K2.5/q_b_proj", + "nvidia/DeepSeek-R1-0528-FP4-V2/q_b_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/q_b_proj", + "nvidia/Kimi-K2.5-NVFP4/q_b_proj", + "nvidia/Kimi-K2.6-NVFP4/q_b_proj" + ] }, { "type": "gemm", "args": { - "m": 42624, - "n": 7168, - "k": 16384, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 8192, + "n": 14336, + "k": 512, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/GLM-5.1-MXFP4/kv_b_proj", + "amd/GLM-5.2-MXFP4/kv_b_proj", + "nvidia/GLM-5-NVFP4/kv_b_proj", + "nvidia/GLM-5.1-NVFP4/kv_b_proj", + "nvidia/GLM-5.2-NVFP4/kv_b_proj" + ] }, { "type": "gemm", "args": { - "m": 42624, - "n": 1152, - "k": 7168, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 8192, + "n": 15360, + "k": 1024, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] }, { "type": "gemm", "args": { - "m": 42624, - "n": 7168, - "k": 1152, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 8192, + "n": 16384, + "k": 512, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/Kimi-K2.5-MXFP4/kv_b_proj", + "moonshotai/Kimi-K2.5/kv_b_proj", + "nvidia/DeepSeek-R1-0528-FP4-V2/kv_b_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/kv_b_proj", + "nvidia/Kimi-K2.5-NVFP4/kv_b_proj", + "nvidia/Kimi-K2.6-NVFP4/kv_b_proj" + ] }, { "type": "gemm", "args": { - "m": 576768, - "n": 1536, - "k": 7168, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 8192, + "n": 16384, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_b" + ] }, { "type": "gemm", "args": { - "m": 576768, + "m": 8192, "n": 24576, "k": 1536, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/q_b_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/q_b_proj" + ] }, { "type": "gemm", "args": { - "m": 576768, - "n": 576, - "k": 7168, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 8192, + "n": 24576, + "k": 1536, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 32 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 1, + 32 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "amd/DeepSeek-R1-0528-MXFP4-Preview/q_b_proj", + "amd/DeepSeek-R1-0528-MXFP4-v2/q_b_proj", + "amd/DeepSeek-R1-0528-MXFP4/q_b_proj" + ] }, { "type": "gemm", "args": { - "m": 576768, - "n": 32768, - "k": 512, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 8192, + "n": 24576, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/q_b_proj" + ] }, { "type": "gemm", "args": { - "m": 576768, - "n": 7168, - "k": 16384, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 8192, + "n": 30720, + "k": 1024, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] }, { "type": "gemm", "args": { - "m": 576768, - "n": 4608, - "k": 7168, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 8192, + "n": 32768, + "k": 512, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/kv_b_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/kv_b_proj" + ] }, { "type": "gemm", "args": { - "m": 576768, - "n": 7168, - "k": 4608, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 8192, + "n": 32768, + "k": 512, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 32 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 1, + 32 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "amd/DeepSeek-R1-0528-MXFP4-Preview/kv_b_proj", + "amd/DeepSeek-R1-0528-MXFP4-v2/kv_b_proj", + "amd/DeepSeek-R1-0528-MXFP4/kv_b_proj" + ] }, { "type": "gemm", "args": { - "m": 1126, - "n": 1536, - "k": 7168, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 8192, + "n": 32768, + "k": 512, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/kv_b_proj" + ] }, { "type": "gemm", "args": { - "m": 1126, - "n": 24576, + "m": 8192, + "n": 65536, "k": 1536, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_b" + ] }, { "type": "gemm", "args": { - "m": 1126, - "n": 576, - "k": 7168, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 8192, + "n": 122880, + "k": 1024, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] }, { "type": "gemm", "args": { - "m": 1126, - "n": 32768, - "k": 512, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 8609, + "n": 576, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/Kimi-K2.5-MXFP4/kv_a_proj_with_mqa", + "moonshotai/Kimi-K2.5/kv_a_proj_with_mqa", + "nvidia/DeepSeek-R1-0528-FP4-V2/kv_a_proj_with_mqa", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/kv_a_proj_with_mqa", + "nvidia/Kimi-K2.5-NVFP4/kv_a_proj_with_mqa", + "nvidia/Kimi-K2.6-NVFP4/kv_a_proj_with_mqa" + ] }, { "type": "gemm", "args": { - "m": 1126, - "n": 7168, - "k": 16384, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 8609, + "n": 576, + "k": 7168, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 32 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 1, + 32 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "amd/DeepSeek-R1-0528-MXFP4-Preview/kv_a_proj_with_mqa", + "amd/DeepSeek-R1-0528-MXFP4-v2/kv_a_proj_with_mqa", + "amd/DeepSeek-R1-0528-MXFP4/kv_a_proj_with_mqa" + ] }, { "type": "gemm", "args": { - "m": 1126, - "n": 4608, + "m": 8609, + "n": 1536, "k": 7168, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/Kimi-K2.5-MXFP4/q_a_proj", + "moonshotai/Kimi-K2.5/q_a_proj", + "nvidia/DeepSeek-R1-0528-FP4-V2/q_a_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/q_a_proj", + "nvidia/Kimi-K2.5-NVFP4/q_a_proj", + "nvidia/Kimi-K2.6-NVFP4/q_a_proj" + ] }, { "type": "gemm", "args": { - "m": 1126, - "n": 7168, - "k": 4608, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 8609, + "n": 1536, + "k": 7168, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 32 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 1, + 32 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "amd/DeepSeek-R1-0528-MXFP4-Preview/q_a_proj", + "amd/DeepSeek-R1-0528-MXFP4-v2/q_a_proj", + "amd/DeepSeek-R1-0528-MXFP4/q_a_proj" + ] }, { "type": "gemm", "args": { - "m": 144192, + "m": 8609, "n": 1536, "k": 7168, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w3" + ] }, { "type": "gemm", "args": { - "m": 144192, - "n": 24576, - "k": 1536, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 8609, + "n": 7168, + "k": 9216, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/down_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/down_proj", + "nvidia/Kimi-K2.5-NVFP4/down_proj" + ] }, { "type": "gemm", "args": { - "m": 144192, - "n": 576, - "k": 7168, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 8609, + "n": 7168, + "k": 16384, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/o_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/o_proj" + ] }, { "type": "gemm", "args": { - "m": 144192, - "n": 32768, - "k": 512, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 8609, + "n": 7168, + "k": 16384, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 32 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 1, + 32 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "amd/DeepSeek-R1-0528-MXFP4-Preview/o_proj", + "amd/DeepSeek-R1-0528-MXFP4-v2/o_proj", + "amd/DeepSeek-R1-0528-MXFP4/o_proj" + ] }, { "type": "gemm", "args": { - "m": 144192, + "m": 8609, "n": 7168, "k": 16384, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] }, { "type": "gemm", "args": { - "m": 144192, - "n": 1152, + "m": 8609, + "n": 9216, "k": 7168, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/gate_up_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/gate_up_proj", + "nvidia/Kimi-K2.5-NVFP4/gate_up_proj" + ] }, { "type": "gemm", "args": { - "m": 144192, - "n": 7168, - "k": 1152, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 8609, + "n": 9216, + "k": 7168, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 32 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 1, + 32 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "amd/DeepSeek-R1-0528-MXFP4-Preview/gate_up_proj", + "amd/DeepSeek-R1-0528-MXFP4-v2/gate_up_proj", + "amd/DeepSeek-R1-0528-MXFP4/gate_up_proj", + "amd/Kimi-K2.5-MXFP4/gate_up_proj" + ] }, { "type": "gemm", "args": { - "m": 281, - "n": 1536, - "k": 7168, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 8609, + "n": 24576, + "k": 1536, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/q_b_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/q_b_proj" + ] }, { "type": "gemm", "args": { - "m": 281, + "m": 8609, "n": 24576, "k": 1536, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 32 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 1, + 32 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "amd/DeepSeek-R1-0528-MXFP4-Preview/q_b_proj", + "amd/DeepSeek-R1-0528-MXFP4-v2/q_b_proj", + "amd/DeepSeek-R1-0528-MXFP4/q_b_proj" + ] }, { "type": "gemm", "args": { - "m": 281, - "n": 576, - "k": 7168, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 8609, + "n": 32768, + "k": 512, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/kv_b_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/kv_b_proj" + ] }, { "type": "gemm", "args": { - "m": 281, + "m": 8609, "n": 32768, "k": 512, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 32 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 1, + 32 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "amd/DeepSeek-R1-0528-MXFP4-Preview/kv_b_proj", + "amd/DeepSeek-R1-0528-MXFP4-v2/kv_b_proj", + "amd/DeepSeek-R1-0528-MXFP4/kv_b_proj" + ] }, { "type": "gemm", "args": { - "m": 281, - "n": 7168, - "k": 16384, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 9216, + "n": 576, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/kv_a_proj_with_mqa" + ] }, { "type": "gemm", "args": { - "m": 281, + "m": 9216, "n": 1152, "k": 7168, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/gate_up_proj" + ] }, { "type": "gemm", "args": { - "m": 281, - "n": 7168, - "k": 1152, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 9216, + "n": 1536, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/q_b_proj" + ] }, { "type": "gemm", "args": { - "m": 2097152, - "n": 4608, + "m": 9216, + "n": 1536, "k": 7168, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 2097152, - "n": 7168, - "k": 4608, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/gate_up_proj", + "deepseek-ai/DeepSeek-R1-0528/q_a_proj", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w3" + ] }, { "type": "gemm", "args": { - "m": 1024, - "n": 4608, + "m": 9216, + "n": 1536, "k": 7168, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w3" + ] }, { "type": "gemm", "args": { - "m": 1024, - "n": 7168, - "k": 4608, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 9216, + "n": 2048, + "k": 512, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/kv_b_proj" + ] }, { "type": "gemm", "args": { - "m": 1, - "n": 4608, + "m": 9216, + "n": 2304, "k": 7168, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 1, - "n": 7168, - "k": 4608, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/gate_up_proj" + ] }, { "type": "gemm", "args": { - "m": 6, - "n": 24576, + "m": 9216, + "n": 3072, "k": 1536, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "MiniMaxAI/MiniMax-M2.5/experts.down_proj", + "MiniMaxAI/MiniMax-M2.5/o_proj", + "deepseek-ai/DeepSeek-R1-0528/q_b_proj" + ] }, { "type": "gemm", "args": { - "m": 6, - "n": 32768, + "m": 9216, + "n": 4096, "k": 512, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "Qwen/Qwen3.5-397B-A17B-FP8/o_proj", + "Qwen/Qwen3.5-397B-A17B-FP8/shared_expert.down_proj", + "deepseek-ai/DeepSeek-R1-0528/kv_b_proj" + ] }, { "type": "gemm", "args": { - "m": 6, + "m": 9216, "n": 7168, - "k": 16384, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 6, - "n": 4608, - "k": 7168, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "k": 1024, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/o_proj", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] }, { "type": "gemm", "args": { - "m": 6, + "m": 9216, "n": 7168, - "k": 4608, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "k": 1152, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/down_proj" + ] }, { "type": "gemm", "args": { - "m": 24, - "n": 1536, - "k": 7168, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 9216, + "n": 7168, + "k": 2048, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/o_proj", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] }, { "type": "gemm", "args": { - "m": 24, - "n": 3072, - "k": 1536, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 9216, + "n": 7168, + "k": 2304, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/down_proj" + ] }, { "type": "gemm", "args": { - "m": 24, + "m": 10240, "n": 576, "k": 7168, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/Kimi-K2.5-MXFP4/kv_a_proj_with_mqa", + "moonshotai/Kimi-K2.5/kv_a_proj_with_mqa", + "nvidia/DeepSeek-R1-0528-FP4-V2/kv_a_proj_with_mqa", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/kv_a_proj_with_mqa", + "nvidia/Kimi-K2.5-NVFP4/kv_a_proj_with_mqa", + "nvidia/Kimi-K2.6-NVFP4/kv_a_proj_with_mqa" + ] }, { "type": "gemm", "args": { - "m": 24, - "n": 4096, - "k": 512, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 10240, + "n": 1536, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/Kimi-K2.5-MXFP4/q_a_proj", + "moonshotai/Kimi-K2.5/q_a_proj", + "nvidia/DeepSeek-R1-0528-FP4-V2/q_a_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/q_a_proj", + "nvidia/Kimi-K2.5-NVFP4/q_a_proj", + "nvidia/Kimi-K2.6-NVFP4/q_a_proj" + ] }, { "type": "gemm", "args": { - "m": 24, - "n": 7168, - "k": 2048, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 10240, + "n": 2304, + "k": 7168, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/gate_up_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/gate_up_proj", + "nvidia/Kimi-K2.5-NVFP4/gate_up_proj" + ] }, { "type": "gemm", "args": { - "m": 24, + "m": 10240, "n": 2304, "k": 7168, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/gate_up_proj" + ] }, { "type": "gemm", "args": { - "m": 24, - "n": 7168, - "k": 2304, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 10240, + "n": 3072, + "k": 1536, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/Kimi-K2.5-MXFP4/q_b_proj", + "amd/MiniMax-M2.5-MXFP4/o_proj", + "moonshotai/Kimi-K2.5/q_b_proj", + "nvidia/DeepSeek-R1-0528-FP4-V2/q_b_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/q_b_proj", + "nvidia/Kimi-K2.5-NVFP4/q_b_proj", + "nvidia/Kimi-K2.6-NVFP4/q_b_proj", + "nvidia/MiniMax-M2.5-NVFP4/o_proj" + ] }, { "type": "gemm", "args": { - "m": 21504, - "n": 1536, - "k": 7168, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 10240, + "n": 3072, + "k": 1536, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/MiniMax-M2.5-NVFP4/experts.down_proj" + ] }, { "type": "gemm", "args": { - "m": 21504, - "n": 24576, + "m": 10240, + "n": 3072, "k": 1536, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 32 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 1, + 32 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "amd/DeepSeek-R1-0528-MXFP4-Preview/q_b_proj", + "amd/DeepSeek-R1-0528-MXFP4-v2/q_b_proj", + "amd/DeepSeek-R1-0528-MXFP4/q_b_proj", + "amd/MiniMax-M2.5-MXFP4/experts.down_proj" + ] }, { "type": "gemm", "args": { - "m": 21504, - "n": 576, - "k": 7168, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 10240, + "n": 3072, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "MiniMaxAI/MiniMax-M2.5/experts.down_proj", + "MiniMaxAI/MiniMax-M2.5/o_proj", + "deepseek-ai/DeepSeek-R1-0528/q_b_proj" + ] }, { "type": "gemm", "args": { - "m": 21504, - "n": 32768, + "m": 10240, + "n": 4096, "k": 512, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "Qwen/Qwen3.5-397B-A17B/o_proj", + "Qwen/Qwen3.5-397B-A17B/shared_expert.down_proj", + "amd/Kimi-K2.5-MXFP4/kv_b_proj", + "amd/Qwen3.5-397B-A17B-MXFP4/o_proj", + "amd/Qwen3.5-397B-A17B-MXFP4/shared_expert.down_proj", + "moonshotai/Kimi-K2.5/kv_b_proj", + "nvidia/DeepSeek-R1-0528-FP4-V2/kv_b_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/kv_b_proj", + "nvidia/Kimi-K2.5-NVFP4/kv_b_proj", + "nvidia/Kimi-K2.6-NVFP4/kv_b_proj", + "nvidia/Qwen3.5-397B-A17B-NVFP4/o_proj", + "nvidia/Qwen3.5-397B-A17B-NVFP4/shared_expert.down_proj" + ] }, { "type": "gemm", "args": { - "m": 21504, - "n": 7168, - "k": 16384, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 10240, + "n": 4096, + "k": 512, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 32 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 1, + 32 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "amd/DeepSeek-R1-0528-MXFP4-Preview/kv_b_proj", + "amd/DeepSeek-R1-0528-MXFP4-v2/kv_b_proj", + "amd/DeepSeek-R1-0528-MXFP4/kv_b_proj", + "amd/Qwen3.5-397B-A17B-MXFP4-AttnFP8-V2/shared_expert.down_proj" + ] }, { "type": "gemm", "args": { - "m": 21504, - "n": 4608, - "k": 7168, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 10240, + "n": 4096, + "k": 512, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "Qwen/Qwen3.5-397B-A17B-FP8/o_proj", + "Qwen/Qwen3.5-397B-A17B-FP8/shared_expert.down_proj", + "deepseek-ai/DeepSeek-R1-0528/kv_b_proj" + ] }, { "type": "gemm", "args": { - "m": 21504, - "n": 7168, - "k": 4608, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 10240, + "n": 4608, + "k": 7168, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/gate_up_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/gate_up_proj", + "nvidia/Kimi-K2.5-NVFP4/gate_up_proj" + ] }, { "type": "gemm", "args": { - "m": 21, + "m": 10240, "n": 4608, "k": 7168, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 32 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 1, + 32 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "amd/DeepSeek-R1-0528-MXFP4-Preview/gate_up_proj", + "amd/DeepSeek-R1-0528-MXFP4-v2/gate_up_proj", + "amd/DeepSeek-R1-0528-MXFP4/gate_up_proj", + "amd/Kimi-K2.5-MXFP4/gate_up_proj" + ] }, { "type": "gemm", "args": { - "m": 21, - "n": 7168, - "k": 4608, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 10240, + "n": 6144, + "k": 1536, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/GLM-5.1-MXFP4/down_proj", + "amd/GLM-5.2-MXFP4/down_proj", + "amd/Kimi-K2.5-MXFP4/q_b_proj", + "amd/MiniMax-M3-MXFP4/down_proj", + "moonshotai/Kimi-K2.5/q_b_proj", + "nvidia/DeepSeek-R1-0528-FP4-V2/q_b_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/q_b_proj", + "nvidia/GLM-5.1-NVFP4/down_proj", + "nvidia/GLM-5.2-NVFP4/down_proj", + "nvidia/Kimi-K2.5-NVFP4/q_b_proj", + "nvidia/Kimi-K2.6-NVFP4/q_b_proj" + ] }, { "type": "gemm", "args": { - "m": 86016, - "n": 1536, - "k": 7168, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 10240, + "n": 6144, + "k": 1536, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/GLM-5-NVFP4/down_proj" + ] }, { "type": "gemm", "args": { - "m": 86016, - "n": 3072, + "m": 10240, + "n": 6144, "k": 1536, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 32 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 1, + 32 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "amd/DeepSeek-R1-0528-MXFP4-Preview/q_b_proj", + "amd/DeepSeek-R1-0528-MXFP4-v2/q_b_proj", + "amd/DeepSeek-R1-0528-MXFP4/q_b_proj" + ] }, { "type": "gemm", "args": { - "m": 86016, - "n": 576, - "k": 7168, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 10240, + "n": 7168, + "k": 2048, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/Kimi-K2.5-MXFP4/o_proj", + "moonshotai/Kimi-K2.5/o_proj", + "nvidia/Kimi-K2.5-NVFP4/o_proj", + "nvidia/Kimi-K2.6-NVFP4/o_proj" + ] }, { "type": "gemm", "args": { - "m": 86016, - "n": 4096, - "k": 512, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 10240, + "n": 7168, + "k": 2048, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/o_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/o_proj" + ] }, { "type": "gemm", "args": { - "m": 86016, + "m": 10240, "n": 7168, "k": 2048, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 32 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 1, + 32 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "amd/DeepSeek-R1-0528-MXFP4-Preview/o_proj", + "amd/DeepSeek-R1-0528-MXFP4-v2/o_proj", + "amd/DeepSeek-R1-0528-MXFP4/o_proj" + ] }, { "type": "gemm", "args": { - "m": 86016, - "n": 2304, - "k": 7168, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 10240, + "n": 7168, + "k": 2048, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/o_proj", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] }, { "type": "gemm", "args": { - "m": 86016, + "m": 10240, "n": 7168, "k": 2304, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/down_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/down_proj", + "nvidia/Kimi-K2.5-NVFP4/down_proj" + ] }, { "type": "gemm", "args": { - "m": 84, - "n": 1536, - "k": 7168, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 10240, + "n": 7168, + "k": 2304, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 32 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 1, + 32 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "amd/DeepSeek-R1-0528-MXFP4-Preview/down_proj", + "amd/DeepSeek-R1-0528-MXFP4-v2/down_proj", + "amd/DeepSeek-R1-0528-MXFP4/down_proj", + "amd/Kimi-K2.5-MXFP4/down_proj" + ] }, { "type": "gemm", "args": { - "m": 84, - "n": 3072, - "k": 1536, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 10240, + "n": 7168, + "k": 2304, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/down_proj" + ] }, { "type": "gemm", "args": { - "m": 84, - "n": 576, - "k": 7168, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 10240, + "n": 7168, + "k": 4096, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/o_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/o_proj" + ] }, { "type": "gemm", "args": { - "m": 84, - "n": 4096, - "k": 512, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 10240, + "n": 7168, + "k": 4096, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 32 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 1, + 32 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "amd/DeepSeek-R1-0528-MXFP4-Preview/o_proj", + "amd/DeepSeek-R1-0528-MXFP4-v2/o_proj", + "amd/DeepSeek-R1-0528-MXFP4/o_proj" + ] }, { "type": "gemm", "args": { - "m": 84, + "m": 10240, "n": 7168, - "k": 2048, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "k": 4608, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/down_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/down_proj", + "nvidia/Kimi-K2.5-NVFP4/down_proj" + ] }, { "type": "gemm", "args": { - "m": 84, - "n": 2304, - "k": 7168, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 10240, + "n": 7168, + "k": 8192, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/Kimi-K2.5-MXFP4/o_proj", + "moonshotai/Kimi-K2.5/o_proj", + "nvidia/Kimi-K2.5-NVFP4/o_proj", + "nvidia/Kimi-K2.6-NVFP4/o_proj" + ] }, { "type": "gemm", "args": { - "m": 84, + "m": 10240, "n": 7168, - "k": 2304, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "k": 9216, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/down_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/down_proj", + "nvidia/Kimi-K2.5-NVFP4/down_proj" + ] }, { "type": "gemm", "args": { - "m": 314624, - "n": 4608, - "k": 7168, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 10240, + "n": 7168, + "k": 16384, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/o_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/o_proj" + ] }, { "type": "gemm", "args": { - "m": 314624, - "n": 7168, - "k": 4608, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 10240, + "n": 8192, + "k": 512, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/Kimi-K2.5-MXFP4/kv_b_proj", + "moonshotai/Kimi-K2.5/kv_b_proj", + "nvidia/DeepSeek-R1-0528-FP4-V2/kv_b_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/kv_b_proj", + "nvidia/Kimi-K2.5-NVFP4/kv_b_proj", + "nvidia/Kimi-K2.6-NVFP4/kv_b_proj" + ] }, { "type": "gemm", "args": { - "m": 307, - "n": 4608, - "k": 7168, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 10240, + "n": 8192, + "k": 512, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 32 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 1, + 32 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "amd/DeepSeek-R1-0528-MXFP4-Preview/kv_b_proj", + "amd/DeepSeek-R1-0528-MXFP4-v2/kv_b_proj", + "amd/DeepSeek-R1-0528-MXFP4/kv_b_proj" + ] }, { "type": "gemm", "args": { - "m": 307, - "n": 7168, - "k": 4608, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 10240, + "n": 9216, + "k": 7168, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/gate_up_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/gate_up_proj", + "nvidia/Kimi-K2.5-NVFP4/gate_up_proj" + ] }, { "type": "gemm", "args": { - "m": 39328, - "n": 1536, - "k": 7168, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 10240, + "n": 12288, + "k": 1536, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/Kimi-K2.5-MXFP4/q_b_proj", + "moonshotai/Kimi-K2.5/q_b_proj", + "nvidia/DeepSeek-R1-0528-FP4-V2/q_b_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/q_b_proj", + "nvidia/Kimi-K2.5-NVFP4/q_b_proj", + "nvidia/Kimi-K2.6-NVFP4/q_b_proj" + ] }, { "type": "gemm", "args": { - "m": 39328, - "n": 24576, - "k": 1536, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 10240, + "n": 16384, + "k": 512, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/Kimi-K2.5-MXFP4/kv_b_proj", + "moonshotai/Kimi-K2.5/kv_b_proj", + "nvidia/DeepSeek-R1-0528-FP4-V2/kv_b_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/kv_b_proj", + "nvidia/Kimi-K2.5-NVFP4/kv_b_proj", + "nvidia/Kimi-K2.6-NVFP4/kv_b_proj" + ] }, { "type": "gemm", "args": { - "m": 39328, - "n": 576, - "k": 7168, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 10240, + "n": 24576, + "k": 1536, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/q_b_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/q_b_proj" + ] }, { "type": "gemm", "args": { - "m": 39328, + "m": 10240, "n": 32768, "k": 512, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/kv_b_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/kv_b_proj" + ] }, { "type": "gemm", "args": { - "m": 39328, - "n": 7168, - "k": 16384, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 10656, + "n": 576, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/Kimi-K2.5-MXFP4/kv_a_proj_with_mqa", + "moonshotai/Kimi-K2.5/kv_a_proj_with_mqa", + "nvidia/DeepSeek-R1-0528-FP4-V2/kv_a_proj_with_mqa", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/kv_a_proj_with_mqa", + "nvidia/Kimi-K2.5-NVFP4/kv_a_proj_with_mqa", + "nvidia/Kimi-K2.6-NVFP4/kv_a_proj_with_mqa" + ] }, { "type": "gemm", "args": { - "m": 39328, - "n": 7168, - "k": 576, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 10656, + "n": 576, + "k": 7168, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 32 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 1, + 32 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "amd/DeepSeek-R1-0528-MXFP4-Preview/kv_a_proj_with_mqa", + "amd/DeepSeek-R1-0528-MXFP4-v2/kv_a_proj_with_mqa", + "amd/DeepSeek-R1-0528-MXFP4/kv_a_proj_with_mqa" + ] }, { "type": "gemm", "args": { - "m": 563, + "m": 10656, "n": 1536, "k": 7168, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/Kimi-K2.5-MXFP4/q_a_proj", + "moonshotai/Kimi-K2.5/q_a_proj", + "nvidia/DeepSeek-R1-0528-FP4-V2/q_a_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/q_a_proj", + "nvidia/Kimi-K2.5-NVFP4/q_a_proj", + "nvidia/Kimi-K2.6-NVFP4/q_a_proj" + ] }, { "type": "gemm", "args": { - "m": 563, - "n": 24576, - "k": 1536, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 10656, + "n": 1536, + "k": 7168, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 32 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 1, + 32 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "amd/DeepSeek-R1-0528-MXFP4-Preview/q_a_proj", + "amd/DeepSeek-R1-0528-MXFP4-v2/q_a_proj", + "amd/DeepSeek-R1-0528-MXFP4/q_a_proj" + ] }, { "type": "gemm", "args": { - "m": 563, - "n": 576, + "m": 10656, + "n": 1536, "k": 7168, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w3" + ] }, { "type": "gemm", "args": { - "m": 563, - "n": 32768, - "k": 512, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 10656, + "n": 7168, + "k": 576, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-NVFP4-v2/down_proj" + ] }, { "type": "gemm", "args": { - "m": 563, + "m": 10656, "n": 7168, "k": 16384, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 563, - "n": 4608, - "k": 7168, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/o_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/o_proj" + ] }, { "type": "gemm", "args": { - "m": 563, + "m": 10656, "n": 7168, - "k": 4608, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 140, - "n": 1152, - "k": 7168, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "k": 16384, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 32 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 1, + 32 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "amd/DeepSeek-R1-0528-MXFP4-Preview/o_proj", + "amd/DeepSeek-R1-0528-MXFP4-v2/o_proj", + "amd/DeepSeek-R1-0528-MXFP4/o_proj" + ] }, { "type": "gemm", "args": { - "m": 140, + "m": 10656, "n": 7168, - "k": 1152, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "k": 16384, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] }, { "type": "gemm", "args": { - "m": 2202112, - "n": 1536, - "k": 7168, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 10656, + "n": 24576, + "k": 1536, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/q_b_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/q_b_proj" + ] }, { "type": "gemm", "args": { - "m": 2202112, + "m": 10656, "n": 24576, "k": 1536, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 32 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 1, + 32 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "amd/DeepSeek-R1-0528-MXFP4-Preview/q_b_proj", + "amd/DeepSeek-R1-0528-MXFP4-v2/q_b_proj", + "amd/DeepSeek-R1-0528-MXFP4/q_b_proj" + ] }, { "type": "gemm", "args": { - "m": 2202112, - "n": 576, - "k": 7168, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 10656, + "n": 32768, + "k": 512, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/kv_b_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/kv_b_proj" + ] }, { "type": "gemm", "args": { - "m": 2202112, + "m": 10656, "n": 32768, "k": 512, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 32 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 1, + 32 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "amd/DeepSeek-R1-0528-MXFP4-Preview/kv_b_proj", + "amd/DeepSeek-R1-0528-MXFP4-v2/kv_b_proj", + "amd/DeepSeek-R1-0528-MXFP4/kv_b_proj" + ] }, { "type": "gemm", "args": { - "m": 2202112, - "n": 7168, - "k": 16384, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 10922, + "n": 384, + "k": 7168, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [] }, { "type": "gemm", "args": { - "m": 2202112, - "n": 4608, + "m": 10922, + "n": 576, "k": 7168, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/Kimi-K2.5-MXFP4/kv_a_proj_with_mqa", + "moonshotai/Kimi-K2.5/kv_a_proj_with_mqa", + "nvidia/DeepSeek-R1-0528-FP4-V2/kv_a_proj_with_mqa", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/kv_a_proj_with_mqa", + "nvidia/Kimi-K2.5-NVFP4/kv_a_proj_with_mqa", + "nvidia/Kimi-K2.6-NVFP4/kv_a_proj_with_mqa" + ] }, { "type": "gemm", "args": { - "m": 2202112, - "n": 7168, - "k": 4608, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 10922, + "n": 576, + "k": 7168, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 32 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 1, + 32 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "amd/DeepSeek-R1-0528-MXFP4-Preview/kv_a_proj_with_mqa", + "amd/DeepSeek-R1-0528-MXFP4-v2/kv_a_proj_with_mqa", + "amd/DeepSeek-R1-0528-MXFP4/kv_a_proj_with_mqa" + ] }, { "type": "gemm", "args": { - "m": 2150, + "m": 10922, "n": 1536, "k": 7168, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/Kimi-K2.5-MXFP4/q_a_proj", + "moonshotai/Kimi-K2.5/q_a_proj", + "nvidia/DeepSeek-R1-0528-FP4-V2/q_a_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/q_a_proj", + "nvidia/Kimi-K2.5-NVFP4/q_a_proj", + "nvidia/Kimi-K2.6-NVFP4/q_a_proj" + ] }, { "type": "gemm", "args": { - "m": 2150, - "n": 24576, - "k": 1536, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 10922, + "n": 1536, + "k": 7168, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 32 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 1, + 32 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "amd/DeepSeek-R1-0528-MXFP4-Preview/q_a_proj", + "amd/DeepSeek-R1-0528-MXFP4-v2/q_a_proj", + "amd/DeepSeek-R1-0528-MXFP4/q_a_proj" + ] }, { "type": "gemm", "args": { - "m": 2150, - "n": 576, + "m": 10922, + "n": 1536, "k": 7168, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w3" + ] }, { "type": "gemm", "args": { - "m": 2150, - "n": 32768, - "k": 512, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 10922, + "n": 7168, + "k": 384, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-NVFP4-v2/down_proj" + ] }, { "type": "gemm", "args": { - "m": 2150, + "m": 10922, "n": 7168, "k": 16384, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/o_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/o_proj" + ] }, { "type": "gemm", "args": { - "m": 2150, - "n": 4608, - "k": 7168, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 10922, + "n": 7168, + "k": 16384, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 32 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 1, + 32 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "amd/DeepSeek-R1-0528-MXFP4-Preview/o_proj", + "amd/DeepSeek-R1-0528-MXFP4-v2/o_proj", + "amd/DeepSeek-R1-0528-MXFP4/o_proj" + ] }, { "type": "gemm", "args": { - "m": 2150, + "m": 10922, "n": 7168, - "k": 4608, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "k": 16384, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] }, { "type": "gemm", "args": { - "m": 1101056, - "n": 1536, - "k": 7168, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 10922, + "n": 24576, + "k": 1536, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/q_b_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/q_b_proj" + ] }, { "type": "gemm", "args": { - "m": 1101056, + "m": 10922, "n": 24576, "k": 1536, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 32 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 1, + 32 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "amd/DeepSeek-R1-0528-MXFP4-Preview/q_b_proj", + "amd/DeepSeek-R1-0528-MXFP4-v2/q_b_proj", + "amd/DeepSeek-R1-0528-MXFP4/q_b_proj" + ] }, { "type": "gemm", "args": { - "m": 1101056, - "n": 576, - "k": 7168, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 10922, + "n": 32768, + "k": 512, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/kv_b_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/kv_b_proj" + ] }, { "type": "gemm", "args": { - "m": 1101056, + "m": 10922, "n": 32768, "k": 512, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 32 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 1, + 32 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "amd/DeepSeek-R1-0528-MXFP4-Preview/kv_b_proj", + "amd/DeepSeek-R1-0528-MXFP4-v2/kv_b_proj", + "amd/DeepSeek-R1-0528-MXFP4/kv_b_proj" + ] }, { "type": "gemm", "args": { - "m": 1101056, - "n": 7168, - "k": 16384, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 11264, + "n": 576, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/Kimi-K2.5-MXFP4/kv_a_proj_with_mqa", + "moonshotai/Kimi-K2.5/kv_a_proj_with_mqa", + "nvidia/DeepSeek-R1-0528-FP4-V2/kv_a_proj_with_mqa", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/kv_a_proj_with_mqa", + "nvidia/Kimi-K2.5-NVFP4/kv_a_proj_with_mqa", + "nvidia/Kimi-K2.6-NVFP4/kv_a_proj_with_mqa" + ] }, { "type": "gemm", "args": { - "m": 1101056, - "n": 2304, + "m": 11264, + "n": 576, "k": 7168, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 32 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 1, + 32 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "amd/DeepSeek-R1-0528-MXFP4-Preview/kv_a_proj_with_mqa", + "amd/DeepSeek-R1-0528-MXFP4-v2/kv_a_proj_with_mqa", + "amd/DeepSeek-R1-0528-MXFP4/kv_a_proj_with_mqa" + ] }, { "type": "gemm", "args": { - "m": 1101056, - "n": 7168, - "k": 2304, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 11264, + "n": 1536, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/Kimi-K2.5-MXFP4/q_a_proj", + "moonshotai/Kimi-K2.5/q_a_proj", + "nvidia/DeepSeek-R1-0528-FP4-V2/q_a_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/q_a_proj", + "nvidia/Kimi-K2.5-NVFP4/q_a_proj", + "nvidia/Kimi-K2.6-NVFP4/q_a_proj" + ] }, { "type": "gemm", "args": { - "m": 3145728, - "n": 4608, + "m": 11264, + "n": 1536, "k": 7168, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 32 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 1, + 32 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "amd/DeepSeek-R1-0528-MXFP4-Preview/q_a_proj", + "amd/DeepSeek-R1-0528-MXFP4-v2/q_a_proj", + "amd/DeepSeek-R1-0528-MXFP4/q_a_proj" + ] }, { "type": "gemm", "args": { - "m": 3145728, - "n": 7168, - "k": 4608, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 11264, + "n": 1536, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w3" + ] }, { "type": "gemm", "args": { - "m": 1572864, + "m": 11264, "n": 2304, "k": 7168, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/gate_up_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/gate_up_proj", + "nvidia/Kimi-K2.5-NVFP4/gate_up_proj" + ] }, { "type": "gemm", "args": { - "m": 1572864, - "n": 7168, - "k": 2304, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 11264, + "n": 3072, + "k": 1536, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/Kimi-K2.5-MXFP4/q_b_proj", + "amd/MiniMax-M2.5-MXFP4/o_proj", + "moonshotai/Kimi-K2.5/q_b_proj", + "nvidia/DeepSeek-R1-0528-FP4-V2/q_b_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/q_b_proj", + "nvidia/Kimi-K2.5-NVFP4/q_b_proj", + "nvidia/Kimi-K2.6-NVFP4/q_b_proj", + "nvidia/MiniMax-M2.5-NVFP4/o_proj" + ] }, { "type": "gemm", "args": { - "m": 1536, - "n": 2304, - "k": 7168, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 11264, + "n": 3072, + "k": 1536, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/MiniMax-M2.5-NVFP4/experts.down_proj" + ] }, { "type": "gemm", "args": { - "m": 1536, - "n": 7168, - "k": 2304, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 11264, + "n": 3072, + "k": 1536, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 32 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 1, + 32 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "amd/DeepSeek-R1-0528-MXFP4-Preview/q_b_proj", + "amd/DeepSeek-R1-0528-MXFP4-v2/q_b_proj", + "amd/DeepSeek-R1-0528-MXFP4/q_b_proj", + "amd/MiniMax-M2.5-MXFP4/experts.down_proj" + ] }, { "type": "gemm", "args": { - "m": 49152, - "n": 4608, - "k": 7168, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 11264, + "n": 4096, + "k": 512, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "Qwen/Qwen3.5-397B-A17B/o_proj", + "Qwen/Qwen3.5-397B-A17B/shared_expert.down_proj", + "amd/Kimi-K2.5-MXFP4/kv_b_proj", + "amd/Qwen3.5-397B-A17B-MXFP4/o_proj", + "amd/Qwen3.5-397B-A17B-MXFP4/shared_expert.down_proj", + "moonshotai/Kimi-K2.5/kv_b_proj", + "nvidia/DeepSeek-R1-0528-FP4-V2/kv_b_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/kv_b_proj", + "nvidia/Kimi-K2.5-NVFP4/kv_b_proj", + "nvidia/Kimi-K2.6-NVFP4/kv_b_proj", + "nvidia/Qwen3.5-397B-A17B-NVFP4/o_proj", + "nvidia/Qwen3.5-397B-A17B-NVFP4/shared_expert.down_proj" + ] }, { "type": "gemm", "args": { - "m": 49152, - "n": 7168, - "k": 4608, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 11264, + "n": 4096, + "k": 512, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 32 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 1, + 32 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "amd/DeepSeek-R1-0528-MXFP4-Preview/kv_b_proj", + "amd/DeepSeek-R1-0528-MXFP4-v2/kv_b_proj", + "amd/DeepSeek-R1-0528-MXFP4/kv_b_proj", + "amd/Qwen3.5-397B-A17B-MXFP4-AttnFP8-V2/shared_expert.down_proj" + ] }, { "type": "gemm", "args": { - "m": 196608, - "n": 3072, - "k": 1536, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 11264, + "n": 7168, + "k": 2048, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/Kimi-K2.5-MXFP4/o_proj", + "moonshotai/Kimi-K2.5/o_proj", + "nvidia/Kimi-K2.5-NVFP4/o_proj", + "nvidia/Kimi-K2.6-NVFP4/o_proj" + ] }, { "type": "gemm", "args": { - "m": 196608, - "n": 4096, - "k": 512, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 11264, + "n": 7168, + "k": 2048, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/o_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/o_proj" + ] }, { "type": "gemm", "args": { - "m": 196608, + "m": 11264, "n": 7168, "k": 2048, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 32 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 1, + 32 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "amd/DeepSeek-R1-0528-MXFP4-Preview/o_proj", + "amd/DeepSeek-R1-0528-MXFP4-v2/o_proj", + "amd/DeepSeek-R1-0528-MXFP4/o_proj" + ] }, { "type": "gemm", "args": { - "m": 196608, - "n": 2304, - "k": 7168, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 11264, + "n": 7168, + "k": 2304, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/down_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/down_proj", + "nvidia/Kimi-K2.5-NVFP4/down_proj" + ] }, { "type": "gemm", "args": { - "m": 196608, + "m": 11264, "n": 7168, "k": 2304, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 32 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 1, + 32 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "amd/DeepSeek-R1-0528-MXFP4-Preview/down_proj", + "amd/DeepSeek-R1-0528-MXFP4-v2/down_proj", + "amd/DeepSeek-R1-0528-MXFP4/down_proj", + "amd/Kimi-K2.5-MXFP4/down_proj" + ] }, { "type": "gemm", "args": { - "m": 681984, - "n": 4608, + "m": 11520, + "n": 576, "k": 7168, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/Kimi-K2.5-MXFP4/kv_a_proj_with_mqa", + "moonshotai/Kimi-K2.5/kv_a_proj_with_mqa", + "nvidia/DeepSeek-R1-0528-FP4-V2/kv_a_proj_with_mqa", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/kv_a_proj_with_mqa", + "nvidia/Kimi-K2.5-NVFP4/kv_a_proj_with_mqa", + "nvidia/Kimi-K2.6-NVFP4/kv_a_proj_with_mqa" + ] }, { "type": "gemm", "args": { - "m": 681984, - "n": 7168, - "k": 4608, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 11520, + "n": 1536, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/Kimi-K2.5-MXFP4/q_a_proj", + "moonshotai/Kimi-K2.5/q_a_proj", + "nvidia/DeepSeek-R1-0528-FP4-V2/q_a_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/q_a_proj", + "nvidia/Kimi-K2.5-NVFP4/q_a_proj", + "nvidia/Kimi-K2.6-NVFP4/q_a_proj" + ] }, { "type": "gemm", "args": { - "m": 166, + "m": 11520, "n": 4608, "k": 7168, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/gate_up_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/gate_up_proj", + "nvidia/Kimi-K2.5-NVFP4/gate_up_proj" + ] }, { "type": "gemm", "args": { - "m": 166, - "n": 7168, - "k": 4608, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 11520, + "n": 4608, + "k": 7168, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 32 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 1, + 32 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "amd/DeepSeek-R1-0528-MXFP4-Preview/gate_up_proj", + "amd/DeepSeek-R1-0528-MXFP4-v2/gate_up_proj", + "amd/DeepSeek-R1-0528-MXFP4/gate_up_proj", + "amd/Kimi-K2.5-MXFP4/gate_up_proj" + ] }, { "type": "gemm", "args": { - "m": 1363968, - "n": 1536, - "k": 7168, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 11520, + "n": 7168, + "k": 4608, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/down_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/down_proj", + "nvidia/Kimi-K2.5-NVFP4/down_proj" + ] }, { "type": "gemm", "args": { - "m": 1363968, - "n": 24576, - "k": 1536, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 11520, + "n": 7168, + "k": 16384, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/o_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/o_proj" + ] }, { "type": "gemm", "args": { - "m": 1363968, - "n": 576, - "k": 7168, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 11520, + "n": 24576, + "k": 1536, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/q_b_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/q_b_proj" + ] }, { "type": "gemm", "args": { - "m": 1363968, + "m": 11520, "n": 32768, "k": 512, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/kv_b_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/kv_b_proj" + ] }, { "type": "gemm", "args": { - "m": 1363968, - "n": 7168, - "k": 16384, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 12288, + "n": 576, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/Kimi-K2.5-MXFP4/kv_a_proj_with_mqa", + "moonshotai/Kimi-K2.5/kv_a_proj_with_mqa", + "nvidia/DeepSeek-R1-0528-FP4-V2/kv_a_proj_with_mqa", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/kv_a_proj_with_mqa", + "nvidia/Kimi-K2.5-NVFP4/kv_a_proj_with_mqa", + "nvidia/Kimi-K2.6-NVFP4/kv_a_proj_with_mqa" + ] }, { "type": "gemm", "args": { - "m": 1363968, - "n": 4608, + "m": 12288, + "n": 1536, "k": 7168, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/Kimi-K2.5-MXFP4/q_a_proj", + "moonshotai/Kimi-K2.5/q_a_proj", + "nvidia/DeepSeek-R1-0528-FP4-V2/q_a_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/q_a_proj", + "nvidia/Kimi-K2.5-NVFP4/q_a_proj", + "nvidia/Kimi-K2.6-NVFP4/q_a_proj" + ] }, { "type": "gemm", "args": { - "m": 1363968, - "n": 7168, - "k": 4608, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 12288, + "n": 2304, + "k": 7168, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/gate_up_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/gate_up_proj", + "nvidia/Kimi-K2.5-NVFP4/gate_up_proj" + ] }, { "type": "gemm", "args": { - "m": 2516992, - "n": 1536, + "m": 12288, + "n": 2304, "k": 7168, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/gate_up_proj" + ] }, { "type": "gemm", "args": { - "m": 2516992, - "n": 24576, + "m": 12288, + "n": 3072, "k": 1536, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/Kimi-K2.5-MXFP4/q_b_proj", + "amd/MiniMax-M2.5-MXFP4/o_proj", + "moonshotai/Kimi-K2.5/q_b_proj", + "nvidia/DeepSeek-R1-0528-FP4-V2/q_b_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/q_b_proj", + "nvidia/Kimi-K2.5-NVFP4/q_b_proj", + "nvidia/Kimi-K2.6-NVFP4/q_b_proj", + "nvidia/MiniMax-M2.5-NVFP4/o_proj" + ] }, { "type": "gemm", "args": { - "m": 2516992, - "n": 576, - "k": 7168, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 12288, + "n": 3072, + "k": 1536, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/MiniMax-M2.5-NVFP4/experts.down_proj" + ] }, { "type": "gemm", "args": { - "m": 2516992, - "n": 32768, - "k": 512, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 12288, + "n": 3072, + "k": 1536, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 32 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 1, + 32 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "amd/DeepSeek-R1-0528-MXFP4-Preview/q_b_proj", + "amd/DeepSeek-R1-0528-MXFP4-v2/q_b_proj", + "amd/DeepSeek-R1-0528-MXFP4/q_b_proj", + "amd/MiniMax-M2.5-MXFP4/experts.down_proj" + ] }, { "type": "gemm", "args": { - "m": 2516992, - "n": 7168, - "k": 16384, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 12288, + "n": 3072, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "MiniMaxAI/MiniMax-M2.5/experts.down_proj", + "MiniMaxAI/MiniMax-M2.5/o_proj", + "deepseek-ai/DeepSeek-R1-0528/q_b_proj" + ] }, { "type": "gemm", "args": { - "m": 2516992, - "n": 4608, - "k": 7168, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 12288, + "n": 4096, + "k": 512, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "Qwen/Qwen3.5-397B-A17B/o_proj", + "Qwen/Qwen3.5-397B-A17B/shared_expert.down_proj", + "amd/Kimi-K2.5-MXFP4/kv_b_proj", + "amd/Qwen3.5-397B-A17B-MXFP4/o_proj", + "amd/Qwen3.5-397B-A17B-MXFP4/shared_expert.down_proj", + "moonshotai/Kimi-K2.5/kv_b_proj", + "nvidia/DeepSeek-R1-0528-FP4-V2/kv_b_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/kv_b_proj", + "nvidia/Kimi-K2.5-NVFP4/kv_b_proj", + "nvidia/Kimi-K2.6-NVFP4/kv_b_proj", + "nvidia/Qwen3.5-397B-A17B-NVFP4/o_proj", + "nvidia/Qwen3.5-397B-A17B-NVFP4/shared_expert.down_proj" + ] }, { "type": "gemm", "args": { - "m": 2516992, - "n": 7168, - "k": 4608, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 12288, + "n": 4096, + "k": 512, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 32 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 1, + 32 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "amd/DeepSeek-R1-0528-MXFP4-Preview/kv_b_proj", + "amd/DeepSeek-R1-0528-MXFP4-v2/kv_b_proj", + "amd/DeepSeek-R1-0528-MXFP4/kv_b_proj", + "amd/Qwen3.5-397B-A17B-MXFP4-AttnFP8-V2/shared_expert.down_proj" + ] }, { "type": "gemm", "args": { - "m": 614, - "n": 1536, - "k": 7168, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 12288, + "n": 4096, + "k": 512, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "Qwen/Qwen3.5-397B-A17B-FP8/o_proj", + "Qwen/Qwen3.5-397B-A17B-FP8/shared_expert.down_proj", + "deepseek-ai/DeepSeek-R1-0528/kv_b_proj" + ] }, { "type": "gemm", "args": { - "m": 614, - "n": 24576, - "k": 1536, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 12288, + "n": 4608, + "k": 7168, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/gate_up_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/gate_up_proj", + "nvidia/Kimi-K2.5-NVFP4/gate_up_proj" + ] }, { "type": "gemm", "args": { - "m": 614, - "n": 576, + "m": 12288, + "n": 4608, "k": 7168, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 32 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 1, + 32 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "amd/DeepSeek-R1-0528-MXFP4-Preview/gate_up_proj", + "amd/DeepSeek-R1-0528-MXFP4-v2/gate_up_proj", + "amd/DeepSeek-R1-0528-MXFP4/gate_up_proj", + "amd/Kimi-K2.5-MXFP4/gate_up_proj" + ] }, { "type": "gemm", "args": { - "m": 614, - "n": 32768, - "k": 512, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 12288, + "n": 7168, + "k": 2048, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/Kimi-K2.5-MXFP4/o_proj", + "moonshotai/Kimi-K2.5/o_proj", + "nvidia/Kimi-K2.5-NVFP4/o_proj", + "nvidia/Kimi-K2.6-NVFP4/o_proj" + ] }, { "type": "gemm", "args": { - "m": 614, + "m": 12288, "n": 7168, - "k": 16384, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "k": 2048, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/o_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/o_proj" + ] }, { "type": "gemm", "args": { - "m": 614, - "n": 4608, - "k": 7168, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 12288, + "n": 7168, + "k": 2048, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 32 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 1, + 32 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "amd/DeepSeek-R1-0528-MXFP4-Preview/o_proj", + "amd/DeepSeek-R1-0528-MXFP4-v2/o_proj", + "amd/DeepSeek-R1-0528-MXFP4/o_proj" + ] }, { "type": "gemm", "args": { - "m": 614, + "m": 12288, "n": 7168, - "k": 4608, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "k": 2048, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/o_proj", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] }, { "type": "gemm", "args": { - "m": 73728, - "n": 4608, - "k": 7168, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 12288, + "n": 7168, + "k": 2304, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/down_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/down_proj", + "nvidia/Kimi-K2.5-NVFP4/down_proj" + ] }, { "type": "gemm", "args": { - "m": 73728, + "m": 12288, "n": 7168, - "k": 4608, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "k": 2304, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 32 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 1, + 32 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "amd/DeepSeek-R1-0528-MXFP4-Preview/down_proj", + "amd/DeepSeek-R1-0528-MXFP4-v2/down_proj", + "amd/DeepSeek-R1-0528-MXFP4/down_proj", + "amd/Kimi-K2.5-MXFP4/down_proj" + ] }, { "type": "gemm", "args": { - "m": 9, - "n": 4608, - "k": 7168, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 12288, + "n": 7168, + "k": 2304, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/down_proj" + ] }, { "type": "gemm", "args": { - "m": 9, + "m": 12288, "n": 7168, "k": 4608, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 294912, - "n": 3072, - "k": 1536, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/down_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/down_proj", + "nvidia/Kimi-K2.5-NVFP4/down_proj" + ] }, { "type": "gemm", "args": { - "m": 294912, - "n": 4096, - "k": 512, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 12288, + "n": 7168, + "k": 9216, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/down_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/down_proj", + "nvidia/Kimi-K2.5-NVFP4/down_proj" + ] }, { "type": "gemm", "args": { - "m": 294912, + "m": 12288, "n": 7168, - "k": 2048, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "k": 16384, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/o_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/o_proj" + ] }, { "type": "gemm", "args": { - "m": 4405248, - "n": 1536, + "m": 12288, + "n": 9216, "k": 7168, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/gate_up_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/gate_up_proj", + "nvidia/Kimi-K2.5-NVFP4/gate_up_proj" + ] }, { "type": "gemm", "args": { - "m": 4405248, + "m": 12288, "n": 24576, "k": 1536, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 4405248, - "n": 576, - "k": 7168, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/q_b_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/q_b_proj" + ] }, { "type": "gemm", "args": { - "m": 4405248, + "m": 12288, "n": 32768, "k": 512, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/kv_b_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/kv_b_proj" + ] }, { "type": "gemm", "args": { - "m": 4405248, - "n": 7168, - "k": 16384, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 12800, + "n": 576, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/Kimi-K2.5-MXFP4/kv_a_proj_with_mqa", + "moonshotai/Kimi-K2.5/kv_a_proj_with_mqa", + "nvidia/DeepSeek-R1-0528-FP4-V2/kv_a_proj_with_mqa", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/kv_a_proj_with_mqa", + "nvidia/Kimi-K2.5-NVFP4/kv_a_proj_with_mqa", + "nvidia/Kimi-K2.6-NVFP4/kv_a_proj_with_mqa" + ] }, { "type": "gemm", "args": { - "m": 4405248, - "n": 4608, + "m": 12800, + "n": 1536, "k": 7168, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/Kimi-K2.5-MXFP4/q_a_proj", + "moonshotai/Kimi-K2.5/q_a_proj", + "nvidia/DeepSeek-R1-0528-FP4-V2/q_a_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/q_a_proj", + "nvidia/Kimi-K2.5-NVFP4/q_a_proj", + "nvidia/Kimi-K2.6-NVFP4/q_a_proj" + ] }, { "type": "gemm", "args": { - "m": 4405248, + "m": 12800, "n": 7168, - "k": 4608, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "k": 9216, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/down_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/down_proj", + "nvidia/Kimi-K2.5-NVFP4/down_proj" + ] }, { "type": "gemm", "args": { - "m": 537, - "n": 4608, - "k": 7168, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 12800, + "n": 7168, + "k": 16384, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/o_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/o_proj" + ] }, { "type": "gemm", "args": { - "m": 537, - "n": 7168, - "k": 4608, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 12800, + "n": 9216, + "k": 7168, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/gate_up_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/gate_up_proj", + "nvidia/Kimi-K2.5-NVFP4/gate_up_proj" + ] }, { "type": "gemm", "args": { - "m": 2202624, - "n": 1536, + "m": 12800, + "n": 9216, "k": 7168, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 32 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 1, + 32 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "amd/DeepSeek-R1-0528-MXFP4-Preview/gate_up_proj", + "amd/DeepSeek-R1-0528-MXFP4-v2/gate_up_proj", + "amd/DeepSeek-R1-0528-MXFP4/gate_up_proj", + "amd/Kimi-K2.5-MXFP4/gate_up_proj" + ] }, { "type": "gemm", "args": { - "m": 2202624, + "m": 12800, "n": 24576, "k": 1536, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/q_b_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/q_b_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 12800, + "n": 32768, + "k": 512, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/kv_b_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/kv_b_proj" + ] }, { "type": "gemm", "args": { - "m": 2202624, + "m": 12926, "n": 576, "k": 7168, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/Kimi-K2.5-MXFP4/kv_a_proj_with_mqa", + "moonshotai/Kimi-K2.5/kv_a_proj_with_mqa", + "nvidia/DeepSeek-R1-0528-FP4-V2/kv_a_proj_with_mqa", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/kv_a_proj_with_mqa", + "nvidia/Kimi-K2.5-NVFP4/kv_a_proj_with_mqa", + "nvidia/Kimi-K2.6-NVFP4/kv_a_proj_with_mqa" + ] }, { "type": "gemm", "args": { - "m": 2202624, - "n": 32768, - "k": 512, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 12926, + "n": 576, + "k": 7168, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 32 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 1, + 32 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "amd/DeepSeek-R1-0528-MXFP4-Preview/kv_a_proj_with_mqa", + "amd/DeepSeek-R1-0528-MXFP4-v2/kv_a_proj_with_mqa", + "amd/DeepSeek-R1-0528-MXFP4/kv_a_proj_with_mqa" + ] }, { "type": "gemm", "args": { - "m": 2202624, - "n": 7168, - "k": 16384, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 12926, + "n": 1536, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/Kimi-K2.5-MXFP4/q_a_proj", + "moonshotai/Kimi-K2.5/q_a_proj", + "nvidia/DeepSeek-R1-0528-FP4-V2/q_a_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/q_a_proj", + "nvidia/Kimi-K2.5-NVFP4/q_a_proj", + "nvidia/Kimi-K2.6-NVFP4/q_a_proj" + ] }, { "type": "gemm", "args": { - "m": 2202624, - "n": 2304, + "m": 12926, + "n": 1536, "k": 7168, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 32 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 1, + 32 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "amd/DeepSeek-R1-0528-MXFP4-Preview/q_a_proj", + "amd/DeepSeek-R1-0528-MXFP4-v2/q_a_proj", + "amd/DeepSeek-R1-0528-MXFP4/q_a_proj" + ] }, { "type": "gemm", "args": { - "m": 2202624, - "n": 7168, - "k": 2304, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 12926, + "n": 1536, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w3" + ] }, { "type": "gemm", "args": { - "m": 1024, - "n": 4096, - "k": 2880, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 12926, + "n": 7168, + "k": 9216, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/down_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/down_proj", + "nvidia/Kimi-K2.5-NVFP4/down_proj" + ] }, { "type": "gemm", "args": { - "m": 1024, - "n": 512, - "k": 2880, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 12926, + "n": 7168, + "k": 16384, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/o_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/o_proj" + ] }, { "type": "gemm", "args": { - "m": 1024, - "n": 2880, - "k": 4096, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 12926, + "n": 7168, + "k": 16384, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 32 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 1, + 32 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "amd/DeepSeek-R1-0528-MXFP4-Preview/o_proj", + "amd/DeepSeek-R1-0528-MXFP4-v2/o_proj", + "amd/DeepSeek-R1-0528-MXFP4/o_proj" + ] }, { "type": "gemm", "args": { - "m": 1, - "n": 4096, - "k": 2880, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 12926, + "n": 7168, + "k": 16384, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] }, { "type": "gemm", "args": { - "m": 1, - "n": 512, - "k": 2880, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 12926, + "n": 9216, + "k": 7168, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/gate_up_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/gate_up_proj", + "nvidia/Kimi-K2.5-NVFP4/gate_up_proj" + ] }, { "type": "gemm", "args": { - "m": 1, - "n": 2880, - "k": 4096, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 12926, + "n": 9216, + "k": 7168, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 32 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 1, + 32 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "amd/DeepSeek-R1-0528-MXFP4-Preview/gate_up_proj", + "amd/DeepSeek-R1-0528-MXFP4-v2/gate_up_proj", + "amd/DeepSeek-R1-0528-MXFP4/gate_up_proj", + "amd/Kimi-K2.5-MXFP4/gate_up_proj" + ] }, { "type": "gemm", "args": { - "m": 2048, - "n": 4096, - "k": 2880, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 12926, + "n": 24576, + "k": 1536, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/q_b_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/q_b_proj" + ] }, { "type": "gemm", "args": { - "m": 2048, - "n": 512, - "k": 2880, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 12926, + "n": 24576, + "k": 1536, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 32 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 1, + 32 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "amd/DeepSeek-R1-0528-MXFP4-Preview/q_b_proj", + "amd/DeepSeek-R1-0528-MXFP4-v2/q_b_proj", + "amd/DeepSeek-R1-0528-MXFP4/q_b_proj" + ] }, { "type": "gemm", "args": { - "m": 2048, - "n": 2880, - "k": 4096, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 12926, + "n": 32768, + "k": 512, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/kv_b_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/kv_b_proj" + ] }, { "type": "gemm", "args": { - "m": 2, - "n": 4096, - "k": 2880, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 12926, + "n": 32768, + "k": 512, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 32 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 1, + 32 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "amd/DeepSeek-R1-0528-MXFP4-Preview/kv_b_proj", + "amd/DeepSeek-R1-0528-MXFP4-v2/kv_b_proj", + "amd/DeepSeek-R1-0528-MXFP4/kv_b_proj" + ] }, { "type": "gemm", "args": { - "m": 2, - "n": 512, - "k": 2880, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 14080, + "n": 576, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/Kimi-K2.5-MXFP4/kv_a_proj_with_mqa", + "moonshotai/Kimi-K2.5/kv_a_proj_with_mqa", + "nvidia/DeepSeek-R1-0528-FP4-V2/kv_a_proj_with_mqa", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/kv_a_proj_with_mqa", + "nvidia/Kimi-K2.5-NVFP4/kv_a_proj_with_mqa", + "nvidia/Kimi-K2.6-NVFP4/kv_a_proj_with_mqa" + ] }, { "type": "gemm", "args": { - "m": 2, - "n": 2880, - "k": 4096, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 14080, + "n": 1536, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/Kimi-K2.5-MXFP4/q_a_proj", + "moonshotai/Kimi-K2.5/q_a_proj", + "nvidia/DeepSeek-R1-0528-FP4-V2/q_a_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/q_a_proj", + "nvidia/Kimi-K2.5-NVFP4/q_a_proj", + "nvidia/Kimi-K2.6-NVFP4/q_a_proj" + ] }, { "type": "gemm", "args": { - "m": 1024, - "n": 1024, - "k": 2880, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 14080, + "n": 4608, + "k": 7168, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/gate_up_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/gate_up_proj", + "nvidia/Kimi-K2.5-NVFP4/gate_up_proj" + ] }, { "type": "gemm", "args": { - "m": 1024, - "n": 128, - "k": 2880, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 14080, + "n": 4608, + "k": 7168, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 32 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 1, + 32 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "amd/DeepSeek-R1-0528-MXFP4-Preview/gate_up_proj", + "amd/DeepSeek-R1-0528-MXFP4-v2/gate_up_proj", + "amd/DeepSeek-R1-0528-MXFP4/gate_up_proj", + "amd/Kimi-K2.5-MXFP4/gate_up_proj" + ] }, { "type": "gemm", "args": { - "m": 1024, - "n": 2880, - "k": 1024, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 14080, + "n": 7168, + "k": 4608, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/down_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/down_proj", + "nvidia/Kimi-K2.5-NVFP4/down_proj" + ] }, { "type": "gemm", "args": { - "m": 1, - "n": 1024, - "k": 2880, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 14080, + "n": 7168, + "k": 8192, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/Kimi-K2.5-MXFP4/o_proj", + "moonshotai/Kimi-K2.5/o_proj", + "nvidia/Kimi-K2.5-NVFP4/o_proj", + "nvidia/Kimi-K2.6-NVFP4/o_proj" + ] }, { "type": "gemm", "args": { - "m": 1, - "n": 128, - "k": 2880, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 14080, + "n": 12288, + "k": 1536, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/Kimi-K2.5-MXFP4/q_b_proj", + "moonshotai/Kimi-K2.5/q_b_proj", + "nvidia/DeepSeek-R1-0528-FP4-V2/q_b_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/q_b_proj", + "nvidia/Kimi-K2.5-NVFP4/q_b_proj", + "nvidia/Kimi-K2.6-NVFP4/q_b_proj" + ] }, { "type": "gemm", "args": { - "m": 1, - "n": 2880, - "k": 1024, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 14080, + "n": 16384, + "k": 512, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/Kimi-K2.5-MXFP4/kv_b_proj", + "moonshotai/Kimi-K2.5/kv_b_proj", + "nvidia/DeepSeek-R1-0528-FP4-V2/kv_b_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/kv_b_proj", + "nvidia/Kimi-K2.5-NVFP4/kv_b_proj", + "nvidia/Kimi-K2.6-NVFP4/kv_b_proj" + ] }, { "type": "gemm", "args": { - "m": 2048, - "n": 1024, - "k": 2880, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 14784, + "n": 576, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/kv_a_proj_with_mqa" + ] }, { "type": "gemm", "args": { - "m": 2048, - "n": 128, - "k": 2880, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 14784, + "n": 1152, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/gate_up_proj" + ] }, { "type": "gemm", "args": { - "m": 2048, - "n": 2880, - "k": 1024, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 14784, + "n": 1536, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/gate_up_proj", + "deepseek-ai/DeepSeek-R1-0528/q_a_proj", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w3" + ] }, { "type": "gemm", "args": { - "m": 2, - "n": 1024, - "k": 2880, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 14784, + "n": 7168, + "k": 1152, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/down_proj" + ] }, { "type": "gemm", "args": { - "m": 2, - "n": 128, - "k": 2880, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 14784, + "n": 7168, + "k": 16384, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/o_proj", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] }, { "type": "gemm", "args": { - "m": 2, - "n": 2880, - "k": 1024, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 14784, + "n": 24576, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/q_b_proj" + ] }, { "type": "gemm", "args": { - "m": 128, - "n": 1024, - "k": 2880, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 14784, + "n": 32768, + "k": 512, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/kv_b_proj" + ] }, { "type": "gemm", "args": { - "m": 128, - "n": 128, - "k": 2880, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 15360, + "n": 576, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/kv_a_proj_with_mqa" + ] }, { "type": "gemm", "args": { - "m": 128, - "n": 2880, - "k": 1024, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 15360, + "n": 1152, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/gate_up_proj" + ] }, { "type": "gemm", "args": { - "m": 512, - "n": 4096, - "k": 2880, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 15360, + "n": 1536, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/gate_up_proj", + "deepseek-ai/DeepSeek-R1-0528/q_a_proj", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w3" + ] }, { "type": "gemm", "args": { - "m": 512, - "n": 512, - "k": 2880, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 15360, + "n": 2304, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/gate_up_proj" + ] }, { "type": "gemm", "args": { - "m": 512, - "n": 2880, - "k": 4096, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 15360, + "n": 3072, + "k": 1536, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/Kimi-K2.5-MXFP4/q_b_proj", + "amd/MiniMax-M2.5-MXFP4/o_proj", + "moonshotai/Kimi-K2.5/q_b_proj", + "nvidia/DeepSeek-R1-0528-FP4-V2/q_b_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/q_b_proj", + "nvidia/Kimi-K2.5-NVFP4/q_b_proj", + "nvidia/Kimi-K2.6-NVFP4/q_b_proj", + "nvidia/MiniMax-M2.5-NVFP4/o_proj" + ] }, { "type": "gemm", "args": { - "m": 2621440, - "n": 4096, - "k": 2880, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 15360, + "n": 3072, + "k": 1536, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/MiniMax-M2.5-NVFP4/experts.down_proj" + ] }, { "type": "gemm", "args": { - "m": 2621440, - "n": 512, - "k": 2880, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 15360, + "n": 3072, + "k": 1536, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 32 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 1, + 32 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "amd/DeepSeek-R1-0528-MXFP4-Preview/q_b_proj", + "amd/DeepSeek-R1-0528-MXFP4-v2/q_b_proj", + "amd/DeepSeek-R1-0528-MXFP4/q_b_proj", + "amd/MiniMax-M2.5-MXFP4/experts.down_proj" + ] }, { "type": "gemm", "args": { - "m": 2621440, - "n": 2880, - "k": 4096, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 15360, + "n": 4096, + "k": 512, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "Qwen/Qwen3.5-397B-A17B/o_proj", + "Qwen/Qwen3.5-397B-A17B/shared_expert.down_proj", + "amd/Kimi-K2.5-MXFP4/kv_b_proj", + "amd/Qwen3.5-397B-A17B-MXFP4/o_proj", + "amd/Qwen3.5-397B-A17B-MXFP4/shared_expert.down_proj", + "moonshotai/Kimi-K2.5/kv_b_proj", + "nvidia/DeepSeek-R1-0528-FP4-V2/kv_b_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/kv_b_proj", + "nvidia/Kimi-K2.5-NVFP4/kv_b_proj", + "nvidia/Kimi-K2.6-NVFP4/kv_b_proj", + "nvidia/Qwen3.5-397B-A17B-NVFP4/o_proj", + "nvidia/Qwen3.5-397B-A17B-NVFP4/shared_expert.down_proj" + ] }, { "type": "gemm", "args": { - "m": 2560, + "m": 15360, "n": 4096, - "k": 2880, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "k": 512, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 32 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 1, + 32 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "amd/DeepSeek-R1-0528-MXFP4-Preview/kv_b_proj", + "amd/DeepSeek-R1-0528-MXFP4-v2/kv_b_proj", + "amd/DeepSeek-R1-0528-MXFP4/kv_b_proj", + "amd/Qwen3.5-397B-A17B-MXFP4-AttnFP8-V2/shared_expert.down_proj" + ] }, { "type": "gemm", "args": { - "m": 2560, - "n": 512, - "k": 2880, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 15360, + "n": 4608, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/gate_up_proj" + ] }, { "type": "gemm", "args": { - "m": 2560, - "n": 2880, - "k": 4096, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 15360, + "n": 6144, + "k": 1536, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/GLM-5.1-MXFP4/down_proj", + "amd/GLM-5.2-MXFP4/down_proj", + "amd/Kimi-K2.5-MXFP4/q_b_proj", + "amd/MiniMax-M3-MXFP4/down_proj", + "moonshotai/Kimi-K2.5/q_b_proj", + "nvidia/DeepSeek-R1-0528-FP4-V2/q_b_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/q_b_proj", + "nvidia/GLM-5.1-NVFP4/down_proj", + "nvidia/GLM-5.2-NVFP4/down_proj", + "nvidia/Kimi-K2.5-NVFP4/q_b_proj", + "nvidia/Kimi-K2.6-NVFP4/q_b_proj" + ] }, { "type": "gemm", "args": { - "m": 1310720, - "n": 4096, - "k": 2880, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 15360, + "n": 6144, + "k": 1536, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/GLM-5-NVFP4/down_proj" + ] }, { "type": "gemm", "args": { - "m": 1310720, - "n": 512, - "k": 2880, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 15360, + "n": 6144, + "k": 1536, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 32 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 1, + 32 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "amd/DeepSeek-R1-0528-MXFP4-Preview/q_b_proj", + "amd/DeepSeek-R1-0528-MXFP4-v2/q_b_proj", + "amd/DeepSeek-R1-0528-MXFP4/q_b_proj" + ] }, { "type": "gemm", "args": { - "m": 1310720, - "n": 2880, - "k": 4096, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 15360, + "n": 7168, + "k": 1152, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/down_proj" + ] }, { "type": "gemm", "args": { - "m": 1280, - "n": 4096, - "k": 2880, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 15360, + "n": 7168, + "k": 2048, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/Kimi-K2.5-MXFP4/o_proj", + "moonshotai/Kimi-K2.5/o_proj", + "nvidia/Kimi-K2.5-NVFP4/o_proj", + "nvidia/Kimi-K2.6-NVFP4/o_proj" + ] }, { "type": "gemm", "args": { - "m": 1280, - "n": 512, - "k": 2880, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 15360, + "n": 7168, + "k": 2048, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/o_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/o_proj" + ] }, { "type": "gemm", "args": { - "m": 1280, - "n": 2880, - "k": 4096, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 15360, + "n": 7168, + "k": 2048, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 32 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 1, + 32 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "amd/DeepSeek-R1-0528-MXFP4-Preview/o_proj", + "amd/DeepSeek-R1-0528-MXFP4-v2/o_proj", + "amd/DeepSeek-R1-0528-MXFP4/o_proj" + ] }, { "type": "gemm", "args": { - "m": 1572864, - "n": 4096, - "k": 2880, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 15360, + "n": 7168, + "k": 2304, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/down_proj" + ] }, { "type": "gemm", "args": { - "m": 1572864, - "n": 512, - "k": 2880, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 15360, + "n": 7168, + "k": 4096, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/o_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/o_proj" + ] }, { "type": "gemm", "args": { - "m": 1572864, - "n": 2880, + "m": 15360, + "n": 7168, "k": 4096, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 32 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 1, + 32 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "amd/DeepSeek-R1-0528-MXFP4-Preview/o_proj", + "amd/DeepSeek-R1-0528-MXFP4-v2/o_proj", + "amd/DeepSeek-R1-0528-MXFP4/o_proj" + ] }, { "type": "gemm", "args": { - "m": 1536, - "n": 4096, - "k": 2880, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 15360, + "n": 7168, + "k": 4608, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/down_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/down_proj", + "nvidia/Kimi-K2.5-NVFP4/down_proj" + ] }, { "type": "gemm", "args": { - "m": 1536, - "n": 512, - "k": 2880, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 15360, + "n": 7168, + "k": 4608, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 32 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 1, + 32 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "amd/DeepSeek-R1-0528-MXFP4-Preview/down_proj", + "amd/DeepSeek-R1-0528-MXFP4-v2/down_proj", + "amd/DeepSeek-R1-0528-MXFP4/down_proj", + "amd/Kimi-K2.5-MXFP4/down_proj" + ] }, { "type": "gemm", "args": { - "m": 1536, - "n": 2880, - "k": 4096, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 15360, + "n": 7168, + "k": 16384, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/o_proj", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] }, { "type": "gemm", "args": { - "m": 393216, - "n": 4096, - "k": 2880, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 15360, + "n": 8192, + "k": 512, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/Kimi-K2.5-MXFP4/kv_b_proj", + "moonshotai/Kimi-K2.5/kv_b_proj", + "nvidia/DeepSeek-R1-0528-FP4-V2/kv_b_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/kv_b_proj", + "nvidia/Kimi-K2.5-NVFP4/kv_b_proj", + "nvidia/Kimi-K2.6-NVFP4/kv_b_proj" + ] }, { "type": "gemm", "args": { - "m": 393216, - "n": 512, - "k": 2880, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 15360, + "n": 8192, + "k": 512, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 32 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 1, + 32 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "amd/DeepSeek-R1-0528-MXFP4-Preview/kv_b_proj", + "amd/DeepSeek-R1-0528-MXFP4-v2/kv_b_proj", + "amd/DeepSeek-R1-0528-MXFP4/kv_b_proj" + ] }, { "type": "gemm", "args": { - "m": 393216, - "n": 2880, - "k": 4096, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 15360, + "n": 24576, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/q_b_proj" + ] }, { "type": "gemm", "args": { - "m": 384, - "n": 4096, - "k": 2880, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 15360, + "n": 32768, + "k": 512, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/kv_b_proj" + ] }, { "type": "gemm", "args": { - "m": 384, - "n": 512, + "m": 16384, + "n": 64, "k": 2880, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/gpt-oss-120b-w-mxfp4-a-fp8/k_proj", + "amd/gpt-oss-120b-w-mxfp4-a-fp8/v_proj", + "openai/gpt-oss-120b/k_proj", + "openai/gpt-oss-120b/v_proj" + ] }, { "type": "gemm", "args": { - "m": 384, - "n": 2880, - "k": 4096, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 16384, + "n": 128, + "k": 2880, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/gpt-oss-120b-w-mxfp4-a-fp8/k_proj", + "amd/gpt-oss-120b-w-mxfp4-a-fp8/v_proj", + "openai/gpt-oss-120b/k_proj", + "openai/gpt-oss-120b/v_proj" + ] }, { "type": "gemm", "args": { - "m": 3145728, - "n": 4096, - "k": 2880, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 16384, + "n": 128, + "k": 3072, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/MiniMax-M2.5-MXFP4/k_proj", + "amd/MiniMax-M2.5-MXFP4/v_proj", + "nvidia/MiniMax-M2.5-NVFP4/k_proj", + "nvidia/MiniMax-M2.5-NVFP4/v_proj" + ] }, { "type": "gemm", "args": { - "m": 3145728, - "n": 512, - "k": 2880, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 16384, + "n": 128, + "k": 3072, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "MiniMaxAI/MiniMax-M2.5/k_proj", + "MiniMaxAI/MiniMax-M2.5/v_proj" + ] }, { "type": "gemm", "args": { - "m": 3145728, - "n": 2880, + "m": 16384, + "n": 128, "k": 4096, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "Qwen/Qwen3.5-397B-A17B/shared_expert.gate_up_proj", + "amd/Qwen3.5-397B-A17B-MXFP4/shared_expert.gate_up_proj", + "nvidia/Qwen3.5-397B-A17B-NVFP4/shared_expert.gate_up_proj" + ] }, { "type": "gemm", "args": { - "m": 3072, - "n": 4096, - "k": 2880, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 16384, + "n": 128, + "k": 4096, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "Qwen/Qwen3.5-397B-A17B-FP8/shared_expert.gate_up_proj" + ] }, { "type": "gemm", "args": { - "m": 3072, - "n": 512, - "k": 2880, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 16384, + "n": 192, + "k": 3072, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/MiniMax-M2.5-NVFP4/experts.gate_up_proj" + ] }, { "type": "gemm", "args": { - "m": 3072, - "n": 2880, - "k": 4096, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 16384, + "n": 192, + "k": 3072, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 32 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 1, + 32 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "amd/MiniMax-M2.5-MXFP4/experts.gate_up_proj" + ] }, { "type": "gemm", "args": { - "m": 786432, - "n": 4096, - "k": 2880, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 16384, + "n": 192, + "k": 3072, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "MiniMaxAI/MiniMax-M2.5/experts.gate_up_proj" + ] }, { "type": "gemm", "args": { - "m": 786432, - "n": 512, + "m": 16384, + "n": 256, "k": 2880, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/gpt-oss-120b-w-mxfp4-a-fp8/k_proj", + "amd/gpt-oss-120b-w-mxfp4-a-fp8/q_proj", + "amd/gpt-oss-120b-w-mxfp4-a-fp8/v_proj", + "openai/gpt-oss-120b/k_proj", + "openai/gpt-oss-120b/q_proj", + "openai/gpt-oss-120b/v_proj" + ] }, { "type": "gemm", "args": { - "m": 786432, - "n": 2880, - "k": 4096, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 16384, + "n": 256, + "k": 3072, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/MiniMax-M2.5-MXFP4/k_proj", + "amd/MiniMax-M2.5-MXFP4/v_proj", + "nvidia/MiniMax-M2.5-NVFP4/k_proj", + "nvidia/MiniMax-M2.5-NVFP4/v_proj" + ] }, { "type": "gemm", "args": { - "m": 768, - "n": 4096, - "k": 2880, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 16384, + "n": 256, + "k": 3072, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "MiniMaxAI/MiniMax-M2.5/k_proj", + "MiniMaxAI/MiniMax-M2.5/v_proj" + ] }, { "type": "gemm", "args": { - "m": 768, - "n": 512, - "k": 2880, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 16384, + "n": 256, + "k": 4096, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "Qwen/Qwen3.5-397B-A17B/k_proj", + "Qwen/Qwen3.5-397B-A17B/shared_expert.gate_up_proj", + "Qwen/Qwen3.5-397B-A17B/v_proj", + "amd/Qwen3.5-397B-A17B-MXFP4/k_proj", + "amd/Qwen3.5-397B-A17B-MXFP4/shared_expert.gate_up_proj", + "amd/Qwen3.5-397B-A17B-MXFP4/v_proj", + "nvidia/Qwen3.5-397B-A17B-NVFP4/k_proj", + "nvidia/Qwen3.5-397B-A17B-NVFP4/shared_expert.gate_up_proj", + "nvidia/Qwen3.5-397B-A17B-NVFP4/v_proj" + ] }, { "type": "gemm", "args": { - "m": 768, - "n": 2880, + "m": 16384, + "n": 256, "k": 4096, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "Qwen/Qwen3.5-397B-A17B-FP8/k_proj", + "Qwen/Qwen3.5-397B-A17B-FP8/shared_expert.gate_up_proj", + "Qwen/Qwen3.5-397B-A17B-FP8/v_proj" + ] }, { "type": "gemm", "args": { - "m": 1, - "n": 64, - "k": 2880, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 16384, + "n": 384, + "k": 3072, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 32 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 1, + 32 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "amd/MiniMax-M2.5-MXFP4/experts.gate_up_proj" + ] }, { "type": "gemm", "args": { - "m": 1, - "n": 2880, - "k": 512, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 16384, + "n": 384, + "k": 3072, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "MiniMaxAI/MiniMax-M2.5/experts.gate_up_proj", + "MiniMaxAI/MiniMax-M2.5/q_proj" + ] }, { "type": "gemm", "args": { - "m": 4194304, - "n": 4096, + "m": 16384, + "n": 512, "k": 2880, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/gpt-oss-120b-w-mxfp4-a-fp8/k_proj", + "amd/gpt-oss-120b-w-mxfp4-a-fp8/q_proj", + "amd/gpt-oss-120b-w-mxfp4-a-fp8/v_proj", + "openai/gpt-oss-120b/k_proj", + "openai/gpt-oss-120b/q_proj", + "openai/gpt-oss-120b/v_proj" + ] }, { "type": "gemm", "args": { - "m": 4194304, + "m": 16384, "n": 512, - "k": 2880, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "k": 3072, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/MiniMax-M2.5-MXFP4/k_proj", + "amd/MiniMax-M2.5-MXFP4/v_proj", + "nvidia/MiniMax-M2.5-NVFP4/k_proj", + "nvidia/MiniMax-M2.5-NVFP4/v_proj" + ] }, { "type": "gemm", "args": { - "m": 4194304, - "n": 2880, + "m": 16384, + "n": 512, + "k": 3072, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "MiniMaxAI/MiniMax-M2.5/k_proj", + "MiniMaxAI/MiniMax-M2.5/v_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 16384, + "n": 512, "k": 4096, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "Qwen/Qwen3.5-397B-A17B/k_proj", + "Qwen/Qwen3.5-397B-A17B/q_proj", + "Qwen/Qwen3.5-397B-A17B/shared_expert.gate_up_proj", + "Qwen/Qwen3.5-397B-A17B/v_proj", + "amd/Qwen3.5-397B-A17B-MXFP4/k_proj", + "amd/Qwen3.5-397B-A17B-MXFP4/q_proj", + "amd/Qwen3.5-397B-A17B-MXFP4/shared_expert.gate_up_proj", + "amd/Qwen3.5-397B-A17B-MXFP4/v_proj", + "nvidia/Qwen3.5-397B-A17B-NVFP4/k_proj", + "nvidia/Qwen3.5-397B-A17B-NVFP4/q_proj", + "nvidia/Qwen3.5-397B-A17B-NVFP4/shared_expert.gate_up_proj", + "nvidia/Qwen3.5-397B-A17B-NVFP4/v_proj" + ] }, { "type": "gemm", "args": { - "m": 1048576, - "n": 1024, - "k": 2880, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 16384, + "n": 512, + "k": 4096, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "Qwen/Qwen3.5-397B-A17B-FP8/k_proj", + "Qwen/Qwen3.5-397B-A17B-FP8/q_proj", + "Qwen/Qwen3.5-397B-A17B-FP8/shared_expert.gate_up_proj", + "Qwen/Qwen3.5-397B-A17B-FP8/v_proj", + "deepseek-ai/DeepSeek-V4.1-Flash/attn.wo_a" + ] }, { "type": "gemm", "args": { - "m": 1048576, - "n": 128, - "k": 2880, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 16384, + "n": 576, + "k": 6144, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/GLM-5.1-MXFP4/kv_a_proj_with_mqa", + "amd/GLM-5.2-MXFP4/kv_a_proj_with_mqa", + "nvidia/GLM-5-NVFP4/kv_a_proj_with_mqa", + "nvidia/GLM-5.1-NVFP4/kv_a_proj_with_mqa", + "nvidia/GLM-5.2-NVFP4/kv_a_proj_with_mqa" + ] }, { "type": "gemm", "args": { - "m": 1048576, - "n": 2880, - "k": 1024, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 16384, + "n": 576, + "k": 6144, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "zai-org/GLM-5-FP8/kv_a_proj_with_mqa", + "zai-org/GLM-5.1-FP8/kv_a_proj_with_mqa", + "zai-org/GLM-5.2-FP8/kv_a_proj_with_mqa" + ] }, { "type": "gemm", "args": { - "m": 4194304, - "n": 1024, - "k": 2880, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 16384, + "n": 576, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/Kimi-K2.5-MXFP4/kv_a_proj_with_mqa", + "moonshotai/Kimi-K2.5/kv_a_proj_with_mqa", + "nvidia/DeepSeek-R1-0528-FP4-V2/kv_a_proj_with_mqa", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/kv_a_proj_with_mqa", + "nvidia/Kimi-K2.5-NVFP4/kv_a_proj_with_mqa", + "nvidia/Kimi-K2.6-NVFP4/kv_a_proj_with_mqa" + ] }, { "type": "gemm", "args": { - "m": 4194304, - "n": 128, - "k": 2880, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 16384, + "n": 576, + "k": 7168, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 32 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 1, + 32 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "amd/DeepSeek-R1-0528-MXFP4-Preview/kv_a_proj_with_mqa", + "amd/DeepSeek-R1-0528-MXFP4-v2/kv_a_proj_with_mqa", + "amd/DeepSeek-R1-0528-MXFP4/kv_a_proj_with_mqa" + ] }, { "type": "gemm", "args": { - "m": 4194304, - "n": 2880, - "k": 1024, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 16384, + "n": 576, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/kv_a_proj_with_mqa" + ] }, { "type": "gemm", "args": { - "m": 512, - "n": 1024, - "k": 2880, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 16384, + "n": 768, + "k": 3072, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/MiniMax-M2.5-NVFP4/experts.gate_up_proj" + ] }, { "type": "gemm", "args": { - "m": 512, - "n": 128, - "k": 2880, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 16384, + "n": 768, + "k": 3072, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 32 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 1, + 32 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "amd/MiniMax-M2.5-MXFP4/experts.gate_up_proj" + ] }, { "type": "gemm", "args": { - "m": 512, - "n": 2880, - "k": 1024, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 16384, + "n": 768, + "k": 3072, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "MiniMaxAI/MiniMax-M2.5/experts.gate_up_proj", + "MiniMaxAI/MiniMax-M2.5/q_proj" + ] }, { "type": "gemm", "args": { - "m": 3145728, - "n": 2048, + "m": 16384, + "n": 1024, "k": 2880, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/gpt-oss-120b-w-mxfp4-a-fp8/q_proj", + "openai/gpt-oss-120b/q_proj" + ] }, { "type": "gemm", "args": { - "m": 3145728, - "n": 256, - "k": 2880, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 16384, + "n": 1024, + "k": 3072, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/MiniMax-M2.5-MXFP4/k_proj", + "amd/MiniMax-M2.5-MXFP4/qkv_proj", + "amd/MiniMax-M2.5-MXFP4/v_proj", + "nvidia/MiniMax-M2.5-NVFP4/k_proj", + "nvidia/MiniMax-M2.5-NVFP4/qkv_proj", + "nvidia/MiniMax-M2.5-NVFP4/v_proj" + ] }, { "type": "gemm", "args": { - "m": 3145728, - "n": 2880, - "k": 2048, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 16384, + "n": 1024, + "k": 3072, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "MiniMaxAI/MiniMax-M2.5/k_proj", + "MiniMaxAI/MiniMax-M2.5/qkv_proj", + "MiniMaxAI/MiniMax-M2.5/v_proj" + ] }, { "type": "gemm", "args": { - "m": 384, - "n": 2048, - "k": 2880, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 16384, + "n": 1024, + "k": 4096, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "Qwen/Qwen3.5-397B-A17B/q_proj", + "Qwen/Qwen3.5-397B-A17B/shared_expert.gate_up_proj", + "amd/Qwen3.5-397B-A17B-MXFP4/q_proj", + "amd/Qwen3.5-397B-A17B-MXFP4/shared_expert.gate_up_proj", + "nvidia/Qwen3.5-397B-A17B-NVFP4/q_proj", + "nvidia/Qwen3.5-397B-A17B-NVFP4/shared_expert.gate_up_proj" + ] }, { "type": "gemm", "args": { - "m": 384, - "n": 256, - "k": 2880, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 16384, + "n": 1024, + "k": 4096, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "Qwen/Qwen3.5-397B-A17B-FP8/q_proj", + "Qwen/Qwen3.5-397B-A17B-FP8/shared_expert.gate_up_proj", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_a", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_a", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_a" + ] }, { "type": "gemm", "args": { - "m": 384, - "n": 2880, - "k": 2048, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 16384, + "n": 1088, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] }, { "type": "gemm", "args": { - "m": 2048, - "n": 3072, - "k": 1536, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 16384, + "n": 1152, + "k": 7168, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-NVFP4-v2/gate_up_proj" + ] }, { "type": "gemm", "args": { - "m": 2048, - "n": 4096, - "k": 512, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 16384, + "n": 1152, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/gate_up_proj" + ] }, { "type": "gemm", "args": { - "m": 2048, - "n": 7168, - "k": 2048, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 16384, + "n": 1536, + "k": 1536, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/Kimi-K2.5-MXFP4/q_b_proj", + "moonshotai/Kimi-K2.5/q_b_proj", + "nvidia/DeepSeek-R1-0528-FP4-V2/q_b_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/q_b_proj", + "nvidia/Kimi-K2.5-NVFP4/q_b_proj", + "nvidia/Kimi-K2.6-NVFP4/q_b_proj" + ] }, { "type": "gemm", "args": { - "m": 40, - "n": 24576, + "m": 16384, + "n": 1536, "k": 1536, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/q_b_proj" + ] }, { "type": "gemm", "args": { - "m": 40, - "n": 32768, - "k": 512, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 16384, + "n": 1536, + "k": 3072, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/MiniMax-M2.5-NVFP4/experts.gate_up_proj" + ] }, { "type": "gemm", "args": { - "m": 40, - "n": 7168, - "k": 16384, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 16384, + "n": 1536, + "k": 3072, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 32 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 1, + 32 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "amd/MiniMax-M2.5-MXFP4/experts.gate_up_proj" + ] }, { "type": "gemm", "args": { - "m": 1024, - "n": 3072, - "k": 1536, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 16384, + "n": 1536, + "k": 3072, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "MiniMaxAI/MiniMax-M2.5/experts.gate_up_proj", + "MiniMaxAI/MiniMax-M2.5/q_proj" + ] }, { "type": "gemm", "args": { - "m": 1024, - "n": 4096, - "k": 512, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 16384, + "n": 1536, + "k": 6144, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "zai-org/GLM-5-FP8/gate_up_proj", + "zai-org/GLM-5.1-FP8/gate_up_proj", + "zai-org/GLM-5.2-FP8/gate_up_proj" + ] }, { "type": "gemm", "args": { - "m": 1024, - "n": 7168, - "k": 2048, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 16384, + "n": 1536, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/Kimi-K2.5-MXFP4/q_a_proj", + "moonshotai/Kimi-K2.5/q_a_proj", + "nvidia/DeepSeek-R1-0528-FP4-V2/q_a_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/q_a_proj", + "nvidia/Kimi-K2.5-NVFP4/q_a_proj", + "nvidia/Kimi-K2.6-NVFP4/q_a_proj" + ] }, { "type": "gemm", "args": { - "m": 1024, - "n": 12288, - "k": 1536, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 16384, + "n": 1536, + "k": 7168, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 32 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 1, + 32 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "amd/DeepSeek-R1-0528-MXFP4-Preview/q_a_proj", + "amd/DeepSeek-R1-0528-MXFP4-v2/q_a_proj", + "amd/DeepSeek-R1-0528-MXFP4/q_a_proj" + ] }, { "type": "gemm", "args": { - "m": 1024, - "n": 16384, - "k": 512, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 16384, + "n": 1536, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/gate_up_proj", + "deepseek-ai/DeepSeek-R1-0528/q_a_proj", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w3" + ] }, { "type": "gemm", "args": { - "m": 1, - "n": 12288, - "k": 1536, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 16384, + "n": 1536, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w3" + ] }, { "type": "gemm", "args": { - "m": 1, - "n": 16384, + "m": 16384, + "n": 2048, "k": 512, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/Kimi-K2.5-MXFP4/kv_b_proj", + "moonshotai/Kimi-K2.5/kv_b_proj", + "nvidia/DeepSeek-R1-0528-FP4-V2/kv_b_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/kv_b_proj", + "nvidia/Kimi-K2.5-NVFP4/kv_b_proj", + "nvidia/Kimi-K2.6-NVFP4/kv_b_proj" + ] }, { "type": "gemm", "args": { - "m": 49152, - "n": 12288, - "k": 1536, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 16384, + "n": 2048, + "k": 512, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/kv_b_proj" + ] }, { "type": "gemm", "args": { - "m": 49152, - "n": 16384, - "k": 512, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 16384, + "n": 2048, + "k": 2048, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "zai-org/GLM-5-FP8/q_b_proj", + "zai-org/GLM-5.1-FP8/q_b_proj", + "zai-org/GLM-5.2-FP8/q_b_proj" + ] }, { "type": "gemm", "args": { - "m": 49152, - "n": 7168, - "k": 8192, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 16384, + "n": 2048, + "k": 2880, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/gpt-oss-120b-w-mxfp4-a-fp8/q_proj", + "openai/gpt-oss-120b/q_proj" + ] }, { "type": "gemm", "args": { - "m": 48, - "n": 12288, - "k": 1536, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 16384, + "n": 2048, + "k": 4096, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "Qwen/Qwen3.5-397B-A17B/q_proj", + "Qwen/Qwen3.5-397B-A17B/shared_expert.gate_up_proj", + "amd/Qwen3.5-397B-A17B-MXFP4/q_proj", + "amd/Qwen3.5-397B-A17B-MXFP4/shared_expert.gate_up_proj", + "nvidia/Qwen3.5-397B-A17B-NVFP4/q_proj", + "nvidia/Qwen3.5-397B-A17B-NVFP4/shared_expert.gate_up_proj" + ] }, { "type": "gemm", "args": { - "m": 48, - "n": 16384, - "k": 512, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 16384, + "n": 2048, + "k": 4096, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "Qwen/Qwen3.5-397B-A17B-FP8/q_proj", + "Qwen/Qwen3.5-397B-A17B-FP8/shared_expert.gate_up_proj" + ] }, { "type": "gemm", "args": { - "m": 48, - "n": 7168, - "k": 8192, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 16384, + "n": 2048, + "k": 6144, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/GLM-5.1-MXFP4/q_a_proj", + "amd/GLM-5.2-MXFP4/q_a_proj", + "amd/MiniMax-M3-MXFP4/q_proj", + "nvidia/GLM-5-NVFP4/q_a_proj", + "nvidia/GLM-5.1-NVFP4/q_a_proj", + "nvidia/GLM-5.2-NVFP4/q_a_proj" + ] }, { "type": "gemm", "args": { - "m": 92160, - "n": 12288, - "k": 1536, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 16384, + "n": 2048, + "k": 6144, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "zai-org/GLM-5-FP8/q_a_proj", + "zai-org/GLM-5.1-FP8/q_a_proj", + "zai-org/GLM-5.2-FP8/q_a_proj" + ] }, { "type": "gemm", "args": { - "m": 92160, - "n": 16384, - "k": 512, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 16384, + "n": 2304, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/gate_up_proj" + ] }, { "type": "gemm", "args": { - "m": 92160, - "n": 7168, - "k": 8192, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 16384, + "n": 2880, + "k": 512, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/gpt-oss-120b-w-mxfp4-a-fp8/o_proj", + "openai/gpt-oss-120b/o_proj" + ] }, { "type": "gemm", "args": { - "m": 92160, - "n": 4608, - "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 16384, + "n": 2880, + "k": 1024, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/gpt-oss-120b-w-mxfp4-a-fp8/o_proj", + "openai/gpt-oss-120b/o_proj" + ] }, { "type": "gemm", "args": { - "m": 92160, - "n": 7168, - "k": 4608, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 16384, + "n": 2880, + "k": 2048, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/gpt-oss-120b-w-mxfp4-a-fp8/o_proj", + "openai/gpt-oss-120b/o_proj" + ] }, { "type": "gemm", "args": { - "m": 90, - "n": 12288, - "k": 1536, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 16384, + "n": 2880, + "k": 4096, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/gpt-oss-120b-w-mxfp4-a-fp8/o_proj", + "openai/gpt-oss-120b/o_proj" + ] }, { "type": "gemm", "args": { - "m": 90, - "n": 16384, - "k": 512, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 16384, + "n": 3072, + "k": 192, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 32 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 1, + 32 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "amd/MiniMax-M2.5-MXFP4/experts.down_proj" + ] }, { "type": "gemm", "args": { - "m": 90, - "n": 7168, - "k": 8192, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 16384, + "n": 3072, + "k": 192, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "MiniMaxAI/MiniMax-M2.5/experts.down_proj" + ] }, { "type": "gemm", "args": { - "m": 171008, - "n": 1536, - "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 16384, + "n": 3072, + "k": 384, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/MiniMax-M2.5-NVFP4/experts.down_proj" + ] }, { "type": "gemm", "args": { - "m": 171008, - "n": 12288, - "k": 1536, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 16384, + "n": 3072, + "k": 384, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 32 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 1, + 32 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "amd/MiniMax-M2.5-MXFP4/experts.down_proj" + ] }, { "type": "gemm", "args": { - "m": 171008, - "n": 576, - "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 16384, + "n": 3072, + "k": 384, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "MiniMaxAI/MiniMax-M2.5/experts.down_proj", + "MiniMaxAI/MiniMax-M2.5/o_proj" + ] }, { "type": "gemm", "args": { - "m": 171008, - "n": 16384, - "k": 512, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 16384, + "n": 3072, + "k": 768, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/MiniMax-M2.5-MXFP4/o_proj", + "nvidia/MiniMax-M2.5-NVFP4/o_proj" + ] }, { "type": "gemm", "args": { - "m": 171008, - "n": 7168, - "k": 8192, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 16384, + "n": 3072, + "k": 768, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/MiniMax-M2.5-NVFP4/experts.down_proj" + ] }, { "type": "gemm", "args": { - "m": 171008, - "n": 4608, - "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 16384, + "n": 3072, + "k": 768, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 32 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 1, + 32 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "amd/MiniMax-M2.5-MXFP4/experts.down_proj" + ] }, { "type": "gemm", "args": { - "m": 171008, - "n": 7168, - "k": 4608, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 16384, + "n": 3072, + "k": 768, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "MiniMaxAI/MiniMax-M2.5/experts.down_proj", + "MiniMaxAI/MiniMax-M2.5/o_proj" + ] }, { "type": "gemm", "args": { - "m": 167, - "n": 1536, - "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 16384, + "n": 3072, + "k": 1536, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/Kimi-K2.5-MXFP4/q_b_proj", + "amd/MiniMax-M2.5-MXFP4/o_proj", + "moonshotai/Kimi-K2.5/q_b_proj", + "nvidia/DeepSeek-R1-0528-FP4-V2/q_b_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/q_b_proj", + "nvidia/Kimi-K2.5-NVFP4/q_b_proj", + "nvidia/Kimi-K2.6-NVFP4/q_b_proj", + "nvidia/MiniMax-M2.5-NVFP4/o_proj" + ] }, { "type": "gemm", "args": { - "m": 167, - "n": 12288, + "m": 16384, + "n": 3072, "k": 1536, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/MiniMax-M2.5-NVFP4/experts.down_proj" + ] }, { "type": "gemm", "args": { - "m": 167, - "n": 576, - "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 16384, + "n": 3072, + "k": 1536, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 32 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 1, + 32 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "amd/DeepSeek-R1-0528-MXFP4-Preview/q_b_proj", + "amd/DeepSeek-R1-0528-MXFP4-v2/q_b_proj", + "amd/DeepSeek-R1-0528-MXFP4/q_b_proj", + "amd/MiniMax-M2.5-MXFP4/experts.down_proj" + ] }, { "type": "gemm", "args": { - "m": 167, - "n": 16384, - "k": 512, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 16384, + "n": 3072, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "MiniMaxAI/MiniMax-M2.5/experts.down_proj", + "MiniMaxAI/MiniMax-M2.5/o_proj", + "deepseek-ai/DeepSeek-R1-0528/q_b_proj" + ] }, { "type": "gemm", "args": { - "m": 167, - "n": 7168, - "k": 8192, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 16384, + "n": 3072, + "k": 3072, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/MiniMax-M2.5-MXFP4/o_proj", + "amd/MiniMax-M2.5-MXFP4/q_proj", + "nvidia/MiniMax-M2.5-NVFP4/o_proj", + "nvidia/MiniMax-M2.5-NVFP4/q_proj" + ] }, { "type": "gemm", "args": { - "m": 167, - "n": 4608, - "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 16384, + "n": 3072, + "k": 3072, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/MiniMax-M2.5-NVFP4/experts.gate_up_proj" + ] }, { "type": "gemm", "args": { - "m": 167, - "n": 7168, - "k": 4608, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 16384, + "n": 3072, + "k": 3072, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 32 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 1, + 32 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "amd/MiniMax-M2.5-MXFP4/experts.gate_up_proj" + ] }, { "type": "gemm", "args": { - "m": 196608, - "n": 1536, - "k": 1536, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 16384, + "n": 3072, + "k": 3072, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "MiniMaxAI/MiniMax-M2.5/experts.gate_up_proj", + "MiniMaxAI/MiniMax-M2.5/o_proj", + "MiniMaxAI/MiniMax-M2.5/q_proj" + ] }, { "type": "gemm", "args": { - "m": 196608, - "n": 2048, - "k": 512, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 16384, + "n": 3072, + "k": 6144, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/GLM-5.1-MXFP4/gate_up_proj", + "amd/GLM-5.2-MXFP4/gate_up_proj", + "amd/MiniMax-M2.5-MXFP4/o_proj", + "amd/MiniMax-M3-MXFP4/gate_up_proj", + "nvidia/GLM-5.1-NVFP4/gate_up_proj", + "nvidia/GLM-5.2-NVFP4/gate_up_proj", + "nvidia/MiniMax-M2.5-NVFP4/o_proj" + ] }, { "type": "gemm", "args": { - "m": 196608, - "n": 7168, - "k": 1024, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 16384, + "n": 3072, + "k": 6144, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "MiniMaxAI/MiniMax-M2.5/o_proj", + "zai-org/GLM-5-FP8/gate_up_proj", + "zai-org/GLM-5.1-FP8/gate_up_proj", + "zai-org/GLM-5.2-FP8/gate_up_proj" + ] }, { "type": "gemm", "args": { - "m": 192, - "n": 1536, - "k": 1536, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 16384, + "n": 3584, + "k": 512, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "zai-org/GLM-5-FP8/kv_b_proj", + "zai-org/GLM-5.1-FP8/kv_b_proj", + "zai-org/GLM-5.2-FP8/kv_b_proj" + ] }, { "type": "gemm", "args": { - "m": 192, - "n": 2048, - "k": 512, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 16384, + "n": 4096, + "k": 128, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "Qwen/Qwen3.5-397B-A17B/shared_expert.down_proj", + "amd/Qwen3.5-397B-A17B-MXFP4/shared_expert.down_proj", + "nvidia/Qwen3.5-397B-A17B-NVFP4/shared_expert.down_proj" + ] }, { "type": "gemm", "args": { - "m": 192, - "n": 7168, - "k": 1024, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 16384, + "n": 4096, + "k": 128, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "Qwen/Qwen3.5-397B-A17B-FP8/shared_expert.down_proj" + ] }, { "type": "gemm", "args": { - "m": 368640, - "n": 1536, - "k": 1536, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 16384, + "n": 4096, + "k": 256, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "Qwen/Qwen3.5-397B-A17B/shared_expert.down_proj", + "amd/Qwen3.5-397B-A17B-MXFP4/shared_expert.down_proj", + "nvidia/Qwen3.5-397B-A17B-NVFP4/shared_expert.down_proj" + ] }, { "type": "gemm", "args": { - "m": 368640, - "n": 2048, - "k": 512, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 16384, + "n": 4096, + "k": 256, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "Qwen/Qwen3.5-397B-A17B-FP8/shared_expert.down_proj" + ] }, { "type": "gemm", "args": { - "m": 368640, - "n": 7168, - "k": 1024, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 16384, + "n": 4096, + "k": 512, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "Qwen/Qwen3.5-397B-A17B/o_proj", + "Qwen/Qwen3.5-397B-A17B/shared_expert.down_proj", + "amd/Kimi-K2.5-MXFP4/kv_b_proj", + "amd/Qwen3.5-397B-A17B-MXFP4/o_proj", + "amd/Qwen3.5-397B-A17B-MXFP4/shared_expert.down_proj", + "moonshotai/Kimi-K2.5/kv_b_proj", + "nvidia/DeepSeek-R1-0528-FP4-V2/kv_b_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/kv_b_proj", + "nvidia/Kimi-K2.5-NVFP4/kv_b_proj", + "nvidia/Kimi-K2.6-NVFP4/kv_b_proj", + "nvidia/Qwen3.5-397B-A17B-NVFP4/o_proj", + "nvidia/Qwen3.5-397B-A17B-NVFP4/shared_expert.down_proj" + ] }, { "type": "gemm", "args": { - "m": 368640, - "n": 2304, - "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 16384, + "n": 4096, + "k": 512, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 32 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 1, + 32 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "amd/DeepSeek-R1-0528-MXFP4-Preview/kv_b_proj", + "amd/DeepSeek-R1-0528-MXFP4-v2/kv_b_proj", + "amd/DeepSeek-R1-0528-MXFP4/kv_b_proj", + "amd/Qwen3.5-397B-A17B-MXFP4-AttnFP8-V2/shared_expert.down_proj" + ] }, { "type": "gemm", "args": { - "m": 368640, - "n": 7168, - "k": 2304, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 16384, + "n": 4096, + "k": 512, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "Qwen/Qwen3.5-397B-A17B-FP8/o_proj", + "Qwen/Qwen3.5-397B-A17B-FP8/shared_expert.down_proj", + "deepseek-ai/DeepSeek-R1-0528/kv_b_proj" + ] }, { "type": "gemm", "args": { - "m": 360, - "n": 1536, - "k": 1536, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 16384, + "n": 4096, + "k": 1024, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "Qwen/Qwen3.5-397B-A17B/o_proj", + "Qwen/Qwen3.5-397B-A17B/shared_expert.down_proj", + "amd/Qwen3.5-397B-A17B-MXFP4/o_proj", + "amd/Qwen3.5-397B-A17B-MXFP4/shared_expert.down_proj", + "nvidia/Qwen3.5-397B-A17B-NVFP4/o_proj", + "nvidia/Qwen3.5-397B-A17B-NVFP4/shared_expert.down_proj" + ] }, { "type": "gemm", "args": { - "m": 360, - "n": 2048, - "k": 512, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 16384, + "n": 4096, + "k": 1024, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "Qwen/Qwen3.5-397B-A17B-FP8/o_proj", + "Qwen/Qwen3.5-397B-A17B-FP8/shared_expert.down_proj" + ] }, { "type": "gemm", "args": { - "m": 360, - "n": 7168, - "k": 1024, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 16384, + "n": 4096, + "k": 2048, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "Qwen/Qwen3.5-397B-A17B/o_proj", + "amd/GLM-5.1-MXFP4/q_b_proj", + "amd/GLM-5.2-MXFP4/q_b_proj", + "amd/Qwen3.5-397B-A17B-MXFP4/o_proj", + "nvidia/GLM-5-NVFP4/q_b_proj", + "nvidia/GLM-5.1-NVFP4/q_b_proj", + "nvidia/GLM-5.2-NVFP4/q_b_proj", + "nvidia/Qwen3.5-397B-A17B-NVFP4/o_proj" + ] }, { "type": "gemm", "args": { - "m": 684032, - "n": 1536, - "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 16384, + "n": 4096, + "k": 2048, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "Qwen/Qwen3.5-397B-A17B-FP8/o_proj", + "zai-org/GLM-5-FP8/q_b_proj", + "zai-org/GLM-5.1-FP8/q_b_proj", + "zai-org/GLM-5.2-FP8/q_b_proj" + ] }, { "type": "gemm", "args": { - "m": 684032, - "n": 1536, - "k": 1536, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 16384, + "n": 4096, + "k": 2880, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/gpt-oss-120b-w-mxfp4-a-fp8/q_proj", + "openai/gpt-oss-120b/q_proj" + ] }, { "type": "gemm", "args": { - "m": 684032, - "n": 576, - "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 16384, + "n": 4096, + "k": 4096, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "Qwen/Qwen3.5-397B-A17B/o_proj", + "Qwen/Qwen3.5-397B-A17B/q_proj", + "amd/Qwen3.5-397B-A17B-MXFP4/o_proj", + "amd/Qwen3.5-397B-A17B-MXFP4/q_proj", + "nvidia/Qwen3.5-397B-A17B-NVFP4/o_proj", + "nvidia/Qwen3.5-397B-A17B-NVFP4/q_proj" + ] }, { "type": "gemm", "args": { - "m": 684032, - "n": 2048, - "k": 512, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 16384, + "n": 4096, + "k": 4096, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "Qwen/Qwen3.5-397B-A17B-FP8/o_proj", + "Qwen/Qwen3.5-397B-A17B-FP8/q_proj" + ] }, { "type": "gemm", "args": { - "m": 684032, - "n": 7168, - "k": 1024, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 16384, + "n": 4608, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "moonshotai/Kimi-K2.5/gate_up_proj", + "nvidia/Kimi-K2.6-NVFP4/gate_up_proj" + ] }, { "type": "gemm", "args": { - "m": 684032, - "n": 2304, + "m": 16384, + "n": 4608, "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/gate_up_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/gate_up_proj", + "nvidia/Kimi-K2.5-NVFP4/gate_up_proj" + ] }, { "type": "gemm", "args": { - "m": 684032, - "n": 7168, - "k": 2304, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 16384, + "n": 4608, + "k": 7168, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 32 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 1, + 32 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "amd/DeepSeek-R1-0528-MXFP4-Preview/gate_up_proj", + "amd/DeepSeek-R1-0528-MXFP4-v2/gate_up_proj", + "amd/DeepSeek-R1-0528-MXFP4/gate_up_proj", + "amd/Kimi-K2.5-MXFP4/gate_up_proj" + ] }, { "type": "gemm", "args": { - "m": 668, - "n": 1536, + "m": 16384, + "n": 4608, "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/gate_up_proj" + ] }, { "type": "gemm", "args": { - "m": 668, - "n": 1536, + "m": 16384, + "n": 6144, "k": 1536, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/GLM-5.1-MXFP4/down_proj", + "amd/GLM-5.2-MXFP4/down_proj", + "amd/Kimi-K2.5-MXFP4/q_b_proj", + "amd/MiniMax-M3-MXFP4/down_proj", + "moonshotai/Kimi-K2.5/q_b_proj", + "nvidia/DeepSeek-R1-0528-FP4-V2/q_b_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/q_b_proj", + "nvidia/GLM-5.1-NVFP4/down_proj", + "nvidia/GLM-5.2-NVFP4/down_proj", + "nvidia/Kimi-K2.5-NVFP4/q_b_proj", + "nvidia/Kimi-K2.6-NVFP4/q_b_proj" + ] }, { "type": "gemm", "args": { - "m": 668, - "n": 576, - "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 16384, + "n": 6144, + "k": 1536, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 32 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 1, + 32 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "amd/DeepSeek-R1-0528-MXFP4-Preview/q_b_proj", + "amd/DeepSeek-R1-0528-MXFP4-v2/q_b_proj", + "amd/DeepSeek-R1-0528-MXFP4/q_b_proj" + ] }, { "type": "gemm", "args": { - "m": 668, - "n": 2048, - "k": 512, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 16384, + "n": 6144, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/q_b_proj", + "zai-org/GLM-5-FP8/down_proj", + "zai-org/GLM-5.1-FP8/down_proj", + "zai-org/GLM-5.2-FP8/down_proj" + ] }, { "type": "gemm", "args": { - "m": 668, - "n": 7168, - "k": 1024, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 16384, + "n": 6144, + "k": 2048, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "zai-org/GLM-5-FP8/o_proj", + "zai-org/GLM-5.1-FP8/o_proj", + "zai-org/GLM-5.2-FP8/o_proj" + ] }, { "type": "gemm", "args": { - "m": 668, - "n": 2304, - "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 16384, + "n": 6144, + "k": 3072, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/GLM-5.1-MXFP4/down_proj", + "amd/GLM-5.2-MXFP4/down_proj", + "amd/MiniMax-M2.5-MXFP4/q_proj", + "amd/MiniMax-M3-MXFP4/down_proj", + "nvidia/GLM-5.1-NVFP4/down_proj", + "nvidia/GLM-5.2-NVFP4/down_proj", + "nvidia/MiniMax-M2.5-NVFP4/q_proj" + ] }, { "type": "gemm", "args": { - "m": 668, - "n": 7168, - "k": 2304, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 16384, + "n": 6144, + "k": 3072, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/GLM-5-NVFP4/down_proj" + ] }, { "type": "gemm", "args": { - "m": 1280, - "n": 12288, - "k": 1536, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 16384, + "n": 6144, + "k": 3072, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "MiniMaxAI/MiniMax-M2.5/q_proj", + "zai-org/GLM-5-FP8/down_proj", + "zai-org/GLM-5.1-FP8/down_proj", + "zai-org/GLM-5.2-FP8/down_proj" + ] }, { "type": "gemm", "args": { - "m": 1280, - "n": 16384, - "k": 512, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 16384, + "n": 6144, + "k": 4096, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/GLM-5.1-MXFP4/o_proj", + "amd/GLM-5.2-MXFP4/o_proj", + "amd/MiniMax-M3-MXFP4/o_proj", + "nvidia/GLM-5-NVFP4/o_proj", + "nvidia/GLM-5.1-NVFP4/o_proj", + "nvidia/GLM-5.2-NVFP4/o_proj" + ] }, { "type": "gemm", "args": { - "m": 1280, - "n": 7168, - "k": 8192, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 16384, + "n": 6144, + "k": 4096, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "zai-org/GLM-5-FP8/o_proj", + "zai-org/GLM-5.1-FP8/o_proj", + "zai-org/GLM-5.2-FP8/o_proj" + ] }, { "type": "gemm", "args": { - "m": 3840, - "n": 12288, - "k": 1536, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 16384, + "n": 6144, + "k": 6144, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/GLM-5.1-MXFP4/down_proj", + "amd/GLM-5.1-MXFP4/gate_up_proj", + "amd/GLM-5.2-MXFP4/down_proj", + "amd/GLM-5.2-MXFP4/gate_up_proj", + "amd/MiniMax-M3-MXFP4/gate_up_proj", + "nvidia/GLM-5.1-NVFP4/down_proj", + "nvidia/GLM-5.1-NVFP4/gate_up_proj", + "nvidia/GLM-5.2-NVFP4/down_proj", + "nvidia/GLM-5.2-NVFP4/gate_up_proj" + ] }, { "type": "gemm", "args": { - "m": 3840, - "n": 16384, - "k": 512, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 16384, + "n": 6144, + "k": 6144, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/GLM-5-NVFP4/down_proj", + "nvidia/GLM-5-NVFP4/gate_up_proj" + ] }, { "type": "gemm", "args": { - "m": 3840, - "n": 7168, - "k": 8192, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 16384, + "n": 6144, + "k": 6144, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "zai-org/GLM-5-FP8/down_proj", + "zai-org/GLM-5-FP8/gate_up_proj", + "zai-org/GLM-5.1-FP8/down_proj", + "zai-org/GLM-5.1-FP8/gate_up_proj", + "zai-org/GLM-5.2-FP8/down_proj", + "zai-org/GLM-5.2-FP8/gate_up_proj" + ] }, { "type": "gemm", "args": { - "m": 3, - "n": 12288, - "k": 1536, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 16384, + "n": 6144, + "k": 8192, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/GLM-5.1-MXFP4/o_proj", + "amd/GLM-5.2-MXFP4/o_proj", + "amd/MiniMax-M3-MXFP4/o_proj", + "nvidia/GLM-5-NVFP4/o_proj", + "nvidia/GLM-5.1-NVFP4/o_proj", + "nvidia/GLM-5.2-NVFP4/o_proj" + ] }, { "type": "gemm", "args": { - "m": 3, - "n": 16384, + "m": 16384, + "n": 7168, "k": 512, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/GLM-5.1-MXFP4/kv_b_proj", + "amd/GLM-5.2-MXFP4/kv_b_proj", + "amd/Kimi-K2.5-MXFP4/o_proj", + "moonshotai/Kimi-K2.5/o_proj", + "nvidia/GLM-5-NVFP4/kv_b_proj", + "nvidia/GLM-5.1-NVFP4/kv_b_proj", + "nvidia/GLM-5.2-NVFP4/kv_b_proj", + "nvidia/Kimi-K2.5-NVFP4/o_proj", + "nvidia/Kimi-K2.6-NVFP4/o_proj" + ] }, { "type": "gemm", "args": { - "m": 3, + "m": 16384, "n": 7168, - "k": 8192, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "k": 512, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "zai-org/GLM-5-FP8/kv_b_proj", + "zai-org/GLM-5.1-FP8/kv_b_proj", + "zai-org/GLM-5.2-FP8/kv_b_proj" + ] }, { "type": "gemm", "args": { - "m": 7680, - "n": 12288, - "k": 1536, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 16384, + "n": 7168, + "k": 576, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-NVFP4-v2/down_proj" + ] }, { "type": "gemm", "args": { - "m": 7680, - "n": 16384, - "k": 512, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 16384, + "n": 7168, + "k": 1024, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/Kimi-K2.5-MXFP4/o_proj", + "moonshotai/Kimi-K2.5/o_proj", + "nvidia/Kimi-K2.5-NVFP4/o_proj", + "nvidia/Kimi-K2.6-NVFP4/o_proj" + ] }, { "type": "gemm", "args": { - "m": 7680, + "m": 16384, "n": 7168, - "k": 8192, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "k": 1024, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/o_proj", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] }, { "type": "gemm", "args": { - "m": 7680, - "n": 4608, - "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 16384, + "n": 7168, + "k": 1152, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/down_proj" + ] }, { "type": "gemm", "args": { - "m": 7680, + "m": 16384, "n": 7168, - "k": 4608, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "k": 2048, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/Kimi-K2.5-MXFP4/o_proj", + "moonshotai/Kimi-K2.5/o_proj", + "nvidia/Kimi-K2.5-NVFP4/o_proj", + "nvidia/Kimi-K2.6-NVFP4/o_proj" + ] }, { "type": "gemm", "args": { - "m": 7, - "n": 12288, - "k": 1536, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 16384, + "n": 7168, + "k": 2048, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/o_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/o_proj" + ] }, { "type": "gemm", "args": { - "m": 7, - "n": 16384, - "k": 512, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 16384, + "n": 7168, + "k": 2048, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 32 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 1, + 32 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "amd/DeepSeek-R1-0528-MXFP4-Preview/o_proj", + "amd/DeepSeek-R1-0528-MXFP4-v2/o_proj", + "amd/DeepSeek-R1-0528-MXFP4/o_proj" + ] }, { "type": "gemm", "args": { - "m": 7, + "m": 16384, "n": 7168, - "k": 8192, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "k": 2048, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/o_proj", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] }, { "type": "gemm", "args": { - "m": 14080, - "n": 1536, - "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 16384, + "n": 7168, + "k": 2304, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "moonshotai/Kimi-K2.5/down_proj", + "nvidia/Kimi-K2.6-NVFP4/down_proj" + ] }, { "type": "gemm", "args": { - "m": 14080, - "n": 12288, - "k": 1536, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 16384, + "n": 7168, + "k": 2304, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/down_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/down_proj", + "nvidia/Kimi-K2.5-NVFP4/down_proj" + ] }, { "type": "gemm", "args": { - "m": 14080, - "n": 576, - "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 16384, + "n": 7168, + "k": 2304, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 32 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 1, + 32 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "amd/DeepSeek-R1-0528-MXFP4-Preview/down_proj", + "amd/DeepSeek-R1-0528-MXFP4-v2/down_proj", + "amd/DeepSeek-R1-0528-MXFP4/down_proj", + "amd/Kimi-K2.5-MXFP4/down_proj" + ] }, { "type": "gemm", "args": { - "m": 14080, - "n": 16384, - "k": 512, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 16384, + "n": 7168, + "k": 2304, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/down_proj" + ] }, { "type": "gemm", "args": { - "m": 14080, + "m": 16384, "n": 7168, - "k": 8192, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "k": 4096, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/o_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/o_proj" + ] }, { "type": "gemm", "args": { - "m": 14080, - "n": 4608, - "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 16384, + "n": 7168, + "k": 4096, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 32 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 1, + 32 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "amd/DeepSeek-R1-0528-MXFP4-Preview/o_proj", + "amd/DeepSeek-R1-0528-MXFP4-v2/o_proj", + "amd/DeepSeek-R1-0528-MXFP4/o_proj" + ] }, { "type": "gemm", "args": { - "m": 14080, + "m": 16384, "n": 7168, - "k": 4608, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "k": 4096, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/o_proj", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] }, { "type": "gemm", "args": { - "m": 13, - "n": 1536, - "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 16384, + "n": 7168, + "k": 4608, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "moonshotai/Kimi-K2.5/down_proj", + "nvidia/Kimi-K2.6-NVFP4/down_proj" + ] }, { "type": "gemm", "args": { - "m": 13, - "n": 12288, - "k": 1536, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 16384, + "n": 7168, + "k": 4608, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/down_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/down_proj", + "nvidia/Kimi-K2.5-NVFP4/down_proj" + ] }, { "type": "gemm", "args": { - "m": 13, - "n": 576, - "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 16384, + "n": 7168, + "k": 4608, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 32 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 1, + 32 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "amd/DeepSeek-R1-0528-MXFP4-Preview/down_proj", + "amd/DeepSeek-R1-0528-MXFP4-v2/down_proj", + "amd/DeepSeek-R1-0528-MXFP4/down_proj", + "amd/Kimi-K2.5-MXFP4/down_proj" + ] }, { "type": "gemm", "args": { - "m": 13, - "n": 16384, - "k": 512, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 16384, + "n": 7168, + "k": 4608, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/down_proj" + ] }, { "type": "gemm", "args": { - "m": 13, + "m": 16384, "n": 7168, "k": 8192, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] }, { "type": "gemm", "args": { - "m": 13, - "n": 4608, - "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 16384, + "n": 7168, + "k": 9216, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/down_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/down_proj", + "nvidia/Kimi-K2.5-NVFP4/down_proj" + ] }, { "type": "gemm", "args": { - "m": 13, + "m": 16384, "n": 7168, - "k": 4608, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "k": 16384, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/o_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/o_proj" + ] }, { "type": "gemm", "args": { - "m": 30720, - "n": 3072, - "k": 1536, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 16384, + "n": 7168, + "k": 16384, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] }, { "type": "gemm", "args": { - "m": 30720, - "n": 4096, - "k": 512, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 16384, + "n": 7168, + "k": 65536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] }, { "type": "gemm", "args": { - "m": 30720, - "n": 7168, - "k": 2048, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 16384, + "n": 8192, + "k": 512, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/Kimi-K2.5-MXFP4/kv_b_proj", + "moonshotai/Kimi-K2.5/kv_b_proj", + "nvidia/DeepSeek-R1-0528-FP4-V2/kv_b_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/kv_b_proj", + "nvidia/Kimi-K2.5-NVFP4/kv_b_proj", + "nvidia/Kimi-K2.6-NVFP4/kv_b_proj" + ] }, { "type": "gemm", "args": { - "m": 56320, - "n": 3072, - "k": 1536, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 16384, + "n": 8192, + "k": 512, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 32 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 1, + 32 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "amd/DeepSeek-R1-0528-MXFP4-Preview/kv_b_proj", + "amd/DeepSeek-R1-0528-MXFP4-v2/kv_b_proj", + "amd/DeepSeek-R1-0528-MXFP4/kv_b_proj" + ] }, { "type": "gemm", "args": { - "m": 56320, - "n": 4096, + "m": 16384, + "n": 8192, "k": 512, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/kv_b_proj" + ] }, { "type": "gemm", "args": { - "m": 56320, - "n": 7168, - "k": 2048, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 16384, + "n": 8192, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_b" + ] }, { "type": "gemm", "args": { - "m": 56320, - "n": 4608, - "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 16384, + "n": 8192, + "k": 2048, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/GLM-5.1-MXFP4/q_b_proj", + "amd/GLM-5.2-MXFP4/q_b_proj", + "nvidia/GLM-5-NVFP4/q_b_proj", + "nvidia/GLM-5.1-NVFP4/q_b_proj", + "nvidia/GLM-5.2-NVFP4/q_b_proj" + ] }, { "type": "gemm", "args": { - "m": 56320, - "n": 7168, - "k": 4608, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 16384, + "n": 9216, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "moonshotai/Kimi-K2.5/gate_up_proj", + "nvidia/Kimi-K2.6-NVFP4/gate_up_proj" + ] }, { "type": "gemm", "args": { - "m": 55, - "n": 3072, - "k": 1536, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 16384, + "n": 9216, + "k": 7168, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/gate_up_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/gate_up_proj", + "nvidia/Kimi-K2.5-NVFP4/gate_up_proj" + ] }, { "type": "gemm", "args": { - "m": 55, - "n": 4096, - "k": 512, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 16384, + "n": 9216, + "k": 7168, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 32 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 1, + 32 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "amd/DeepSeek-R1-0528-MXFP4-Preview/gate_up_proj", + "amd/DeepSeek-R1-0528-MXFP4-v2/gate_up_proj", + "amd/DeepSeek-R1-0528-MXFP4/gate_up_proj", + "amd/Kimi-K2.5-MXFP4/gate_up_proj" + ] }, { "type": "gemm", "args": { - "m": 55, - "n": 7168, - "k": 2048, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 16384, + "n": 9216, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/gate_up_proj" + ] }, { "type": "gemm", "args": { - "m": 55, - "n": 4608, - "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 16384, + "n": 12288, + "k": 1536, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/Kimi-K2.5-MXFP4/q_b_proj", + "moonshotai/Kimi-K2.5/q_b_proj", + "nvidia/DeepSeek-R1-0528-FP4-V2/q_b_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/q_b_proj", + "nvidia/Kimi-K2.5-NVFP4/q_b_proj", + "nvidia/Kimi-K2.6-NVFP4/q_b_proj" + ] }, { "type": "gemm", "args": { - "m": 55, - "n": 7168, - "k": 4608, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 16384, + "n": 14336, + "k": 512, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/GLM-5.1-MXFP4/kv_b_proj", + "amd/GLM-5.2-MXFP4/kv_b_proj", + "nvidia/GLM-5-NVFP4/kv_b_proj", + "nvidia/GLM-5.1-NVFP4/kv_b_proj", + "nvidia/GLM-5.2-NVFP4/kv_b_proj" + ] }, { "type": "gemm", "args": { - "m": 170496, - "n": 12288, - "k": 1536, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 16384, + "n": 15360, + "k": 1024, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] }, { "type": "gemm", "args": { - "m": 170496, + "m": 16384, "n": 16384, "k": 512, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/Kimi-K2.5-MXFP4/kv_b_proj", + "moonshotai/Kimi-K2.5/kv_b_proj", + "nvidia/DeepSeek-R1-0528-FP4-V2/kv_b_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/kv_b_proj", + "nvidia/Kimi-K2.5-NVFP4/kv_b_proj", + "nvidia/Kimi-K2.6-NVFP4/kv_b_proj" + ] }, { "type": "gemm", "args": { - "m": 170496, - "n": 7168, - "k": 8192, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 16384, + "n": 16384, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_b" + ] }, { "type": "gemm", "args": { - "m": 166, - "n": 12288, + "m": 16384, + "n": 24576, "k": 1536, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/q_b_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/q_b_proj" + ] }, { "type": "gemm", "args": { - "m": 166, - "n": 16384, - "k": 512, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 16384, + "n": 30720, + "k": 1024, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] }, { "type": "gemm", "args": { - "m": 166, - "n": 7168, - "k": 8192, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 16384, + "n": 32768, + "k": 512, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/kv_b_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/kv_b_proj" + ] }, { "type": "gemm", "args": { - "m": 42624, - "n": 1536, - "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 16384, + "n": 65536, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_b" + ] }, { "type": "gemm", "args": { - "m": 42624, - "n": 12288, - "k": 1536, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 16384, + "n": 122880, + "k": 1024, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] }, { "type": "gemm", "args": { - "m": 42624, + "m": 18048, "n": 576, "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/kv_a_proj_with_mqa" + ] }, { "type": "gemm", "args": { - "m": 42624, - "n": 16384, - "k": 512, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 18048, + "n": 1536, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/gate_up_proj", + "deepseek-ai/DeepSeek-R1-0528/q_a_proj", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w3" + ] }, { "type": "gemm", "args": { - "m": 42624, - "n": 7168, - "k": 8192, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 18048, + "n": 1536, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w3" + ] }, { "type": "gemm", "args": { - "m": 42624, - "n": 1152, - "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 18048, + "n": 7168, + "k": 576, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/down_proj" + ] }, { "type": "gemm", "args": { - "m": 42624, + "m": 18048, "n": 7168, - "k": 1152, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "k": 16384, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/o_proj", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] }, { "type": "gemm", "args": { - "m": 41, - "n": 12288, + "m": 18048, + "n": 24576, "k": 1536, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/q_b_proj" + ] }, { "type": "gemm", "args": { - "m": 41, - "n": 16384, + "m": 18048, + "n": 32768, "k": 512, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/kv_b_proj" + ] }, { "type": "gemm", "args": { - "m": 41, - "n": 7168, - "k": 8192, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 18432, + "n": 576, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/kv_a_proj_with_mqa" + ] }, { "type": "gemm", "args": { - "m": 41, - "n": 1152, + "m": 18432, + "n": 1536, "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/gate_up_proj", + "deepseek-ai/DeepSeek-R1-0528/q_a_proj", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w3" + ] }, { "type": "gemm", "args": { - "m": 41, - "n": 7168, - "k": 1152, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 18432, + "n": 2304, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/gate_up_proj" + ] }, { "type": "gemm", "args": { - "m": 576768, - "n": 12288, + "m": 18432, + "n": 3072, "k": 1536, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "MiniMaxAI/MiniMax-M2.5/experts.down_proj", + "MiniMaxAI/MiniMax-M2.5/o_proj", + "deepseek-ai/DeepSeek-R1-0528/q_b_proj" + ] }, { "type": "gemm", "args": { - "m": 576768, - "n": 16384, + "m": 18432, + "n": 4096, "k": 512, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "Qwen/Qwen3.5-397B-A17B-FP8/o_proj", + "Qwen/Qwen3.5-397B-A17B-FP8/shared_expert.down_proj", + "deepseek-ai/DeepSeek-R1-0528/kv_b_proj" + ] }, { "type": "gemm", "args": { - "m": 576768, + "m": 18432, "n": 7168, - "k": 8192, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "k": 2048, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/o_proj", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] }, { "type": "gemm", "args": { - "m": 563, - "n": 12288, - "k": 1536, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 18432, + "n": 7168, + "k": 2304, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/down_proj" + ] }, { "type": "gemm", "args": { - "m": 563, - "n": 16384, - "k": 512, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 18432, + "n": 7168, + "k": 16384, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/o_proj", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] }, { "type": "gemm", "args": { - "m": 563, - "n": 7168, - "k": 8192, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 18432, + "n": 24576, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/q_b_proj" + ] }, { "type": "gemm", "args": { - "m": 72096, - "n": 12288, - "k": 1536, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 18432, + "n": 32768, + "k": 512, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/kv_b_proj" + ] }, { "type": "gemm", "args": { - "m": 72096, - "n": 16384, - "k": 512, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 19680, + "n": 576, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/kv_a_proj_with_mqa" + ] }, { "type": "gemm", "args": { - "m": 72096, - "n": 7168, - "k": 8192, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 19680, + "n": 1536, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/gate_up_proj", + "deepseek-ai/DeepSeek-R1-0528/q_a_proj", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w3" + ] }, { "type": "gemm", "args": { - "m": 70, - "n": 12288, - "k": 1536, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 19680, + "n": 1536, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w3" + ] }, { "type": "gemm", "args": { - "m": 70, - "n": 16384, - "k": 512, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 19680, + "n": 7168, + "k": 576, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/down_proj" + ] }, { "type": "gemm", "args": { - "m": 70, + "m": 19680, "n": 7168, - "k": 8192, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "k": 16384, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/o_proj", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] }, { "type": "gemm", "args": { - "m": 1101056, - "n": 12288, + "m": 19680, + "n": 24576, "k": 1536, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/q_b_proj" + ] }, { "type": "gemm", "args": { - "m": 1101056, - "n": 16384, + "m": 19680, + "n": 32768, "k": 512, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/kv_b_proj" + ] }, { "type": "gemm", "args": { - "m": 1101056, - "n": 7168, - "k": 8192, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 20480, + "n": 576, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/Kimi-K2.5-MXFP4/kv_a_proj_with_mqa", + "moonshotai/Kimi-K2.5/kv_a_proj_with_mqa", + "nvidia/DeepSeek-R1-0528-FP4-V2/kv_a_proj_with_mqa", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/kv_a_proj_with_mqa", + "nvidia/Kimi-K2.5-NVFP4/kv_a_proj_with_mqa", + "nvidia/Kimi-K2.6-NVFP4/kv_a_proj_with_mqa" + ] }, { "type": "gemm", "args": { - "m": 1075, - "n": 12288, - "k": 1536, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 20480, + "n": 1536, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/Kimi-K2.5-MXFP4/q_a_proj", + "moonshotai/Kimi-K2.5/q_a_proj", + "nvidia/DeepSeek-R1-0528-FP4-V2/q_a_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/q_a_proj", + "nvidia/Kimi-K2.5-NVFP4/q_a_proj", + "nvidia/Kimi-K2.6-NVFP4/q_a_proj" + ] }, { "type": "gemm", "args": { - "m": 1075, - "n": 16384, - "k": 512, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 20480, + "n": 2304, + "k": 7168, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/gate_up_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/gate_up_proj", + "nvidia/Kimi-K2.5-NVFP4/gate_up_proj" + ] }, { "type": "gemm", "args": { - "m": 1075, - "n": 7168, - "k": 8192, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 20480, + "n": 3072, + "k": 1536, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/Kimi-K2.5-MXFP4/q_b_proj", + "amd/MiniMax-M2.5-MXFP4/o_proj", + "moonshotai/Kimi-K2.5/q_b_proj", + "nvidia/DeepSeek-R1-0528-FP4-V2/q_b_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/q_b_proj", + "nvidia/Kimi-K2.5-NVFP4/q_b_proj", + "nvidia/Kimi-K2.6-NVFP4/q_b_proj", + "nvidia/MiniMax-M2.5-NVFP4/o_proj" + ] }, { "type": "gemm", "args": { - "m": 1651712, - "n": 12288, + "m": 20480, + "n": 3072, "k": 1536, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/MiniMax-M2.5-NVFP4/experts.down_proj" + ] }, { "type": "gemm", "args": { - "m": 1651712, - "n": 16384, - "k": 512, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 20480, + "n": 3072, + "k": 1536, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 32 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 1, + 32 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "amd/DeepSeek-R1-0528-MXFP4-Preview/q_b_proj", + "amd/DeepSeek-R1-0528-MXFP4-v2/q_b_proj", + "amd/DeepSeek-R1-0528-MXFP4/q_b_proj", + "amd/MiniMax-M2.5-MXFP4/experts.down_proj" + ] }, { "type": "gemm", "args": { - "m": 1651712, - "n": 7168, - "k": 8192, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 20480, + "n": 4096, + "k": 512, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "Qwen/Qwen3.5-397B-A17B/o_proj", + "Qwen/Qwen3.5-397B-A17B/shared_expert.down_proj", + "amd/Kimi-K2.5-MXFP4/kv_b_proj", + "amd/Qwen3.5-397B-A17B-MXFP4/o_proj", + "amd/Qwen3.5-397B-A17B-MXFP4/shared_expert.down_proj", + "moonshotai/Kimi-K2.5/kv_b_proj", + "nvidia/DeepSeek-R1-0528-FP4-V2/kv_b_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/kv_b_proj", + "nvidia/Kimi-K2.5-NVFP4/kv_b_proj", + "nvidia/Kimi-K2.6-NVFP4/kv_b_proj", + "nvidia/Qwen3.5-397B-A17B-NVFP4/o_proj", + "nvidia/Qwen3.5-397B-A17B-NVFP4/shared_expert.down_proj" + ] }, { "type": "gemm", "args": { - "m": 1651712, - "n": 4608, - "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 20480, + "n": 4096, + "k": 512, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 32 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 1, + 32 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "amd/DeepSeek-R1-0528-MXFP4-Preview/kv_b_proj", + "amd/DeepSeek-R1-0528-MXFP4-v2/kv_b_proj", + "amd/DeepSeek-R1-0528-MXFP4/kv_b_proj", + "amd/Qwen3.5-397B-A17B-MXFP4-AttnFP8-V2/shared_expert.down_proj" + ] }, { "type": "gemm", "args": { - "m": 1651712, - "n": 7168, - "k": 4608, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 20480, + "n": 4608, + "k": 7168, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/gate_up_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/gate_up_proj", + "nvidia/Kimi-K2.5-NVFP4/gate_up_proj" + ] }, { "type": "gemm", "args": { - "m": 1613, - "n": 12288, - "k": 1536, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 20480, + "n": 4608, + "k": 7168, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 32 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 1, + 32 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "amd/DeepSeek-R1-0528-MXFP4-Preview/gate_up_proj", + "amd/DeepSeek-R1-0528-MXFP4-v2/gate_up_proj", + "amd/DeepSeek-R1-0528-MXFP4/gate_up_proj", + "amd/Kimi-K2.5-MXFP4/gate_up_proj" + ] }, { "type": "gemm", "args": { - "m": 1613, - "n": 16384, - "k": 512, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 20480, + "n": 7168, + "k": 2048, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/Kimi-K2.5-MXFP4/o_proj", + "moonshotai/Kimi-K2.5/o_proj", + "nvidia/Kimi-K2.5-NVFP4/o_proj", + "nvidia/Kimi-K2.6-NVFP4/o_proj" + ] }, { "type": "gemm", "args": { - "m": 1613, + "m": 20480, "n": 7168, - "k": 8192, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "k": 2048, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/o_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/o_proj" + ] }, { "type": "gemm", "args": { - "m": 550528, - "n": 12288, - "k": 1536, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 20480, + "n": 7168, + "k": 2048, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 32 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 1, + 32 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "amd/DeepSeek-R1-0528-MXFP4-Preview/o_proj", + "amd/DeepSeek-R1-0528-MXFP4-v2/o_proj", + "amd/DeepSeek-R1-0528-MXFP4/o_proj" + ] }, { "type": "gemm", "args": { - "m": 550528, - "n": 16384, - "k": 512, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 20480, + "n": 7168, + "k": 2304, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/down_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/down_proj", + "nvidia/Kimi-K2.5-NVFP4/down_proj" + ] }, { "type": "gemm", "args": { - "m": 550528, + "m": 20480, "n": 7168, - "k": 8192, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "k": 2304, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 32 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 1, + 32 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "amd/DeepSeek-R1-0528-MXFP4-Preview/down_proj", + "amd/DeepSeek-R1-0528-MXFP4-v2/down_proj", + "amd/DeepSeek-R1-0528-MXFP4/down_proj", + "amd/Kimi-K2.5-MXFP4/down_proj" + ] }, { "type": "gemm", "args": { - "m": 537, - "n": 12288, - "k": 1536, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 20480, + "n": 7168, + "k": 4608, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/down_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/down_proj", + "nvidia/Kimi-K2.5-NVFP4/down_proj" + ] }, { "type": "gemm", "args": { - "m": 537, - "n": 16384, - "k": 512, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 20480, + "n": 7168, + "k": 9216, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/down_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/down_proj", + "nvidia/Kimi-K2.5-NVFP4/down_proj" + ] }, { "type": "gemm", "args": { - "m": 537, + "m": 20480, "n": 7168, - "k": 8192, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "k": 16384, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/o_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/o_proj" + ] }, { "type": "gemm", "args": { - "m": 825856, - "n": 12288, - "k": 1536, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 20480, + "n": 9216, + "k": 7168, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/gate_up_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/gate_up_proj", + "nvidia/Kimi-K2.5-NVFP4/gate_up_proj" + ] }, { "type": "gemm", "args": { - "m": 825856, - "n": 16384, - "k": 512, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 20480, + "n": 24576, + "k": 1536, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/q_b_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/q_b_proj" + ] }, { "type": "gemm", "args": { - "m": 825856, - "n": 7168, - "k": 8192, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 20480, + "n": 32768, + "k": 512, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/kv_b_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/kv_b_proj" + ] }, { "type": "gemm", "args": { - "m": 825856, - "n": 2304, + "m": 21312, + "n": 576, "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/Kimi-K2.5-MXFP4/kv_a_proj_with_mqa", + "moonshotai/Kimi-K2.5/kv_a_proj_with_mqa", + "nvidia/DeepSeek-R1-0528-FP4-V2/kv_a_proj_with_mqa", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/kv_a_proj_with_mqa", + "nvidia/Kimi-K2.5-NVFP4/kv_a_proj_with_mqa", + "nvidia/Kimi-K2.6-NVFP4/kv_a_proj_with_mqa" + ] }, { "type": "gemm", "args": { - "m": 825856, - "n": 7168, - "k": 2304, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 21312, + "n": 576, + "k": 7168, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 32 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 1, + 32 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "amd/DeepSeek-R1-0528-MXFP4-Preview/kv_a_proj_with_mqa", + "amd/DeepSeek-R1-0528-MXFP4-v2/kv_a_proj_with_mqa", + "amd/DeepSeek-R1-0528-MXFP4/kv_a_proj_with_mqa" + ] }, { "type": "gemm", "args": { - "m": 806, + "m": 21312, "n": 1536, "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/Kimi-K2.5-MXFP4/q_a_proj", + "moonshotai/Kimi-K2.5/q_a_proj", + "nvidia/DeepSeek-R1-0528-FP4-V2/q_a_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/q_a_proj", + "nvidia/Kimi-K2.5-NVFP4/q_a_proj", + "nvidia/Kimi-K2.6-NVFP4/q_a_proj" + ] }, { "type": "gemm", "args": { - "m": 806, - "n": 12288, - "k": 1536, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 21312, + "n": 1536, + "k": 7168, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 32 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 1, + 32 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "amd/DeepSeek-R1-0528-MXFP4-Preview/q_a_proj", + "amd/DeepSeek-R1-0528-MXFP4-v2/q_a_proj", + "amd/DeepSeek-R1-0528-MXFP4/q_a_proj" + ] }, { "type": "gemm", "args": { - "m": 806, - "n": 576, + "m": 21312, + "n": 1536, "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w3" + ] }, { "type": "gemm", "args": { - "m": 806, - "n": 16384, - "k": 512, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 21312, + "n": 7168, + "k": 576, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-NVFP4-v2/down_proj" + ] }, { "type": "gemm", "args": { - "m": 806, + "m": 21312, "n": 7168, - "k": 8192, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "k": 16384, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/o_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/o_proj" + ] }, { "type": "gemm", "args": { - "m": 806, - "n": 2304, - "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 21312, + "n": 7168, + "k": 16384, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 32 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 1, + 32 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "amd/DeepSeek-R1-0528-MXFP4-Preview/o_proj", + "amd/DeepSeek-R1-0528-MXFP4-v2/o_proj", + "amd/DeepSeek-R1-0528-MXFP4/o_proj" + ] }, { "type": "gemm", "args": { - "m": 806, + "m": 21312, "n": 7168, - "k": 2304, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "k": 16384, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] }, { "type": "gemm", "args": { - "m": 275264, - "n": 12288, + "m": 21312, + "n": 24576, "k": 1536, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 275264, - "n": 16384, - "k": 512, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/q_b_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/q_b_proj" + ] }, { "type": "gemm", "args": { - "m": 275264, - "n": 7168, - "k": 8192, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 21312, + "n": 24576, + "k": 1536, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 32 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 1, + 32 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "amd/DeepSeek-R1-0528-MXFP4-Preview/q_b_proj", + "amd/DeepSeek-R1-0528-MXFP4-v2/q_b_proj", + "amd/DeepSeek-R1-0528-MXFP4/q_b_proj" + ] }, { "type": "gemm", "args": { - "m": 268, - "n": 12288, - "k": 1536, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 21312, + "n": 32768, + "k": 512, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/kv_b_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/kv_b_proj" + ] }, { "type": "gemm", "args": { - "m": 268, - "n": 16384, + "m": 21312, + "n": 32768, "k": 512, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 32 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 1, + 32 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "amd/DeepSeek-R1-0528-MXFP4-Preview/kv_b_proj", + "amd/DeepSeek-R1-0528-MXFP4-v2/kv_b_proj", + "amd/DeepSeek-R1-0528-MXFP4/kv_b_proj" + ] }, { "type": "gemm", "args": { - "m": 268, - "n": 7168, - "k": 8192, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 21504, + "n": 576, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/kv_a_proj_with_mqa" + ] }, { "type": "gemm", "args": { - "m": 137632, + "m": 21504, "n": 1536, "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/gate_up_proj", + "deepseek-ai/DeepSeek-R1-0528/q_a_proj", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w3" + ] }, { "type": "gemm", "args": { - "m": 137632, - "n": 12288, - "k": 1536, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 21504, + "n": 4608, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/gate_up_proj" + ] }, { "type": "gemm", "args": { - "m": 137632, - "n": 576, - "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 21504, + "n": 7168, + "k": 4608, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/down_proj" + ] }, { "type": "gemm", "args": { - "m": 137632, - "n": 16384, - "k": 512, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 21504, + "n": 7168, + "k": 16384, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/o_proj", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] }, { "type": "gemm", "args": { - "m": 137632, - "n": 7168, - "k": 8192, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 21504, + "n": 24576, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/q_b_proj" + ] }, { "type": "gemm", "args": { - "m": 137632, - "n": 7168, - "k": 576, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 21504, + "n": 32768, + "k": 512, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/kv_b_proj" + ] }, { "type": "gemm", "args": { - "m": 134, - "n": 1536, + "m": 21845, + "n": 384, "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w3" + ] }, { "type": "gemm", "args": { - "m": 134, - "n": 12288, - "k": 1536, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 21845, + "n": 576, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/kv_a_proj_with_mqa" + ] }, { "type": "gemm", "args": { - "m": 134, - "n": 576, + "m": 21845, + "n": 1536, "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/gate_up_proj", + "deepseek-ai/DeepSeek-R1-0528/q_a_proj", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w3" + ] }, { "type": "gemm", "args": { - "m": 134, - "n": 16384, - "k": 512, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 21845, + "n": 1536, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w3" + ] }, { "type": "gemm", "args": { - "m": 134, + "m": 21845, "n": 7168, - "k": 8192, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "k": 384, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w2" + ] }, { "type": "gemm", "args": { - "m": 134, + "m": 21845, "n": 7168, - "k": 576, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "k": 16384, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/o_proj", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] }, { "type": "gemm", "args": { - "m": 8192, - "n": 12288, + "m": 21845, + "n": 24576, "k": 1536, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/q_b_proj" + ] }, { "type": "gemm", "args": { - "m": 8192, - "n": 16384, + "m": 21845, + "n": 32768, "k": 512, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/kv_b_proj" + ] }, { "type": "gemm", "args": { - "m": 319488, - "n": 1536, + "m": 23040, + "n": 576, "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/Kimi-K2.5-MXFP4/kv_a_proj_with_mqa", + "moonshotai/Kimi-K2.5/kv_a_proj_with_mqa", + "nvidia/DeepSeek-R1-0528-FP4-V2/kv_a_proj_with_mqa", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/kv_a_proj_with_mqa", + "nvidia/Kimi-K2.5-NVFP4/kv_a_proj_with_mqa", + "nvidia/Kimi-K2.6-NVFP4/kv_a_proj_with_mqa" + ] }, { "type": "gemm", "args": { - "m": 319488, - "n": 12288, - "k": 1536, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 23040, + "n": 1536, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/Kimi-K2.5-MXFP4/q_a_proj", + "moonshotai/Kimi-K2.5/q_a_proj", + "nvidia/DeepSeek-R1-0528-FP4-V2/q_a_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/q_a_proj", + "nvidia/Kimi-K2.5-NVFP4/q_a_proj", + "nvidia/Kimi-K2.6-NVFP4/q_a_proj" + ] }, { "type": "gemm", "args": { - "m": 319488, - "n": 576, + "m": 23040, + "n": 4608, "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/gate_up_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/gate_up_proj", + "nvidia/Kimi-K2.5-NVFP4/gate_up_proj" + ] }, { "type": "gemm", "args": { - "m": 319488, - "n": 16384, - "k": 512, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 23040, + "n": 4608, + "k": 7168, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 32 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 1, + 32 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "amd/DeepSeek-R1-0528-MXFP4-Preview/gate_up_proj", + "amd/DeepSeek-R1-0528-MXFP4-v2/gate_up_proj", + "amd/DeepSeek-R1-0528-MXFP4/gate_up_proj", + "amd/Kimi-K2.5-MXFP4/gate_up_proj" + ] }, { "type": "gemm", "args": { - "m": 319488, + "m": 23040, "n": 7168, - "k": 8192, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "k": 576, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/down_proj" + ] }, { "type": "gemm", "args": { - "m": 319488, - "n": 4608, - "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 23040, + "n": 7168, + "k": 4608, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/down_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/down_proj", + "nvidia/Kimi-K2.5-NVFP4/down_proj" + ] }, { "type": "gemm", "args": { - "m": 319488, + "m": 23040, "n": 7168, - "k": 4608, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "k": 16384, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/o_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/o_proj" + ] }, { "type": "gemm", "args": { - "m": 39, - "n": 1536, - "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 23040, + "n": 24576, + "k": 1536, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/q_b_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/q_b_proj" + ] }, { "type": "gemm", "args": { - "m": 39, - "n": 12288, - "k": 1536, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 23040, + "n": 32768, + "k": 512, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/kv_b_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/kv_b_proj" + ] }, { "type": "gemm", "args": { - "m": 39, + "m": 24576, "n": 576, "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/Kimi-K2.5-MXFP4/kv_a_proj_with_mqa", + "moonshotai/Kimi-K2.5/kv_a_proj_with_mqa", + "nvidia/DeepSeek-R1-0528-FP4-V2/kv_a_proj_with_mqa", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/kv_a_proj_with_mqa", + "nvidia/Kimi-K2.5-NVFP4/kv_a_proj_with_mqa", + "nvidia/Kimi-K2.6-NVFP4/kv_a_proj_with_mqa" + ] }, { "type": "gemm", "args": { - "m": 39, - "n": 16384, - "k": 512, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 24576, + "n": 1536, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/Kimi-K2.5-MXFP4/q_a_proj", + "moonshotai/Kimi-K2.5/q_a_proj", + "nvidia/DeepSeek-R1-0528-FP4-V2/q_a_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/q_a_proj", + "nvidia/Kimi-K2.5-NVFP4/q_a_proj", + "nvidia/Kimi-K2.6-NVFP4/q_a_proj" + ] }, { "type": "gemm", "args": { - "m": 39, - "n": 7168, - "k": 8192, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 24576, + "n": 2304, + "k": 7168, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/gate_up_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/gate_up_proj", + "nvidia/Kimi-K2.5-NVFP4/gate_up_proj" + ] }, { "type": "gemm", "args": { - "m": 39, - "n": 4608, + "m": 24576, + "n": 2304, "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/gate_up_proj" + ] }, { "type": "gemm", "args": { - "m": 39, - "n": 7168, - "k": 4608, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 24576, + "n": 3072, + "k": 1536, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/Kimi-K2.5-MXFP4/q_b_proj", + "amd/MiniMax-M2.5-MXFP4/o_proj", + "moonshotai/Kimi-K2.5/q_b_proj", + "nvidia/DeepSeek-R1-0528-FP4-V2/q_b_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/q_b_proj", + "nvidia/Kimi-K2.5-NVFP4/q_b_proj", + "nvidia/Kimi-K2.6-NVFP4/q_b_proj", + "nvidia/MiniMax-M2.5-NVFP4/o_proj" + ] }, { "type": "gemm", "args": { - "m": 1277952, - "n": 1536, - "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 24576, + "n": 3072, + "k": 1536, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/MiniMax-M2.5-NVFP4/experts.down_proj" + ] }, { "type": "gemm", "args": { - "m": 1277952, + "m": 24576, "n": 3072, "k": 1536, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 32 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 1, + 32 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "amd/DeepSeek-R1-0528-MXFP4-Preview/q_b_proj", + "amd/DeepSeek-R1-0528-MXFP4-v2/q_b_proj", + "amd/DeepSeek-R1-0528-MXFP4/q_b_proj", + "amd/MiniMax-M2.5-MXFP4/experts.down_proj" + ] }, { "type": "gemm", "args": { - "m": 1277952, - "n": 576, - "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 24576, + "n": 3072, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "MiniMaxAI/MiniMax-M2.5/experts.down_proj", + "MiniMaxAI/MiniMax-M2.5/o_proj", + "deepseek-ai/DeepSeek-R1-0528/q_b_proj" + ] }, { "type": "gemm", "args": { - "m": 1277952, + "m": 24576, "n": 4096, "k": 512, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 1277952, - "n": 7168, - "k": 2048, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "Qwen/Qwen3.5-397B-A17B/o_proj", + "Qwen/Qwen3.5-397B-A17B/shared_expert.down_proj", + "amd/Kimi-K2.5-MXFP4/kv_b_proj", + "amd/Qwen3.5-397B-A17B-MXFP4/o_proj", + "amd/Qwen3.5-397B-A17B-MXFP4/shared_expert.down_proj", + "moonshotai/Kimi-K2.5/kv_b_proj", + "nvidia/DeepSeek-R1-0528-FP4-V2/kv_b_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/kv_b_proj", + "nvidia/Kimi-K2.5-NVFP4/kv_b_proj", + "nvidia/Kimi-K2.6-NVFP4/kv_b_proj", + "nvidia/Qwen3.5-397B-A17B-NVFP4/o_proj", + "nvidia/Qwen3.5-397B-A17B-NVFP4/shared_expert.down_proj" + ] }, { "type": "gemm", "args": { - "m": 1277952, - "n": 4608, - "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 24576, + "n": 4096, + "k": 512, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 32 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 1, + 32 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "amd/DeepSeek-R1-0528-MXFP4-Preview/kv_b_proj", + "amd/DeepSeek-R1-0528-MXFP4-v2/kv_b_proj", + "amd/DeepSeek-R1-0528-MXFP4/kv_b_proj", + "amd/Qwen3.5-397B-A17B-MXFP4-AttnFP8-V2/shared_expert.down_proj" + ] }, { "type": "gemm", "args": { - "m": 1277952, - "n": 7168, - "k": 4608, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 24576, + "n": 4096, + "k": 512, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "Qwen/Qwen3.5-397B-A17B-FP8/o_proj", + "Qwen/Qwen3.5-397B-A17B-FP8/shared_expert.down_proj", + "deepseek-ai/DeepSeek-R1-0528/kv_b_proj" + ] }, { "type": "gemm", "args": { - "m": 156, - "n": 1536, + "m": 24576, + "n": 4608, "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/gate_up_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/gate_up_proj", + "nvidia/Kimi-K2.5-NVFP4/gate_up_proj" + ] }, { "type": "gemm", "args": { - "m": 156, - "n": 3072, - "k": 1536, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 24576, + "n": 4608, + "k": 7168, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 32 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 1, + 32 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "amd/DeepSeek-R1-0528-MXFP4-Preview/gate_up_proj", + "amd/DeepSeek-R1-0528-MXFP4-v2/gate_up_proj", + "amd/DeepSeek-R1-0528-MXFP4/gate_up_proj", + "amd/Kimi-K2.5-MXFP4/gate_up_proj" + ] }, { "type": "gemm", "args": { - "m": 156, - "n": 576, - "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 24576, + "n": 7168, + "k": 2048, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/Kimi-K2.5-MXFP4/o_proj", + "moonshotai/Kimi-K2.5/o_proj", + "nvidia/Kimi-K2.5-NVFP4/o_proj", + "nvidia/Kimi-K2.6-NVFP4/o_proj" + ] }, { "type": "gemm", "args": { - "m": 156, - "n": 4096, - "k": 512, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 24576, + "n": 7168, + "k": 2048, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/o_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/o_proj" + ] }, { "type": "gemm", "args": { - "m": 156, + "m": 24576, "n": 7168, "k": 2048, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 32 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 1, + 32 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "amd/DeepSeek-R1-0528-MXFP4-Preview/o_proj", + "amd/DeepSeek-R1-0528-MXFP4-v2/o_proj", + "amd/DeepSeek-R1-0528-MXFP4/o_proj" + ] }, { "type": "gemm", "args": { - "m": 156, - "n": 4608, - "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 24576, + "n": 7168, + "k": 2048, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/o_proj", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] }, { "type": "gemm", "args": { - "m": 156, + "m": 24576, "n": 7168, - "k": 4608, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "k": 2304, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/down_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/down_proj", + "nvidia/Kimi-K2.5-NVFP4/down_proj" + ] }, { "type": "gemm", "args": { - "m": 10240, - "n": 12288, - "k": 1536, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 24576, + "n": 7168, + "k": 2304, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 32 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 1, + 32 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "amd/DeepSeek-R1-0528-MXFP4-Preview/down_proj", + "amd/DeepSeek-R1-0528-MXFP4-v2/down_proj", + "amd/DeepSeek-R1-0528-MXFP4/down_proj", + "amd/Kimi-K2.5-MXFP4/down_proj" + ] }, { "type": "gemm", "args": { - "m": 10240, - "n": 16384, - "k": 512, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 24576, + "n": 7168, + "k": 2304, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/down_proj" + ] }, { "type": "gemm", "args": { - "m": 10240, + "m": 24576, "n": 7168, - "k": 8192, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "k": 4608, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/down_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/down_proj", + "nvidia/Kimi-K2.5-NVFP4/down_proj" + ] }, { "type": "gemm", "args": { - "m": 30720, - "n": 12288, - "k": 1536, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 24576, + "n": 7168, + "k": 9216, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/down_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/down_proj", + "nvidia/Kimi-K2.5-NVFP4/down_proj" + ] }, { "type": "gemm", "args": { - "m": 30720, - "n": 16384, - "k": 512, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 24576, + "n": 7168, + "k": 16384, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/o_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/o_proj" + ] }, { "type": "gemm", "args": { - "m": 30720, - "n": 7168, - "k": 8192, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 24576, + "n": 9216, + "k": 7168, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/gate_up_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/gate_up_proj", + "nvidia/Kimi-K2.5-NVFP4/gate_up_proj" + ] }, { "type": "gemm", "args": { - "m": 61440, - "n": 12288, + "m": 24576, + "n": 24576, "k": 1536, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/q_b_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/q_b_proj" + ] }, { "type": "gemm", "args": { - "m": 61440, - "n": 16384, + "m": 24576, + "n": 32768, "k": 512, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/kv_b_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/kv_b_proj" + ] }, { "type": "gemm", "args": { - "m": 61440, - "n": 7168, - "k": 8192, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 25600, + "n": 576, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/Kimi-K2.5-MXFP4/kv_a_proj_with_mqa", + "moonshotai/Kimi-K2.5/kv_a_proj_with_mqa", + "nvidia/DeepSeek-R1-0528-FP4-V2/kv_a_proj_with_mqa", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/kv_a_proj_with_mqa", + "nvidia/Kimi-K2.5-NVFP4/kv_a_proj_with_mqa", + "nvidia/Kimi-K2.6-NVFP4/kv_a_proj_with_mqa" + ] }, { "type": "gemm", "args": { - "m": 122880, - "n": 12288, - "k": 1536, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 25600, + "n": 576, + "k": 7168, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 32 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 1, + 32 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "amd/DeepSeek-R1-0528-MXFP4-Preview/kv_a_proj_with_mqa", + "amd/DeepSeek-R1-0528-MXFP4-v2/kv_a_proj_with_mqa", + "amd/DeepSeek-R1-0528-MXFP4/kv_a_proj_with_mqa" + ] }, { "type": "gemm", "args": { - "m": 122880, - "n": 16384, - "k": 512, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 25600, + "n": 1536, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/Kimi-K2.5-MXFP4/q_a_proj", + "moonshotai/Kimi-K2.5/q_a_proj", + "nvidia/DeepSeek-R1-0528-FP4-V2/q_a_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/q_a_proj", + "nvidia/Kimi-K2.5-NVFP4/q_a_proj", + "nvidia/Kimi-K2.6-NVFP4/q_a_proj" + ] }, { "type": "gemm", "args": { - "m": 122880, - "n": 7168, - "k": 8192, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 25600, + "n": 1536, + "k": 7168, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 32 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 1, + 32 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "amd/DeepSeek-R1-0528-MXFP4-Preview/q_a_proj", + "amd/DeepSeek-R1-0528-MXFP4-v2/q_a_proj", + "amd/DeepSeek-R1-0528-MXFP4/q_a_proj" + ] }, { "type": "gemm", "args": { - "m": 15, - "n": 12288, - "k": 1536, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 25600, + "n": 1536, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w3" + ] }, { "type": "gemm", "args": { - "m": 15, - "n": 16384, - "k": 512, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 25600, + "n": 2304, + "k": 7168, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/gate_up_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/gate_up_proj", + "nvidia/Kimi-K2.5-NVFP4/gate_up_proj" + ] }, { "type": "gemm", "args": { - "m": 15, - "n": 7168, - "k": 8192, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 25600, + "n": 3072, + "k": 1536, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/Kimi-K2.5-MXFP4/q_b_proj", + "amd/MiniMax-M2.5-MXFP4/o_proj", + "moonshotai/Kimi-K2.5/q_b_proj", + "nvidia/DeepSeek-R1-0528-FP4-V2/q_b_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/q_b_proj", + "nvidia/Kimi-K2.5-NVFP4/q_b_proj", + "nvidia/Kimi-K2.6-NVFP4/q_b_proj", + "nvidia/MiniMax-M2.5-NVFP4/o_proj" + ] }, { "type": "gemm", "args": { - "m": 215040, - "n": 1536, - "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 25600, + "n": 3072, + "k": 1536, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/MiniMax-M2.5-NVFP4/experts.down_proj" + ] }, { "type": "gemm", "args": { - "m": 215040, - "n": 12288, + "m": 25600, + "n": 3072, "k": 1536, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 32 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 1, + 32 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "amd/DeepSeek-R1-0528-MXFP4-Preview/q_b_proj", + "amd/DeepSeek-R1-0528-MXFP4-v2/q_b_proj", + "amd/DeepSeek-R1-0528-MXFP4/q_b_proj", + "amd/MiniMax-M2.5-MXFP4/experts.down_proj" + ] }, { "type": "gemm", "args": { - "m": 215040, - "n": 576, - "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 25600, + "n": 4096, + "k": 512, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "Qwen/Qwen3.5-397B-A17B/o_proj", + "Qwen/Qwen3.5-397B-A17B/shared_expert.down_proj", + "amd/Kimi-K2.5-MXFP4/kv_b_proj", + "amd/Qwen3.5-397B-A17B-MXFP4/o_proj", + "amd/Qwen3.5-397B-A17B-MXFP4/shared_expert.down_proj", + "moonshotai/Kimi-K2.5/kv_b_proj", + "nvidia/DeepSeek-R1-0528-FP4-V2/kv_b_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/kv_b_proj", + "nvidia/Kimi-K2.5-NVFP4/kv_b_proj", + "nvidia/Kimi-K2.6-NVFP4/kv_b_proj", + "nvidia/Qwen3.5-397B-A17B-NVFP4/o_proj", + "nvidia/Qwen3.5-397B-A17B-NVFP4/shared_expert.down_proj" + ] }, { "type": "gemm", "args": { - "m": 215040, - "n": 16384, + "m": 25600, + "n": 4096, "k": 512, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 32 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 1, + 32 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "amd/DeepSeek-R1-0528-MXFP4-Preview/kv_b_proj", + "amd/DeepSeek-R1-0528-MXFP4-v2/kv_b_proj", + "amd/DeepSeek-R1-0528-MXFP4/kv_b_proj", + "amd/Qwen3.5-397B-A17B-MXFP4-AttnFP8-V2/shared_expert.down_proj" + ] }, { "type": "gemm", "args": { - "m": 215040, - "n": 7168, - "k": 8192, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 25600, + "n": 4608, + "k": 7168, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/gate_up_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/gate_up_proj", + "nvidia/Kimi-K2.5-NVFP4/gate_up_proj" + ] }, { "type": "gemm", "args": { - "m": 215040, + "m": 25600, "n": 4608, "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 32 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 1, + 32 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "amd/DeepSeek-R1-0528-MXFP4-Preview/gate_up_proj", + "amd/DeepSeek-R1-0528-MXFP4-v2/gate_up_proj", + "amd/DeepSeek-R1-0528-MXFP4/gate_up_proj", + "amd/Kimi-K2.5-MXFP4/gate_up_proj" + ] }, { "type": "gemm", "args": { - "m": 215040, - "n": 7168, - "k": 4608, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 25600, + "n": 6144, + "k": 1536, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/GLM-5.1-MXFP4/down_proj", + "amd/GLM-5.2-MXFP4/down_proj", + "amd/Kimi-K2.5-MXFP4/q_b_proj", + "amd/MiniMax-M3-MXFP4/down_proj", + "moonshotai/Kimi-K2.5/q_b_proj", + "nvidia/DeepSeek-R1-0528-FP4-V2/q_b_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/q_b_proj", + "nvidia/GLM-5.1-NVFP4/down_proj", + "nvidia/GLM-5.2-NVFP4/down_proj", + "nvidia/Kimi-K2.5-NVFP4/q_b_proj", + "nvidia/Kimi-K2.6-NVFP4/q_b_proj" + ] }, { "type": "gemm", "args": { - "m": 26, - "n": 1536, - "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 25600, + "n": 6144, + "k": 1536, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/GLM-5-NVFP4/down_proj" + ] }, { "type": "gemm", "args": { - "m": 26, - "n": 12288, + "m": 25600, + "n": 6144, "k": 1536, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 32 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 1, + 32 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "amd/DeepSeek-R1-0528-MXFP4-Preview/q_b_proj", + "amd/DeepSeek-R1-0528-MXFP4-v2/q_b_proj", + "amd/DeepSeek-R1-0528-MXFP4/q_b_proj" + ] }, { "type": "gemm", "args": { - "m": 26, - "n": 576, - "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 25600, + "n": 7168, + "k": 2048, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/Kimi-K2.5-MXFP4/o_proj", + "moonshotai/Kimi-K2.5/o_proj", + "nvidia/Kimi-K2.5-NVFP4/o_proj", + "nvidia/Kimi-K2.6-NVFP4/o_proj" + ] }, { "type": "gemm", "args": { - "m": 26, - "n": 16384, - "k": 512, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 25600, + "n": 7168, + "k": 2048, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/o_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/o_proj" + ] }, { "type": "gemm", "args": { - "m": 26, + "m": 25600, "n": 7168, - "k": 8192, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "k": 2048, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 32 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 1, + 32 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "amd/DeepSeek-R1-0528-MXFP4-Preview/o_proj", + "amd/DeepSeek-R1-0528-MXFP4-v2/o_proj", + "amd/DeepSeek-R1-0528-MXFP4/o_proj" + ] }, { "type": "gemm", "args": { - "m": 26, - "n": 4608, - "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 25600, + "n": 7168, + "k": 2304, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/down_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/down_proj", + "nvidia/Kimi-K2.5-NVFP4/down_proj" + ] }, { "type": "gemm", "args": { - "m": 26, + "m": 25600, "n": 7168, - "k": 4608, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "k": 2304, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 32 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 1, + 32 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "amd/DeepSeek-R1-0528-MXFP4-Preview/down_proj", + "amd/DeepSeek-R1-0528-MXFP4-v2/down_proj", + "amd/DeepSeek-R1-0528-MXFP4/down_proj", + "amd/Kimi-K2.5-MXFP4/down_proj" + ] }, { "type": "gemm", "args": { - "m": 491520, - "n": 4608, - "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 25600, + "n": 7168, + "k": 4096, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/o_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/o_proj" + ] }, { "type": "gemm", "args": { - "m": 491520, + "m": 25600, "n": 7168, - "k": 4608, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "k": 4096, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 32 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 1, + 32 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "amd/DeepSeek-R1-0528-MXFP4-Preview/o_proj", + "amd/DeepSeek-R1-0528-MXFP4-v2/o_proj", + "amd/DeepSeek-R1-0528-MXFP4/o_proj" + ] }, { "type": "gemm", "args": { - "m": 60, - "n": 4608, - "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 25600, + "n": 7168, + "k": 4608, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/down_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/down_proj", + "nvidia/Kimi-K2.5-NVFP4/down_proj" + ] }, { "type": "gemm", "args": { - "m": 60, + "m": 25600, "n": 7168, "k": 4608, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 32 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 1, + 32 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "amd/DeepSeek-R1-0528-MXFP4-Preview/down_proj", + "amd/DeepSeek-R1-0528-MXFP4-v2/down_proj", + "amd/DeepSeek-R1-0528-MXFP4/down_proj", + "amd/Kimi-K2.5-MXFP4/down_proj" + ] }, { "type": "gemm", "args": { - "m": 860160, - "n": 3072, - "k": 1536, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 25600, + "n": 8192, + "k": 512, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/Kimi-K2.5-MXFP4/kv_b_proj", + "moonshotai/Kimi-K2.5/kv_b_proj", + "nvidia/DeepSeek-R1-0528-FP4-V2/kv_b_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/kv_b_proj", + "nvidia/Kimi-K2.5-NVFP4/kv_b_proj", + "nvidia/Kimi-K2.6-NVFP4/kv_b_proj" + ] }, { "type": "gemm", "args": { - "m": 860160, - "n": 4096, + "m": 25600, + "n": 8192, "k": 512, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 32 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 1, + 32 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "amd/DeepSeek-R1-0528-MXFP4-Preview/kv_b_proj", + "amd/DeepSeek-R1-0528-MXFP4-v2/kv_b_proj", + "amd/DeepSeek-R1-0528-MXFP4/kv_b_proj" + ] }, { "type": "gemm", "args": { - "m": 860160, - "n": 7168, - "k": 2048, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 27648, + "n": 576, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/kv_a_proj_with_mqa" + ] }, { "type": "gemm", "args": { - "m": 105, - "n": 3072, - "k": 1536, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 27648, + "n": 1536, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/gate_up_proj", + "deepseek-ai/DeepSeek-R1-0528/q_a_proj", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w3" + ] }, { "type": "gemm", "args": { - "m": 105, - "n": 4096, - "k": 512, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 27648, + "n": 1536, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w3" + ] }, { "type": "gemm", "args": { - "m": 105, - "n": 7168, - "k": 2048, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 27648, + "n": 2304, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/gate_up_proj" + ] }, { "type": "gemm", "args": { - "m": 681984, - "n": 12288, + "m": 27648, + "n": 3072, "k": 1536, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "MiniMaxAI/MiniMax-M2.5/experts.down_proj", + "MiniMaxAI/MiniMax-M2.5/o_proj", + "deepseek-ai/DeepSeek-R1-0528/q_b_proj" + ] }, { "type": "gemm", "args": { - "m": 681984, - "n": 16384, + "m": 27648, + "n": 4096, "k": 512, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "Qwen/Qwen3.5-397B-A17B-FP8/o_proj", + "Qwen/Qwen3.5-397B-A17B-FP8/shared_expert.down_proj", + "deepseek-ai/DeepSeek-R1-0528/kv_b_proj" + ] }, { "type": "gemm", "args": { - "m": 681984, + "m": 27648, "n": 7168, - "k": 8192, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "k": 2048, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/o_proj", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] }, { "type": "gemm", "args": { - "m": 83, - "n": 12288, - "k": 1536, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 27648, + "n": 7168, + "k": 2304, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/down_proj" + ] }, { "type": "gemm", "args": { - "m": 83, - "n": 16384, - "k": 512, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 29568, + "n": 576, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/kv_a_proj_with_mqa" + ] }, { "type": "gemm", "args": { - "m": 83, - "n": 7168, - "k": 8192, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 29568, + "n": 1152, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/gate_up_proj" + ] }, { "type": "gemm", "args": { - "m": 170496, - "n": 1152, + "m": 29568, + "n": 1536, "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/gate_up_proj", + "deepseek-ai/DeepSeek-R1-0528/q_a_proj", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w3" + ] }, { "type": "gemm", "args": { - "m": 170496, + "m": 29568, "n": 7168, "k": 1152, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/down_proj" + ] }, { "type": "gemm", "args": { - "m": 20, - "n": 12288, + "m": 29568, + "n": 7168, + "k": 16384, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/o_proj", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 29568, + "n": 24576, "k": 1536, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/q_b_proj" + ] }, { "type": "gemm", "args": { - "m": 20, - "n": 16384, + "m": 29568, + "n": 32768, "k": 512, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/kv_b_proj" + ] }, { "type": "gemm", "args": { - "m": 20, - "n": 7168, - "k": 8192, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 30720, + "n": 576, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/Kimi-K2.5-MXFP4/kv_a_proj_with_mqa", + "moonshotai/Kimi-K2.5/kv_a_proj_with_mqa", + "nvidia/DeepSeek-R1-0528-FP4-V2/kv_a_proj_with_mqa", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/kv_a_proj_with_mqa", + "nvidia/Kimi-K2.5-NVFP4/kv_a_proj_with_mqa", + "nvidia/Kimi-K2.6-NVFP4/kv_a_proj_with_mqa" + ] }, { "type": "gemm", "args": { - "m": 20, + "m": 30720, "n": 1152, "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/gate_up_proj" + ] }, { "type": "gemm", "args": { - "m": 20, - "n": 7168, - "k": 1152, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 30720, + "n": 1536, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/q_b_proj" + ] }, { "type": "gemm", "args": { - "m": 1259520, + "m": 30720, "n": 1536, "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/Kimi-K2.5-MXFP4/q_a_proj", + "moonshotai/Kimi-K2.5/q_a_proj", + "nvidia/DeepSeek-R1-0528-FP4-V2/q_a_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/q_a_proj", + "nvidia/Kimi-K2.5-NVFP4/q_a_proj", + "nvidia/Kimi-K2.6-NVFP4/q_a_proj" + ] }, { "type": "gemm", "args": { - "m": 1259520, - "n": 12288, - "k": 1536, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 30720, + "n": 2048, + "k": 512, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/kv_b_proj" + ] }, { "type": "gemm", "args": { - "m": 1259520, - "n": 576, - "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 30720, + "n": 3072, + "k": 1536, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/Kimi-K2.5-MXFP4/q_b_proj", + "amd/MiniMax-M2.5-MXFP4/o_proj", + "moonshotai/Kimi-K2.5/q_b_proj", + "nvidia/DeepSeek-R1-0528-FP4-V2/q_b_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/q_b_proj", + "nvidia/Kimi-K2.5-NVFP4/q_b_proj", + "nvidia/Kimi-K2.6-NVFP4/q_b_proj", + "nvidia/MiniMax-M2.5-NVFP4/o_proj" + ] }, { "type": "gemm", "args": { - "m": 1259520, - "n": 16384, - "k": 512, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 30720, + "n": 3072, + "k": 1536, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/MiniMax-M2.5-NVFP4/experts.down_proj" + ] }, { "type": "gemm", "args": { - "m": 1259520, - "n": 7168, - "k": 8192, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 30720, + "n": 3072, + "k": 1536, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 32 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 1, + 32 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "amd/DeepSeek-R1-0528-MXFP4-Preview/q_b_proj", + "amd/DeepSeek-R1-0528-MXFP4-v2/q_b_proj", + "amd/DeepSeek-R1-0528-MXFP4/q_b_proj", + "amd/MiniMax-M2.5-MXFP4/experts.down_proj" + ] }, { "type": "gemm", "args": { - "m": 1259520, - "n": 4608, - "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 30720, + "n": 4096, + "k": 512, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "Qwen/Qwen3.5-397B-A17B/o_proj", + "Qwen/Qwen3.5-397B-A17B/shared_expert.down_proj", + "amd/Kimi-K2.5-MXFP4/kv_b_proj", + "amd/Qwen3.5-397B-A17B-MXFP4/o_proj", + "amd/Qwen3.5-397B-A17B-MXFP4/shared_expert.down_proj", + "moonshotai/Kimi-K2.5/kv_b_proj", + "nvidia/DeepSeek-R1-0528-FP4-V2/kv_b_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/kv_b_proj", + "nvidia/Kimi-K2.5-NVFP4/kv_b_proj", + "nvidia/Kimi-K2.6-NVFP4/kv_b_proj", + "nvidia/Qwen3.5-397B-A17B-NVFP4/o_proj", + "nvidia/Qwen3.5-397B-A17B-NVFP4/shared_expert.down_proj" + ] }, { "type": "gemm", "args": { - "m": 1259520, - "n": 7168, - "k": 4608, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 30720, + "n": 4096, + "k": 512, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 32 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 1, + 32 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "amd/DeepSeek-R1-0528-MXFP4-Preview/kv_b_proj", + "amd/DeepSeek-R1-0528-MXFP4-v2/kv_b_proj", + "amd/DeepSeek-R1-0528-MXFP4/kv_b_proj", + "amd/Qwen3.5-397B-A17B-MXFP4-AttnFP8-V2/shared_expert.down_proj" + ] }, { "type": "gemm", "args": { - "m": 153, - "n": 12288, - "k": 1536, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 30720, + "n": 4608, + "k": 7168, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/gate_up_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/gate_up_proj", + "nvidia/Kimi-K2.5-NVFP4/gate_up_proj" + ] }, { "type": "gemm", "args": { - "m": 153, - "n": 16384, - "k": 512, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 30720, + "n": 4608, + "k": 7168, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 32 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 1, + 32 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "amd/DeepSeek-R1-0528-MXFP4-Preview/gate_up_proj", + "amd/DeepSeek-R1-0528-MXFP4-v2/gate_up_proj", + "amd/DeepSeek-R1-0528-MXFP4/gate_up_proj", + "amd/Kimi-K2.5-MXFP4/gate_up_proj" + ] }, { "type": "gemm", "args": { - "m": 153, + "m": 30720, "n": 7168, - "k": 8192, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "k": 1024, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/o_proj", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] }, { "type": "gemm", "args": { - "m": 153, - "n": 4608, - "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 30720, + "n": 7168, + "k": 1152, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/down_proj" + ] }, { "type": "gemm", "args": { - "m": 153, + "m": 30720, "n": 7168, - "k": 4608, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "k": 2048, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/Kimi-K2.5-MXFP4/o_proj", + "moonshotai/Kimi-K2.5/o_proj", + "nvidia/Kimi-K2.5-NVFP4/o_proj", + "nvidia/Kimi-K2.6-NVFP4/o_proj" + ] }, { "type": "gemm", "args": { - "m": 314880, - "n": 1536, - "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 30720, + "n": 7168, + "k": 2048, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/o_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/o_proj" + ] }, { "type": "gemm", "args": { - "m": 314880, - "n": 12288, - "k": 1536, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 30720, + "n": 7168, + "k": 2048, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 32 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 1, + 32 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "amd/DeepSeek-R1-0528-MXFP4-Preview/o_proj", + "amd/DeepSeek-R1-0528-MXFP4-v2/o_proj", + "amd/DeepSeek-R1-0528-MXFP4/o_proj" + ] }, { "type": "gemm", "args": { - "m": 314880, - "n": 576, - "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 30720, + "n": 7168, + "k": 4608, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/down_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/down_proj", + "nvidia/Kimi-K2.5-NVFP4/down_proj" + ] }, { "type": "gemm", "args": { - "m": 314880, - "n": 16384, - "k": 512, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 30720, + "n": 7168, + "k": 8192, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/Kimi-K2.5-MXFP4/o_proj", + "moonshotai/Kimi-K2.5/o_proj", + "nvidia/Kimi-K2.5-NVFP4/o_proj", + "nvidia/Kimi-K2.6-NVFP4/o_proj" + ] }, { "type": "gemm", "args": { - "m": 314880, + "m": 30720, "n": 7168, - "k": 8192, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "k": 9216, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/down_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/down_proj", + "nvidia/Kimi-K2.5-NVFP4/down_proj" + ] }, { "type": "gemm", "args": { - "m": 314880, - "n": 1152, - "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 30720, + "n": 7168, + "k": 16384, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/o_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/o_proj" + ] }, { "type": "gemm", "args": { - "m": 314880, - "n": 7168, - "k": 1152, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 30720, + "n": 9216, + "k": 7168, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/gate_up_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/gate_up_proj", + "nvidia/Kimi-K2.5-NVFP4/gate_up_proj" + ] }, { "type": "gemm", "args": { - "m": 38, + "m": 30720, "n": 12288, "k": 1536, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/Kimi-K2.5-MXFP4/q_b_proj", + "moonshotai/Kimi-K2.5/q_b_proj", + "nvidia/DeepSeek-R1-0528-FP4-V2/q_b_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/q_b_proj", + "nvidia/Kimi-K2.5-NVFP4/q_b_proj", + "nvidia/Kimi-K2.6-NVFP4/q_b_proj" + ] }, { "type": "gemm", "args": { - "m": 38, + "m": 30720, "n": 16384, "k": 512, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/Kimi-K2.5-MXFP4/kv_b_proj", + "moonshotai/Kimi-K2.5/kv_b_proj", + "nvidia/DeepSeek-R1-0528-FP4-V2/kv_b_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/kv_b_proj", + "nvidia/Kimi-K2.5-NVFP4/kv_b_proj", + "nvidia/Kimi-K2.6-NVFP4/kv_b_proj" + ] }, { "type": "gemm", "args": { - "m": 38, - "n": 7168, - "k": 8192, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 30720, + "n": 24576, + "k": 1536, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/q_b_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/q_b_proj" + ] }, { "type": "gemm", "args": { - "m": 38, - "n": 1152, - "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 30720, + "n": 32768, + "k": 512, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/kv_b_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/kv_b_proj" + ] }, { "type": "gemm", "args": { - "m": 38, - "n": 7168, - "k": 1152, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 32768, + "n": 64, + "k": 2880, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/gpt-oss-120b-w-mxfp4-a-fp8/k_proj", + "amd/gpt-oss-120b-w-mxfp4-a-fp8/v_proj", + "openai/gpt-oss-120b/k_proj", + "openai/gpt-oss-120b/v_proj" + ] }, { "type": "gemm", "args": { - "m": 4405248, - "n": 1536, - "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 32768, + "n": 128, + "k": 2880, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/gpt-oss-120b-w-mxfp4-a-fp8/k_proj", + "amd/gpt-oss-120b-w-mxfp4-a-fp8/v_proj", + "openai/gpt-oss-120b/k_proj", + "openai/gpt-oss-120b/v_proj" + ] }, { "type": "gemm", "args": { - "m": 4405248, - "n": 12288, - "k": 1536, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 32768, + "n": 128, + "k": 3072, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/MiniMax-M2.5-MXFP4/k_proj", + "amd/MiniMax-M2.5-MXFP4/v_proj", + "nvidia/MiniMax-M2.5-NVFP4/k_proj", + "nvidia/MiniMax-M2.5-NVFP4/v_proj" + ] }, { "type": "gemm", "args": { - "m": 4405248, - "n": 576, - "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 32768, + "n": 128, + "k": 3072, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "MiniMaxAI/MiniMax-M2.5/k_proj", + "MiniMaxAI/MiniMax-M2.5/v_proj" + ] }, { "type": "gemm", "args": { - "m": 4405248, - "n": 16384, - "k": 512, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 32768, + "n": 128, + "k": 4096, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "Qwen/Qwen3.5-397B-A17B/shared_expert.gate_up_proj", + "amd/Qwen3.5-397B-A17B-MXFP4/shared_expert.gate_up_proj", + "nvidia/Qwen3.5-397B-A17B-NVFP4/shared_expert.gate_up_proj" + ] }, { "type": "gemm", "args": { - "m": 4405248, - "n": 7168, - "k": 8192, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 32768, + "n": 128, + "k": 4096, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "Qwen/Qwen3.5-397B-A17B-FP8/shared_expert.gate_up_proj" + ] }, { "type": "gemm", "args": { - "m": 4405248, - "n": 4608, - "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 32768, + "n": 192, + "k": 3072, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/MiniMax-M2.5-NVFP4/experts.gate_up_proj" + ] }, { "type": "gemm", "args": { - "m": 4405248, - "n": 7168, - "k": 4608, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 32768, + "n": 192, + "k": 3072, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 32 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 1, + 32 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "amd/MiniMax-M2.5-MXFP4/experts.gate_up_proj" + ] }, { "type": "gemm", "args": { - "m": 537, - "n": 4608, - "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 32768, + "n": 192, + "k": 3072, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "MiniMaxAI/MiniMax-M2.5/experts.gate_up_proj" + ] }, { "type": "gemm", "args": { - "m": 537, - "n": 7168, - "k": 4608, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 32768, + "n": 256, + "k": 2880, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/gpt-oss-120b-w-mxfp4-a-fp8/k_proj", + "amd/gpt-oss-120b-w-mxfp4-a-fp8/q_proj", + "amd/gpt-oss-120b-w-mxfp4-a-fp8/v_proj", + "openai/gpt-oss-120b/k_proj", + "openai/gpt-oss-120b/q_proj", + "openai/gpt-oss-120b/v_proj" + ] }, { "type": "gemm", "args": { - "m": 2202624, - "n": 1536, - "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 32768, + "n": 256, + "k": 3072, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/MiniMax-M2.5-MXFP4/k_proj", + "amd/MiniMax-M2.5-MXFP4/v_proj", + "nvidia/MiniMax-M2.5-NVFP4/k_proj", + "nvidia/MiniMax-M2.5-NVFP4/v_proj" + ] }, { "type": "gemm", "args": { - "m": 2202624, - "n": 12288, - "k": 1536, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 32768, + "n": 256, + "k": 3072, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "MiniMaxAI/MiniMax-M2.5/k_proj", + "MiniMaxAI/MiniMax-M2.5/v_proj" + ] }, { "type": "gemm", "args": { - "m": 2202624, - "n": 576, - "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 32768, + "n": 256, + "k": 4096, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "Qwen/Qwen3.5-397B-A17B/k_proj", + "Qwen/Qwen3.5-397B-A17B/shared_expert.gate_up_proj", + "Qwen/Qwen3.5-397B-A17B/v_proj", + "amd/Qwen3.5-397B-A17B-MXFP4/k_proj", + "amd/Qwen3.5-397B-A17B-MXFP4/shared_expert.gate_up_proj", + "amd/Qwen3.5-397B-A17B-MXFP4/v_proj", + "nvidia/Qwen3.5-397B-A17B-NVFP4/k_proj", + "nvidia/Qwen3.5-397B-A17B-NVFP4/shared_expert.gate_up_proj", + "nvidia/Qwen3.5-397B-A17B-NVFP4/v_proj" + ] }, { "type": "gemm", "args": { - "m": 2202624, - "n": 16384, - "k": 512, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 32768, + "n": 256, + "k": 4096, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "Qwen/Qwen3.5-397B-A17B-FP8/k_proj", + "Qwen/Qwen3.5-397B-A17B-FP8/shared_expert.gate_up_proj", + "Qwen/Qwen3.5-397B-A17B-FP8/v_proj" + ] }, { "type": "gemm", "args": { - "m": 2202624, - "n": 7168, - "k": 8192, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 32768, + "n": 384, + "k": 3072, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/MiniMax-M2.5-NVFP4/experts.gate_up_proj" + ] }, { "type": "gemm", "args": { - "m": 2202624, - "n": 2304, - "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 32768, + "n": 384, + "k": 3072, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 32 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 1, + 32 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "amd/MiniMax-M2.5-MXFP4/experts.gate_up_proj" + ] }, { "type": "gemm", "args": { - "m": 2202624, - "n": 7168, - "k": 2304, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 32768, + "n": 384, + "k": 3072, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "MiniMaxAI/MiniMax-M2.5/experts.gate_up_proj", + "MiniMaxAI/MiniMax-M2.5/q_proj" + ] }, { "type": "gemm", "args": { - "m": 268, - "n": 2304, - "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 32768, + "n": 512, + "k": 2880, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/gpt-oss-120b-w-mxfp4-a-fp8/k_proj", + "amd/gpt-oss-120b-w-mxfp4-a-fp8/q_proj", + "amd/gpt-oss-120b-w-mxfp4-a-fp8/v_proj", + "openai/gpt-oss-120b/k_proj", + "openai/gpt-oss-120b/q_proj", + "openai/gpt-oss-120b/v_proj" + ] }, { "type": "gemm", "args": { - "m": 268, - "n": 7168, - "k": 2304, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 32768, + "n": 512, + "k": 3072, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/MiniMax-M2.5-MXFP4/k_proj", + "amd/MiniMax-M2.5-MXFP4/v_proj", + "nvidia/MiniMax-M2.5-NVFP4/k_proj", + "nvidia/MiniMax-M2.5-NVFP4/v_proj" + ] }, { "type": "gemm", "args": { - "m": 4614144, - "n": 12288, - "k": 1536, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 32768, + "n": 512, + "k": 3072, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "MiniMaxAI/MiniMax-M2.5/k_proj", + "MiniMaxAI/MiniMax-M2.5/v_proj" + ] }, { "type": "gemm", "args": { - "m": 4614144, - "n": 16384, - "k": 512, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 32768, + "n": 512, + "k": 4096, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "Qwen/Qwen3.5-397B-A17B/k_proj", + "Qwen/Qwen3.5-397B-A17B/q_proj", + "Qwen/Qwen3.5-397B-A17B/shared_expert.gate_up_proj", + "Qwen/Qwen3.5-397B-A17B/v_proj", + "amd/Qwen3.5-397B-A17B-MXFP4/k_proj", + "amd/Qwen3.5-397B-A17B-MXFP4/q_proj", + "amd/Qwen3.5-397B-A17B-MXFP4/shared_expert.gate_up_proj", + "amd/Qwen3.5-397B-A17B-MXFP4/v_proj", + "nvidia/Qwen3.5-397B-A17B-NVFP4/k_proj", + "nvidia/Qwen3.5-397B-A17B-NVFP4/q_proj", + "nvidia/Qwen3.5-397B-A17B-NVFP4/shared_expert.gate_up_proj", + "nvidia/Qwen3.5-397B-A17B-NVFP4/v_proj" + ] }, { "type": "gemm", "args": { - "m": 4614144, - "n": 7168, - "k": 8192, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 32768, + "n": 512, + "k": 4096, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "Qwen/Qwen3.5-397B-A17B-FP8/k_proj", + "Qwen/Qwen3.5-397B-A17B-FP8/q_proj", + "Qwen/Qwen3.5-397B-A17B-FP8/shared_expert.gate_up_proj", + "Qwen/Qwen3.5-397B-A17B-FP8/v_proj", + "deepseek-ai/DeepSeek-V4.1-Flash/attn.wo_a" + ] }, { "type": "gemm", "args": { - "m": 1153536, - "n": 12288, - "k": 1536, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 32768, + "n": 576, + "k": 6144, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/GLM-5.1-MXFP4/kv_a_proj_with_mqa", + "amd/GLM-5.2-MXFP4/kv_a_proj_with_mqa", + "nvidia/GLM-5-NVFP4/kv_a_proj_with_mqa", + "nvidia/GLM-5.1-NVFP4/kv_a_proj_with_mqa", + "nvidia/GLM-5.2-NVFP4/kv_a_proj_with_mqa" + ] }, { "type": "gemm", "args": { - "m": 1153536, - "n": 16384, - "k": 512, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 32768, + "n": 576, + "k": 6144, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "zai-org/GLM-5-FP8/kv_a_proj_with_mqa", + "zai-org/GLM-5.1-FP8/kv_a_proj_with_mqa", + "zai-org/GLM-5.2-FP8/kv_a_proj_with_mqa" + ] }, { "type": "gemm", "args": { - "m": 1153536, - "n": 7168, - "k": 8192, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 32768, + "n": 576, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/Kimi-K2.5-MXFP4/kv_a_proj_with_mqa", + "moonshotai/Kimi-K2.5/kv_a_proj_with_mqa", + "nvidia/DeepSeek-R1-0528-FP4-V2/kv_a_proj_with_mqa", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/kv_a_proj_with_mqa", + "nvidia/Kimi-K2.5-NVFP4/kv_a_proj_with_mqa", + "nvidia/Kimi-K2.6-NVFP4/kv_a_proj_with_mqa" + ] }, { "type": "gemm", "args": { - "m": 140, - "n": 12288, - "k": 1536, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 32768, + "n": 576, + "k": 7168, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 32 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 1, + 32 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "amd/DeepSeek-R1-0528-MXFP4-Preview/kv_a_proj_with_mqa", + "amd/DeepSeek-R1-0528-MXFP4-v2/kv_a_proj_with_mqa", + "amd/DeepSeek-R1-0528-MXFP4/kv_a_proj_with_mqa" + ] }, { "type": "gemm", "args": { - "m": 140, - "n": 16384, - "k": 512, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 32768, + "n": 576, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/kv_a_proj_with_mqa" + ] }, { "type": "gemm", "args": { - "m": 140, - "n": 7168, - "k": 8192, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 32768, + "n": 768, + "k": 3072, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/MiniMax-M2.5-NVFP4/experts.gate_up_proj" + ] }, { "type": "gemm", "args": { - "m": 65536, - "n": 12288, - "k": 1536, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 32768, + "n": 768, + "k": 3072, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 32 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 1, + 32 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "amd/MiniMax-M2.5-MXFP4/experts.gate_up_proj" + ] }, { "type": "gemm", "args": { - "m": 65536, - "n": 16384, - "k": 512, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 32768, + "n": 768, + "k": 3072, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "MiniMaxAI/MiniMax-M2.5/experts.gate_up_proj", + "MiniMaxAI/MiniMax-M2.5/q_proj" + ] }, { "type": "gemm", "args": { - "m": 64, - "n": 12288, - "k": 1536, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 32768, + "n": 1024, + "k": 2880, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/gpt-oss-120b-w-mxfp4-a-fp8/q_proj", + "openai/gpt-oss-120b/q_proj" + ] }, { "type": "gemm", "args": { - "m": 64, - "n": 16384, - "k": 512, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 32768, + "n": 1024, + "k": 3072, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/MiniMax-M2.5-MXFP4/k_proj", + "amd/MiniMax-M2.5-MXFP4/qkv_proj", + "amd/MiniMax-M2.5-MXFP4/v_proj", + "nvidia/MiniMax-M2.5-NVFP4/k_proj", + "nvidia/MiniMax-M2.5-NVFP4/qkv_proj", + "nvidia/MiniMax-M2.5-NVFP4/v_proj" + ] }, { "type": "gemm", "args": { - "m": 131072, - "n": 12288, - "k": 1536, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 32768, + "n": 1024, + "k": 3072, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "MiniMaxAI/MiniMax-M2.5/k_proj", + "MiniMaxAI/MiniMax-M2.5/qkv_proj", + "MiniMaxAI/MiniMax-M2.5/v_proj" + ] }, { "type": "gemm", "args": { - "m": 131072, - "n": 16384, - "k": 512, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 32768, + "n": 1024, + "k": 4096, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "Qwen/Qwen3.5-397B-A17B/q_proj", + "Qwen/Qwen3.5-397B-A17B/shared_expert.gate_up_proj", + "amd/Qwen3.5-397B-A17B-MXFP4/q_proj", + "amd/Qwen3.5-397B-A17B-MXFP4/shared_expert.gate_up_proj", + "nvidia/Qwen3.5-397B-A17B-NVFP4/q_proj", + "nvidia/Qwen3.5-397B-A17B-NVFP4/shared_expert.gate_up_proj" + ] }, { "type": "gemm", "args": { - "m": 128, - "n": 12288, - "k": 1536, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 32768, + "n": 1024, + "k": 4096, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "Qwen/Qwen3.5-397B-A17B-FP8/q_proj", + "Qwen/Qwen3.5-397B-A17B-FP8/shared_expert.gate_up_proj", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_a", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_a", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_a" + ] }, { "type": "gemm", "args": { - "m": 128, - "n": 16384, - "k": 512, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 32768, + "n": 1088, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] }, { "type": "gemm", "args": { - "m": 262144, - "n": 12288, - "k": 1536, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 32768, + "n": 1152, + "k": 7168, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-NVFP4-v2/gate_up_proj" + ] }, { "type": "gemm", "args": { - "m": 262144, - "n": 16384, - "k": 512, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 32768, + "n": 1152, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/gate_up_proj" + ] }, { "type": "gemm", "args": { - "m": 256, - "n": 12288, + "m": 32768, + "n": 1536, "k": 1536, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/Kimi-K2.5-MXFP4/q_b_proj", + "moonshotai/Kimi-K2.5/q_b_proj", + "nvidia/DeepSeek-R1-0528-FP4-V2/q_b_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/q_b_proj", + "nvidia/Kimi-K2.5-NVFP4/q_b_proj", + "nvidia/Kimi-K2.6-NVFP4/q_b_proj" + ] }, { "type": "gemm", "args": { - "m": 256, - "n": 16384, - "k": 512, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 32768, + "n": 1536, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/q_b_proj" + ] }, { "type": "gemm", "args": { - "m": 524288, - "n": 12288, - "k": 1536, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 32768, + "n": 1536, + "k": 3072, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/MiniMax-M2.5-NVFP4/experts.gate_up_proj" + ] }, { "type": "gemm", "args": { - "m": 524288, - "n": 16384, - "k": 512, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 32768, + "n": 1536, + "k": 3072, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 32 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 1, + 32 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "amd/MiniMax-M2.5-MXFP4/experts.gate_up_proj" + ] }, { "type": "gemm", "args": { - "m": 512, - "n": 12288, - "k": 1536, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 32768, + "n": 1536, + "k": 3072, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "MiniMaxAI/MiniMax-M2.5/experts.gate_up_proj", + "MiniMaxAI/MiniMax-M2.5/q_proj" + ] }, { "type": "gemm", "args": { - "m": 512, - "n": 16384, - "k": 512, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 32768, + "n": 1536, + "k": 6144, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/GLM-5-NVFP4/gate_up_proj" + ] }, { "type": "gemm", "args": { - "m": 512, - "n": 7168, - "k": 8192, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 32768, + "n": 1536, + "k": 6144, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "zai-org/GLM-5-FP8/gate_up_proj", + "zai-org/GLM-5.1-FP8/gate_up_proj", + "zai-org/GLM-5.2-FP8/gate_up_proj" + ] }, { "type": "gemm", "args": { - "m": 786432, + "m": 32768, "n": 1536, "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/Kimi-K2.5-MXFP4/q_a_proj", + "moonshotai/Kimi-K2.5/q_a_proj", + "nvidia/DeepSeek-R1-0528-FP4-V2/q_a_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/q_a_proj", + "nvidia/Kimi-K2.5-NVFP4/q_a_proj", + "nvidia/Kimi-K2.6-NVFP4/q_a_proj" + ] }, { "type": "gemm", "args": { - "m": 786432, - "n": 12288, - "k": 1536, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 32768, + "n": 1536, + "k": 7168, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 32 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 1, + 32 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "amd/DeepSeek-R1-0528-MXFP4-Preview/q_a_proj", + "amd/DeepSeek-R1-0528-MXFP4-v2/q_a_proj", + "amd/DeepSeek-R1-0528-MXFP4/q_a_proj" + ] }, { "type": "gemm", "args": { - "m": 786432, - "n": 576, + "m": 32768, + "n": 1536, "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/gate_up_proj", + "deepseek-ai/DeepSeek-R1-0528/q_a_proj", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w3" + ] }, { "type": "gemm", "args": { - "m": 786432, - "n": 16384, - "k": 512, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 32768, + "n": 1536, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w3" + ] }, { "type": "gemm", "args": { - "m": 786432, - "n": 7168, - "k": 8192, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 32768, + "n": 2048, + "k": 512, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/Kimi-K2.5-MXFP4/kv_b_proj", + "moonshotai/Kimi-K2.5/kv_b_proj", + "nvidia/DeepSeek-R1-0528-FP4-V2/kv_b_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/kv_b_proj", + "nvidia/Kimi-K2.5-NVFP4/kv_b_proj", + "nvidia/Kimi-K2.6-NVFP4/kv_b_proj" + ] }, { "type": "gemm", "args": { - "m": 786432, - "n": 4608, - "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 32768, + "n": 2048, + "k": 512, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/kv_b_proj" + ] }, { "type": "gemm", "args": { - "m": 786432, - "n": 7168, - "k": 4608, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 32768, + "n": 2048, + "k": 2048, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/GLM-5.1-MXFP4/q_b_proj", + "amd/GLM-5.2-MXFP4/q_b_proj", + "nvidia/GLM-5-NVFP4/q_b_proj", + "nvidia/GLM-5.1-NVFP4/q_b_proj", + "nvidia/GLM-5.2-NVFP4/q_b_proj" + ] }, { "type": "gemm", "args": { - "m": 768, - "n": 1536, - "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 32768, + "n": 2048, + "k": 2048, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "zai-org/GLM-5-FP8/q_b_proj", + "zai-org/GLM-5.1-FP8/q_b_proj", + "zai-org/GLM-5.2-FP8/q_b_proj" + ] }, { "type": "gemm", "args": { - "m": 768, - "n": 12288, - "k": 1536, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 32768, + "n": 2048, + "k": 2880, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/gpt-oss-120b-w-mxfp4-a-fp8/q_proj", + "openai/gpt-oss-120b/q_proj" + ] }, { "type": "gemm", "args": { - "m": 768, - "n": 576, - "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 32768, + "n": 2048, + "k": 4096, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "Qwen/Qwen3.5-397B-A17B/q_proj", + "Qwen/Qwen3.5-397B-A17B/shared_expert.gate_up_proj", + "amd/Qwen3.5-397B-A17B-MXFP4/q_proj", + "amd/Qwen3.5-397B-A17B-MXFP4/shared_expert.gate_up_proj", + "nvidia/Qwen3.5-397B-A17B-NVFP4/q_proj", + "nvidia/Qwen3.5-397B-A17B-NVFP4/shared_expert.gate_up_proj" + ] }, { "type": "gemm", "args": { - "m": 768, - "n": 16384, - "k": 512, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 32768, + "n": 2048, + "k": 4096, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "Qwen/Qwen3.5-397B-A17B-FP8/q_proj", + "Qwen/Qwen3.5-397B-A17B-FP8/shared_expert.gate_up_proj" + ] }, { "type": "gemm", "args": { - "m": 768, - "n": 7168, - "k": 8192, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 32768, + "n": 2048, + "k": 6144, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/GLM-5.1-MXFP4/q_a_proj", + "amd/GLM-5.2-MXFP4/q_a_proj", + "amd/MiniMax-M3-MXFP4/q_proj", + "nvidia/GLM-5-NVFP4/q_a_proj", + "nvidia/GLM-5.1-NVFP4/q_a_proj", + "nvidia/GLM-5.2-NVFP4/q_a_proj" + ] }, { "type": "gemm", "args": { - "m": 768, - "n": 4608, - "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 32768, + "n": 2048, + "k": 6144, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "zai-org/GLM-5-FP8/q_a_proj", + "zai-org/GLM-5.1-FP8/q_a_proj", + "zai-org/GLM-5.2-FP8/q_a_proj" + ] }, { "type": "gemm", "args": { - "m": 768, - "n": 7168, - "k": 4608, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 32768, + "n": 2304, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/gate_up_proj" + ] }, { "type": "gemm", "args": { - "m": 1048576, - "n": 12288, - "k": 1536, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 32768, + "n": 2880, + "k": 512, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/gpt-oss-120b-w-mxfp4-a-fp8/o_proj", + "openai/gpt-oss-120b/o_proj" + ] }, { "type": "gemm", "args": { - "m": 1048576, - "n": 16384, - "k": 512, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 32768, + "n": 2880, + "k": 1024, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/gpt-oss-120b-w-mxfp4-a-fp8/o_proj", + "openai/gpt-oss-120b/o_proj" + ] }, { "type": "gemm", "args": { - "m": 1048576, - "n": 7168, - "k": 8192, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 32768, + "n": 2880, + "k": 2048, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/gpt-oss-120b-w-mxfp4-a-fp8/o_proj", + "openai/gpt-oss-120b/o_proj" + ] }, { "type": "gemm", "args": { - "m": 16384, - "n": 12288, - "k": 1536, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 32768, + "n": 2880, + "k": 4096, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/gpt-oss-120b-w-mxfp4-a-fp8/o_proj", + "openai/gpt-oss-120b/o_proj" + ] }, { "type": "gemm", "args": { - "m": 16384, - "n": 16384, - "k": 512, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 32768, + "n": 3072, + "k": 192, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/MiniMax-M2.5-NVFP4/experts.down_proj" + ] }, { "type": "gemm", "args": { - "m": 16384, - "n": 1152, - "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 32768, + "n": 3072, + "k": 192, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 32 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 1, + 32 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "amd/MiniMax-M2.5-MXFP4/experts.down_proj" + ] }, { "type": "gemm", "args": { - "m": 16384, - "n": 7168, - "k": 1152, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 32768, + "n": 3072, + "k": 192, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "MiniMaxAI/MiniMax-M2.5/experts.down_proj" + ] }, { "type": "gemm", "args": { - "m": 16, - "n": 12288, - "k": 1536, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 32768, + "n": 3072, + "k": 384, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/MiniMax-M2.5-NVFP4/experts.down_proj" + ] }, { "type": "gemm", "args": { - "m": 16, - "n": 16384, - "k": 512, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 32768, + "n": 3072, + "k": 384, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 32 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 1, + 32 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "amd/MiniMax-M2.5-MXFP4/experts.down_proj" + ] }, { "type": "gemm", "args": { - "m": 16, - "n": 1152, - "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 32768, + "n": 3072, + "k": 384, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "MiniMaxAI/MiniMax-M2.5/experts.down_proj", + "MiniMaxAI/MiniMax-M2.5/o_proj" + ] }, { "type": "gemm", "args": { - "m": 16, - "n": 7168, - "k": 1152, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 32768, + "n": 3072, + "k": 768, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/MiniMax-M2.5-MXFP4/o_proj", + "nvidia/MiniMax-M2.5-NVFP4/o_proj" + ] }, { "type": "gemm", "args": { "m": 32768, - "n": 12288, - "k": 1536, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "n": 3072, + "k": 768, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/MiniMax-M2.5-NVFP4/experts.down_proj" + ] }, { "type": "gemm", "args": { "m": 32768, - "n": 16384, - "k": 512, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "n": 3072, + "k": 768, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 32 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 1, + 32 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "amd/MiniMax-M2.5-MXFP4/experts.down_proj" + ] }, { "type": "gemm", "args": { "m": 32768, - "n": 1152, - "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "n": 3072, + "k": 768, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "MiniMaxAI/MiniMax-M2.5/experts.down_proj", + "MiniMaxAI/MiniMax-M2.5/o_proj" + ] }, { "type": "gemm", "args": { "m": 32768, - "n": 7168, - "k": 1152, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "n": 3072, + "k": 1536, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/Kimi-K2.5-MXFP4/q_b_proj", + "amd/MiniMax-M2.5-MXFP4/o_proj", + "moonshotai/Kimi-K2.5/q_b_proj", + "nvidia/DeepSeek-R1-0528-FP4-V2/q_b_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/q_b_proj", + "nvidia/Kimi-K2.5-NVFP4/q_b_proj", + "nvidia/Kimi-K2.6-NVFP4/q_b_proj", + "nvidia/MiniMax-M2.5-NVFP4/o_proj" + ] }, { "type": "gemm", "args": { - "m": 32, - "n": 12288, + "m": 32768, + "n": 3072, "k": 1536, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/MiniMax-M2.5-NVFP4/experts.down_proj" + ] }, { "type": "gemm", "args": { - "m": 32, - "n": 16384, - "k": 512, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 32768, + "n": 3072, + "k": 1536, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 32 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 1, + 32 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "amd/DeepSeek-R1-0528-MXFP4-Preview/q_b_proj", + "amd/DeepSeek-R1-0528-MXFP4-v2/q_b_proj", + "amd/DeepSeek-R1-0528-MXFP4/q_b_proj", + "amd/MiniMax-M2.5-MXFP4/experts.down_proj" + ] }, { "type": "gemm", "args": { - "m": 32, - "n": 1152, - "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 32768, + "n": 3072, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "MiniMaxAI/MiniMax-M2.5/experts.down_proj", + "MiniMaxAI/MiniMax-M2.5/o_proj", + "deepseek-ai/DeepSeek-R1-0528/q_b_proj" + ] }, { "type": "gemm", "args": { - "m": 32, - "n": 7168, - "k": 1152, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 32768, + "n": 3072, + "k": 3072, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/MiniMax-M2.5-MXFP4/o_proj", + "amd/MiniMax-M2.5-MXFP4/q_proj", + "nvidia/MiniMax-M2.5-NVFP4/o_proj", + "nvidia/MiniMax-M2.5-NVFP4/q_proj" + ] }, { "type": "gemm", "args": { - "m": 65536, - "n": 1152, - "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 32768, + "n": 3072, + "k": 3072, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/MiniMax-M2.5-NVFP4/experts.gate_up_proj" + ] }, { "type": "gemm", "args": { - "m": 65536, - "n": 7168, - "k": 1152, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 32768, + "n": 3072, + "k": 3072, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 32 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 1, + 32 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "amd/MiniMax-M2.5-MXFP4/experts.gate_up_proj" + ] }, { "type": "gemm", "args": { - "m": 64, - "n": 1152, - "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 32768, + "n": 3072, + "k": 3072, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "MiniMaxAI/MiniMax-M2.5/experts.gate_up_proj", + "MiniMaxAI/MiniMax-M2.5/o_proj", + "MiniMaxAI/MiniMax-M2.5/q_proj" + ] }, { "type": "gemm", "args": { - "m": 64, - "n": 7168, - "k": 1152, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 32768, + "n": 3072, + "k": 6144, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/GLM-5.1-MXFP4/gate_up_proj", + "amd/GLM-5.2-MXFP4/gate_up_proj", + "amd/MiniMax-M2.5-MXFP4/o_proj", + "amd/MiniMax-M3-MXFP4/gate_up_proj", + "nvidia/GLM-5.1-NVFP4/gate_up_proj", + "nvidia/GLM-5.2-NVFP4/gate_up_proj", + "nvidia/MiniMax-M2.5-NVFP4/o_proj" + ] }, { "type": "gemm", "args": { - "m": 131072, - "n": 1152, - "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 32768, + "n": 3072, + "k": 6144, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/GLM-5-NVFP4/gate_up_proj" + ] }, { "type": "gemm", "args": { - "m": 131072, - "n": 7168, - "k": 1152, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 32768, + "n": 3072, + "k": 6144, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "MiniMaxAI/MiniMax-M2.5/o_proj", + "zai-org/GLM-5-FP8/gate_up_proj", + "zai-org/GLM-5.1-FP8/gate_up_proj", + "zai-org/GLM-5.2-FP8/gate_up_proj" + ] }, { "type": "gemm", "args": { - "m": 128, - "n": 1152, - "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 32768, + "n": 3584, + "k": 512, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/GLM-5.1-MXFP4/kv_b_proj", + "amd/GLM-5.2-MXFP4/kv_b_proj", + "nvidia/GLM-5-NVFP4/kv_b_proj", + "nvidia/GLM-5.1-NVFP4/kv_b_proj", + "nvidia/GLM-5.2-NVFP4/kv_b_proj" + ] }, { "type": "gemm", "args": { - "m": 128, - "n": 7168, - "k": 1152, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 32768, + "n": 3584, + "k": 512, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "zai-org/GLM-5-FP8/kv_b_proj", + "zai-org/GLM-5.1-FP8/kv_b_proj", + "zai-org/GLM-5.2-FP8/kv_b_proj" + ] }, { "type": "gemm", "args": { - "m": 196608, - "n": 12288, - "k": 1536, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 32768, + "n": 4096, + "k": 128, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "Qwen/Qwen3.5-397B-A17B/shared_expert.down_proj", + "amd/Qwen3.5-397B-A17B-MXFP4/shared_expert.down_proj", + "nvidia/Qwen3.5-397B-A17B-NVFP4/shared_expert.down_proj" + ] }, { "type": "gemm", "args": { - "m": 196608, - "n": 16384, - "k": 512, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 32768, + "n": 4096, + "k": 128, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "Qwen/Qwen3.5-397B-A17B-FP8/shared_expert.down_proj" + ] }, { "type": "gemm", "args": { - "m": 196608, - "n": 7168, - "k": 8192, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 32768, + "n": 4096, + "k": 256, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "Qwen/Qwen3.5-397B-A17B/shared_expert.down_proj", + "amd/Qwen3.5-397B-A17B-MXFP4/shared_expert.down_proj", + "nvidia/Qwen3.5-397B-A17B-NVFP4/shared_expert.down_proj" + ] }, { "type": "gemm", "args": { - "m": 196608, - "n": 1152, - "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 32768, + "n": 4096, + "k": 256, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "Qwen/Qwen3.5-397B-A17B-FP8/shared_expert.down_proj" + ] }, { "type": "gemm", "args": { - "m": 196608, - "n": 7168, - "k": 1152, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 32768, + "n": 4096, + "k": 512, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "Qwen/Qwen3.5-397B-A17B/o_proj", + "Qwen/Qwen3.5-397B-A17B/shared_expert.down_proj", + "amd/Kimi-K2.5-MXFP4/kv_b_proj", + "amd/Qwen3.5-397B-A17B-MXFP4/o_proj", + "amd/Qwen3.5-397B-A17B-MXFP4/shared_expert.down_proj", + "moonshotai/Kimi-K2.5/kv_b_proj", + "nvidia/DeepSeek-R1-0528-FP4-V2/kv_b_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/kv_b_proj", + "nvidia/Kimi-K2.5-NVFP4/kv_b_proj", + "nvidia/Kimi-K2.6-NVFP4/kv_b_proj", + "nvidia/Qwen3.5-397B-A17B-NVFP4/o_proj", + "nvidia/Qwen3.5-397B-A17B-NVFP4/shared_expert.down_proj" + ] }, { "type": "gemm", "args": { - "m": 192, - "n": 12288, - "k": 1536, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 32768, + "n": 4096, + "k": 512, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 32 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 1, + 32 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "amd/DeepSeek-R1-0528-MXFP4-Preview/kv_b_proj", + "amd/DeepSeek-R1-0528-MXFP4-v2/kv_b_proj", + "amd/DeepSeek-R1-0528-MXFP4/kv_b_proj", + "amd/Qwen3.5-397B-A17B-MXFP4-AttnFP8-V2/shared_expert.down_proj" + ] }, { "type": "gemm", "args": { - "m": 192, - "n": 16384, + "m": 32768, + "n": 4096, "k": 512, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "Qwen/Qwen3.5-397B-A17B-FP8/o_proj", + "Qwen/Qwen3.5-397B-A17B-FP8/shared_expert.down_proj", + "deepseek-ai/DeepSeek-R1-0528/kv_b_proj" + ] }, { "type": "gemm", "args": { - "m": 192, - "n": 7168, - "k": 8192, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 32768, + "n": 4096, + "k": 1024, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "Qwen/Qwen3.5-397B-A17B/o_proj", + "Qwen/Qwen3.5-397B-A17B/shared_expert.down_proj", + "amd/Qwen3.5-397B-A17B-MXFP4/o_proj", + "amd/Qwen3.5-397B-A17B-MXFP4/shared_expert.down_proj", + "nvidia/Qwen3.5-397B-A17B-NVFP4/o_proj", + "nvidia/Qwen3.5-397B-A17B-NVFP4/shared_expert.down_proj" + ] }, { "type": "gemm", "args": { - "m": 192, - "n": 1152, - "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 32768, + "n": 4096, + "k": 1024, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "Qwen/Qwen3.5-397B-A17B-FP8/o_proj", + "Qwen/Qwen3.5-397B-A17B-FP8/shared_expert.down_proj" + ] }, { "type": "gemm", "args": { - "m": 192, - "n": 7168, - "k": 1152, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 32768, + "n": 4096, + "k": 2048, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "Qwen/Qwen3.5-397B-A17B/o_proj", + "amd/GLM-5.1-MXFP4/q_b_proj", + "amd/GLM-5.2-MXFP4/q_b_proj", + "amd/Qwen3.5-397B-A17B-MXFP4/o_proj", + "nvidia/GLM-5-NVFP4/q_b_proj", + "nvidia/GLM-5.1-NVFP4/q_b_proj", + "nvidia/GLM-5.2-NVFP4/q_b_proj", + "nvidia/Qwen3.5-397B-A17B-NVFP4/o_proj" + ] }, { "type": "gemm", "args": { - "m": 262144, - "n": 1152, - "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 32768, + "n": 4096, + "k": 2048, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "Qwen/Qwen3.5-397B-A17B-FP8/o_proj", + "zai-org/GLM-5-FP8/q_b_proj", + "zai-org/GLM-5.1-FP8/q_b_proj", + "zai-org/GLM-5.2-FP8/q_b_proj" + ] }, { "type": "gemm", "args": { - "m": 262144, - "n": 7168, - "k": 1152, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 32768, + "n": 4096, + "k": 2880, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/gpt-oss-120b-w-mxfp4-a-fp8/q_proj", + "openai/gpt-oss-120b/q_proj" + ] }, { "type": "gemm", "args": { - "m": 2048, - "n": 12288, - "k": 1536, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 32768, + "n": 4096, + "k": 4096, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "Qwen/Qwen3.5-397B-A17B/o_proj", + "Qwen/Qwen3.5-397B-A17B/q_proj", + "amd/Qwen3.5-397B-A17B-MXFP4/o_proj", + "amd/Qwen3.5-397B-A17B-MXFP4/q_proj", + "nvidia/Qwen3.5-397B-A17B-NVFP4/o_proj", + "nvidia/Qwen3.5-397B-A17B-NVFP4/q_proj" + ] }, { "type": "gemm", "args": { - "m": 2048, - "n": 16384, - "k": 512, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 32768, + "n": 4096, + "k": 4096, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "Qwen/Qwen3.5-397B-A17B-FP8/o_proj", + "Qwen/Qwen3.5-397B-A17B-FP8/q_proj" + ] }, { "type": "gemm", "args": { - "m": 2, - "n": 12288, - "k": 1536, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 32768, + "n": 4608, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "moonshotai/Kimi-K2.5/gate_up_proj", + "nvidia/Kimi-K2.6-NVFP4/gate_up_proj" + ] }, { "type": "gemm", "args": { - "m": 2, - "n": 16384, - "k": 512, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 32768, + "n": 4608, + "k": 7168, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/gate_up_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/gate_up_proj", + "nvidia/Kimi-K2.5-NVFP4/gate_up_proj" + ] }, { "type": "gemm", "args": { - "m": 4096, - "n": 12288, - "k": 1536, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 32768, + "n": 4608, + "k": 7168, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 32 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 1, + 32 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "amd/DeepSeek-R1-0528-MXFP4-Preview/gate_up_proj", + "amd/DeepSeek-R1-0528-MXFP4-v2/gate_up_proj", + "amd/DeepSeek-R1-0528-MXFP4/gate_up_proj", + "amd/Kimi-K2.5-MXFP4/gate_up_proj" + ] }, { "type": "gemm", "args": { - "m": 4096, - "n": 16384, - "k": 512, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 32768, + "n": 4608, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/gate_up_proj" + ] }, { "type": "gemm", "args": { - "m": 4, - "n": 12288, + "m": 32768, + "n": 6144, "k": 1536, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/GLM-5.1-MXFP4/down_proj", + "amd/GLM-5.2-MXFP4/down_proj", + "amd/Kimi-K2.5-MXFP4/q_b_proj", + "amd/MiniMax-M3-MXFP4/down_proj", + "moonshotai/Kimi-K2.5/q_b_proj", + "nvidia/DeepSeek-R1-0528-FP4-V2/q_b_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/q_b_proj", + "nvidia/GLM-5.1-NVFP4/down_proj", + "nvidia/GLM-5.2-NVFP4/down_proj", + "nvidia/Kimi-K2.5-NVFP4/q_b_proj", + "nvidia/Kimi-K2.6-NVFP4/q_b_proj" + ] }, { "type": "gemm", "args": { - "m": 4, - "n": 16384, - "k": 512, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 32768, + "n": 6144, + "k": 1536, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/GLM-5-NVFP4/down_proj" + ] }, { "type": "gemm", "args": { - "m": 8, - "n": 12288, + "m": 32768, + "n": 6144, "k": 1536, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 32 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 1, + 32 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "amd/DeepSeek-R1-0528-MXFP4-Preview/q_b_proj", + "amd/DeepSeek-R1-0528-MXFP4-v2/q_b_proj", + "amd/DeepSeek-R1-0528-MXFP4/q_b_proj" + ] }, { "type": "gemm", "args": { - "m": 8, - "n": 16384, - "k": 512, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 32768, + "n": 6144, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/q_b_proj", + "zai-org/GLM-5-FP8/down_proj", + "zai-org/GLM-5.1-FP8/down_proj", + "zai-org/GLM-5.2-FP8/down_proj" + ] }, { "type": "gemm", "args": { - "m": 2097152, - "n": 12288, - "k": 1536, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 32768, + "n": 6144, + "k": 2048, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/GLM-5.1-MXFP4/o_proj", + "amd/GLM-5.2-MXFP4/o_proj", + "amd/MiniMax-M3-MXFP4/o_proj", + "nvidia/GLM-5-NVFP4/o_proj", + "nvidia/GLM-5.1-NVFP4/o_proj", + "nvidia/GLM-5.2-NVFP4/o_proj" + ] }, { "type": "gemm", "args": { - "m": 2097152, - "n": 16384, - "k": 512, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 32768, + "n": 6144, + "k": 2048, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "zai-org/GLM-5-FP8/o_proj", + "zai-org/GLM-5.1-FP8/o_proj", + "zai-org/GLM-5.2-FP8/o_proj" + ] }, { "type": "gemm", "args": { - "m": 2097152, - "n": 7168, - "k": 8192, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 32768, + "n": 6144, + "k": 3072, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/GLM-5.1-MXFP4/down_proj", + "amd/GLM-5.2-MXFP4/down_proj", + "amd/MiniMax-M2.5-MXFP4/q_proj", + "amd/MiniMax-M3-MXFP4/down_proj", + "nvidia/GLM-5.1-NVFP4/down_proj", + "nvidia/GLM-5.2-NVFP4/down_proj", + "nvidia/MiniMax-M2.5-NVFP4/q_proj" + ] }, { "type": "gemm", "args": { - "m": 524288, - "n": 1152, - "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 32768, + "n": 6144, + "k": 3072, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/GLM-5-NVFP4/down_proj" + ] }, { "type": "gemm", "args": { - "m": 524288, - "n": 7168, - "k": 1152, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 32768, + "n": 6144, + "k": 3072, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "MiniMaxAI/MiniMax-M2.5/q_proj", + "zai-org/GLM-5-FP8/down_proj", + "zai-org/GLM-5.1-FP8/down_proj", + "zai-org/GLM-5.2-FP8/down_proj" + ] }, { "type": "gemm", "args": { - "m": 4194304, - "n": 12288, - "k": 1536, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 32768, + "n": 6144, + "k": 4096, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/GLM-5.1-MXFP4/o_proj", + "amd/GLM-5.2-MXFP4/o_proj", + "amd/MiniMax-M3-MXFP4/o_proj", + "nvidia/GLM-5-NVFP4/o_proj", + "nvidia/GLM-5.1-NVFP4/o_proj", + "nvidia/GLM-5.2-NVFP4/o_proj" + ] }, { "type": "gemm", "args": { - "m": 4194304, - "n": 16384, - "k": 512, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 32768, + "n": 6144, + "k": 4096, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "zai-org/GLM-5-FP8/o_proj", + "zai-org/GLM-5.1-FP8/o_proj", + "zai-org/GLM-5.2-FP8/o_proj" + ] }, { "type": "gemm", "args": { - "m": 4194304, - "n": 7168, - "k": 8192, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 32768, + "n": 6144, + "k": 6144, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/GLM-5.1-MXFP4/down_proj", + "amd/GLM-5.1-MXFP4/gate_up_proj", + "amd/GLM-5.2-MXFP4/down_proj", + "amd/GLM-5.2-MXFP4/gate_up_proj", + "amd/MiniMax-M3-MXFP4/gate_up_proj", + "nvidia/GLM-5.1-NVFP4/down_proj", + "nvidia/GLM-5.1-NVFP4/gate_up_proj", + "nvidia/GLM-5.2-NVFP4/down_proj", + "nvidia/GLM-5.2-NVFP4/gate_up_proj" + ] }, { "type": "gemm", "args": { - "m": 6291456, - "n": 1536, - "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 32768, + "n": 6144, + "k": 6144, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/GLM-5-NVFP4/down_proj", + "nvidia/GLM-5-NVFP4/gate_up_proj" + ] }, { "type": "gemm", "args": { - "m": 6291456, - "n": 12288, - "k": 1536, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 32768, + "n": 6144, + "k": 6144, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "zai-org/GLM-5-FP8/down_proj", + "zai-org/GLM-5-FP8/gate_up_proj", + "zai-org/GLM-5.1-FP8/down_proj", + "zai-org/GLM-5.1-FP8/gate_up_proj", + "zai-org/GLM-5.2-FP8/down_proj", + "zai-org/GLM-5.2-FP8/gate_up_proj" + ] }, { "type": "gemm", "args": { - "m": 6291456, - "n": 576, - "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 32768, + "n": 6144, + "k": 8192, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/GLM-5.1-MXFP4/o_proj", + "amd/GLM-5.2-MXFP4/o_proj", + "amd/MiniMax-M3-MXFP4/o_proj", + "nvidia/GLM-5-NVFP4/o_proj", + "nvidia/GLM-5.1-NVFP4/o_proj", + "nvidia/GLM-5.2-NVFP4/o_proj" + ] }, { "type": "gemm", "args": { - "m": 6291456, - "n": 16384, + "m": 32768, + "n": 7168, "k": 512, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/GLM-5.1-MXFP4/kv_b_proj", + "amd/GLM-5.2-MXFP4/kv_b_proj", + "amd/Kimi-K2.5-MXFP4/o_proj", + "moonshotai/Kimi-K2.5/o_proj", + "nvidia/GLM-5-NVFP4/kv_b_proj", + "nvidia/GLM-5.1-NVFP4/kv_b_proj", + "nvidia/GLM-5.2-NVFP4/kv_b_proj", + "nvidia/Kimi-K2.5-NVFP4/o_proj", + "nvidia/Kimi-K2.6-NVFP4/o_proj" + ] }, { "type": "gemm", "args": { - "m": 6291456, + "m": 32768, "n": 7168, - "k": 8192, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "k": 512, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "zai-org/GLM-5-FP8/kv_b_proj", + "zai-org/GLM-5.1-FP8/kv_b_proj", + "zai-org/GLM-5.2-FP8/kv_b_proj" + ] }, { "type": "gemm", "args": { - "m": 6291456, - "n": 4608, - "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 32768, + "n": 7168, + "k": 576, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/down_proj" + ] }, { "type": "gemm", "args": { - "m": 6291456, + "m": 32768, "n": 7168, - "k": 4608, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "k": 1024, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/Kimi-K2.5-MXFP4/o_proj", + "moonshotai/Kimi-K2.5/o_proj", + "nvidia/Kimi-K2.5-NVFP4/o_proj", + "nvidia/Kimi-K2.6-NVFP4/o_proj" + ] }, { "type": "gemm", "args": { - "m": 8388608, - "n": 12288, - "k": 1536, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 32768, + "n": 7168, + "k": 1024, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/o_proj", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] }, { "type": "gemm", "args": { - "m": 8388608, - "n": 16384, - "k": 512, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 32768, + "n": 7168, + "k": 1152, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/down_proj" + ] }, { "type": "gemm", "args": { - "m": 8388608, + "m": 32768, "n": 7168, - "k": 8192, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "k": 2048, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/Kimi-K2.5-MXFP4/o_proj", + "moonshotai/Kimi-K2.5/o_proj", + "nvidia/Kimi-K2.5-NVFP4/o_proj", + "nvidia/Kimi-K2.6-NVFP4/o_proj" + ] }, { "type": "gemm", "args": { - "m": 1572864, - "n": 12288, - "k": 1536, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 32768, + "n": 7168, + "k": 2048, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/o_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/o_proj" + ] }, { "type": "gemm", "args": { - "m": 1572864, - "n": 16384, - "k": 512, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 32768, + "n": 7168, + "k": 2048, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 32 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 1, + 32 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "amd/DeepSeek-R1-0528-MXFP4-Preview/o_proj", + "amd/DeepSeek-R1-0528-MXFP4-v2/o_proj", + "amd/DeepSeek-R1-0528-MXFP4/o_proj" + ] }, { "type": "gemm", "args": { - "m": 1572864, + "m": 32768, "n": 7168, - "k": 8192, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "k": 2048, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/o_proj", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] }, { "type": "gemm", "args": { - "m": 1572864, - "n": 1152, - "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 32768, + "n": 7168, + "k": 2304, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "moonshotai/Kimi-K2.5/down_proj", + "nvidia/Kimi-K2.6-NVFP4/down_proj" + ] }, { "type": "gemm", "args": { - "m": 1572864, + "m": 32768, "n": 7168, - "k": 1152, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "k": 2304, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/down_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/down_proj", + "nvidia/Kimi-K2.5-NVFP4/down_proj" + ] }, { "type": "gemm", "args": { - "m": 0, - "n": 65536, - "k": 1536, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 32768, + "n": 7168, + "k": 2304, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 32 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 1, + 32 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "amd/DeepSeek-R1-0528-MXFP4-Preview/down_proj", + "amd/DeepSeek-R1-0528-MXFP4-v2/down_proj", + "amd/DeepSeek-R1-0528-MXFP4/down_proj", + "amd/Kimi-K2.5-MXFP4/down_proj" + ] }, { "type": "gemm", "args": { - "m": 0, - "n": 1088, - "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 32768, + "n": 7168, + "k": 2304, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/down_proj" + ] }, { "type": "gemm", "args": { - "m": 0, - "n": 122880, - "k": 1024, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 32768, + "n": 7168, + "k": 4096, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/o_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/o_proj" + ] }, { "type": "gemm", "args": { - "m": 0, + "m": 32768, "n": 7168, - "k": 65536, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "k": 4096, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 32 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 1, + 32 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "amd/DeepSeek-R1-0528-MXFP4-Preview/o_proj", + "amd/DeepSeek-R1-0528-MXFP4-v2/o_proj", + "amd/DeepSeek-R1-0528-MXFP4/o_proj" + ] }, { "type": "gemm", "args": { - "m": 2, - "n": 65536, - "k": 1536, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 32768, + "n": 7168, + "k": 4096, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/o_proj", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] }, { "type": "gemm", "args": { - "m": 2, - "n": 122880, - "k": 1024, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 32768, + "n": 7168, + "k": 4608, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "moonshotai/Kimi-K2.5/down_proj", + "nvidia/Kimi-K2.6-NVFP4/down_proj" + ] }, { "type": "gemm", "args": { - "m": 2, + "m": 32768, "n": 7168, - "k": 65536, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "k": 4608, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/down_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/down_proj", + "nvidia/Kimi-K2.5-NVFP4/down_proj" + ] }, { "type": "gemm", "args": { - "m": 1048576, - "n": 8192, - "k": 1536, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 32768, + "n": 7168, + "k": 4608, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 32 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 1, + 32 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "amd/DeepSeek-R1-0528-MXFP4-Preview/down_proj", + "amd/DeepSeek-R1-0528-MXFP4-v2/down_proj", + "amd/DeepSeek-R1-0528-MXFP4/down_proj", + "amd/Kimi-K2.5-MXFP4/down_proj" + ] }, { "type": "gemm", "args": { - "m": 1048576, - "n": 15360, - "k": 1024, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 32768, + "n": 7168, + "k": 4608, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/down_proj" + ] }, { "type": "gemm", "args": { - "m": 8388608, - "n": 65536, - "k": 1536, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 32768, + "n": 7168, + "k": 8192, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] }, { "type": "gemm", "args": { - "m": 8388608, - "n": 1088, - "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 32768, + "n": 7168, + "k": 9216, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/down_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/down_proj", + "nvidia/Kimi-K2.5-NVFP4/down_proj" + ] }, { "type": "gemm", "args": { - "m": 8388608, - "n": 122880, - "k": 1024, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 32768, + "n": 7168, + "k": 16384, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] }, { "type": "gemm", "args": { - "m": 8388608, + "m": 32768, "n": 7168, "k": 65536, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] }, { "type": "gemm", "args": { - "m": 12641280, - "n": 1536, - "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 32768, + "n": 8192, + "k": 512, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/Kimi-K2.5-MXFP4/kv_b_proj", + "moonshotai/Kimi-K2.5/kv_b_proj", + "nvidia/DeepSeek-R1-0528-FP4-V2/kv_b_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/kv_b_proj", + "nvidia/Kimi-K2.5-NVFP4/kv_b_proj", + "nvidia/Kimi-K2.6-NVFP4/kv_b_proj" + ] }, { "type": "gemm", "args": { - "m": 12641280, - "n": 65536, - "k": 1536, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 32768, + "n": 8192, + "k": 512, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 32 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 1, + 32 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "amd/DeepSeek-R1-0528-MXFP4-Preview/kv_b_proj", + "amd/DeepSeek-R1-0528-MXFP4-v2/kv_b_proj", + "amd/DeepSeek-R1-0528-MXFP4/kv_b_proj" + ] }, { "type": "gemm", "args": { - "m": 12641280, - "n": 1088, - "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 32768, + "n": 8192, + "k": 512, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/kv_b_proj" + ] }, { "type": "gemm", "args": { - "m": 12641280, - "n": 122880, - "k": 1024, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 32768, + "n": 8192, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_b" + ] }, { "type": "gemm", "args": { - "m": 12641280, - "n": 7168, - "k": 65536, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 32768, + "n": 8192, + "k": 2048, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/GLM-5.1-MXFP4/q_b_proj", + "amd/GLM-5.2-MXFP4/q_b_proj", + "nvidia/GLM-5-NVFP4/q_b_proj", + "nvidia/GLM-5.1-NVFP4/q_b_proj", + "nvidia/GLM-5.2-NVFP4/q_b_proj" + ] }, { "type": "gemm", "args": { - "m": 1543, - "n": 1536, + "m": 32768, + "n": 9216, "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "moonshotai/Kimi-K2.5/gate_up_proj", + "nvidia/Kimi-K2.6-NVFP4/gate_up_proj" + ] }, { "type": "gemm", "args": { - "m": 1543, - "n": 65536, - "k": 1536, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 32768, + "n": 9216, + "k": 7168, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/gate_up_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/gate_up_proj", + "nvidia/Kimi-K2.5-NVFP4/gate_up_proj" + ] }, { "type": "gemm", "args": { - "m": 1543, - "n": 1088, + "m": 32768, + "n": 9216, "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 32 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 1, + 32 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "amd/DeepSeek-R1-0528-MXFP4-Preview/gate_up_proj", + "amd/DeepSeek-R1-0528-MXFP4-v2/gate_up_proj", + "amd/DeepSeek-R1-0528-MXFP4/gate_up_proj", + "amd/Kimi-K2.5-MXFP4/gate_up_proj" + ] }, { "type": "gemm", "args": { - "m": 1543, - "n": 122880, - "k": 1024, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 32768, + "n": 9216, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/gate_up_proj" + ] }, { "type": "gemm", "args": { - "m": 1543, - "n": 7168, - "k": 65536, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 32768, + "n": 12288, + "k": 1536, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/Kimi-K2.5-MXFP4/q_b_proj", + "moonshotai/Kimi-K2.5/q_b_proj", + "nvidia/DeepSeek-R1-0528-FP4-V2/q_b_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/q_b_proj", + "nvidia/Kimi-K2.5-NVFP4/q_b_proj", + "nvidia/Kimi-K2.6-NVFP4/q_b_proj" + ] }, { "type": "gemm", "args": { - "m": 2097152, - "n": 8192, - "k": 1536, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 32768, + "n": 14336, + "k": 512, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/GLM-5.1-MXFP4/kv_b_proj", + "amd/GLM-5.2-MXFP4/kv_b_proj", + "nvidia/GLM-5-NVFP4/kv_b_proj", + "nvidia/GLM-5.1-NVFP4/kv_b_proj", + "nvidia/GLM-5.2-NVFP4/kv_b_proj" + ] }, { "type": "gemm", "args": { - "m": 2097152, + "m": 32768, "n": 15360, "k": 1024, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] }, { "type": "gemm", "args": { - "m": 4194304, - "n": 8192, - "k": 1536, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 32768, + "n": 16384, + "k": 512, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/Kimi-K2.5-MXFP4/kv_b_proj", + "moonshotai/Kimi-K2.5/kv_b_proj", + "nvidia/DeepSeek-R1-0528-FP4-V2/kv_b_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/kv_b_proj", + "nvidia/Kimi-K2.5-NVFP4/kv_b_proj", + "nvidia/Kimi-K2.6-NVFP4/kv_b_proj" + ] }, { "type": "gemm", "args": { - "m": 4194304, - "n": 15360, - "k": 1024, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 32768, + "n": 16384, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_b" + ] }, { "type": "gemm", "args": { - "m": 512, - "n": 8192, + "m": 32768, + "n": 24576, "k": 1536, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/q_b_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/q_b_proj" + ] }, { "type": "gemm", "args": { - "m": 512, - "n": 15360, - "k": 1024, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 32768, + "n": 24576, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/q_b_proj" + ] }, { "type": "gemm", "args": { - "m": 393216, - "n": 65536, - "k": 1536, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 32768, + "n": 30720, + "k": 1024, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] }, { "type": "gemm", "args": { - "m": 393216, - "n": 1088, - "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 32768, + "n": 32768, + "k": 512, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/kv_b_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/kv_b_proj" + ] }, { "type": "gemm", "args": { - "m": 393216, - "n": 122880, - "k": 1024, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 32768, + "n": 32768, + "k": 512, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/kv_b_proj" + ] }, { "type": "gemm", "args": { - "m": 393216, - "n": 7168, - "k": 65536, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 32768, + "n": 65536, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_b" + ] }, { "type": "gemm", "args": { - "m": 48, - "n": 65536, - "k": 1536, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 32768, + "n": 122880, + "k": 1024, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] }, { "type": "gemm", "args": { - "m": 48, - "n": 1088, + "m": 34688, + "n": 576, "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/Kimi-K2.5-MXFP4/kv_a_proj_with_mqa", + "moonshotai/Kimi-K2.5/kv_a_proj_with_mqa", + "nvidia/DeepSeek-R1-0528-FP4-V2/kv_a_proj_with_mqa", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/kv_a_proj_with_mqa", + "nvidia/Kimi-K2.5-NVFP4/kv_a_proj_with_mqa", + "nvidia/Kimi-K2.6-NVFP4/kv_a_proj_with_mqa" + ] }, { "type": "gemm", "args": { - "m": 48, - "n": 122880, - "k": 1024, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 34688, + "n": 576, + "k": 7168, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 32 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 1, + 32 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "amd/DeepSeek-R1-0528-MXFP4-Preview/kv_a_proj_with_mqa", + "amd/DeepSeek-R1-0528-MXFP4-v2/kv_a_proj_with_mqa", + "amd/DeepSeek-R1-0528-MXFP4/kv_a_proj_with_mqa" + ] }, { "type": "gemm", "args": { - "m": 48, - "n": 7168, - "k": 65536, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 34688, + "n": 1536, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/Kimi-K2.5-MXFP4/q_a_proj", + "moonshotai/Kimi-K2.5/q_a_proj", + "nvidia/DeepSeek-R1-0528-FP4-V2/q_a_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/q_a_proj", + "nvidia/Kimi-K2.5-NVFP4/q_a_proj", + "nvidia/Kimi-K2.6-NVFP4/q_a_proj" + ] }, { "type": "gemm", "args": { - "m": 1572864, - "n": 16384, - "k": 1536, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 34688, + "n": 1536, + "k": 7168, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 32 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 1, + 32 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "amd/DeepSeek-R1-0528-MXFP4-Preview/q_a_proj", + "amd/DeepSeek-R1-0528-MXFP4-v2/q_a_proj", + "amd/DeepSeek-R1-0528-MXFP4/q_a_proj" + ] }, { "type": "gemm", "args": { - "m": 1572864, - "n": 1088, + "m": 34688, + "n": 1536, "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w3" + ] }, { "type": "gemm", "args": { - "m": 1572864, - "n": 30720, - "k": 1024, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 34688, + "n": 2304, + "k": 7168, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/gate_up_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/gate_up_proj", + "nvidia/Kimi-K2.5-NVFP4/gate_up_proj" + ] }, { "type": "gemm", "args": { - "m": 192, - "n": 16384, - "k": 1536, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 34688, + "n": 7168, + "k": 2304, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/down_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/down_proj", + "nvidia/Kimi-K2.5-NVFP4/down_proj" + ] }, { "type": "gemm", "args": { - "m": 192, - "n": 1088, - "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 34688, + "n": 7168, + "k": 2304, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 32 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 1, + 32 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "amd/DeepSeek-R1-0528-MXFP4-Preview/down_proj", + "amd/DeepSeek-R1-0528-MXFP4-v2/down_proj", + "amd/DeepSeek-R1-0528-MXFP4/down_proj", + "amd/Kimi-K2.5-MXFP4/down_proj" + ] }, { "type": "gemm", "args": { - "m": 192, - "n": 30720, - "k": 1024, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 34688, + "n": 7168, + "k": 16384, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/o_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/o_proj" + ] }, { "type": "gemm", "args": { - "m": 192, + "m": 34688, "n": 7168, "k": 16384, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 32 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 1, + 32 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "amd/DeepSeek-R1-0528-MXFP4-Preview/o_proj", + "amd/DeepSeek-R1-0528-MXFP4-v2/o_proj", + "amd/DeepSeek-R1-0528-MXFP4/o_proj" + ] }, { "type": "gemm", "args": { - "m": 147456, - "n": 1536, - "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 34688, + "n": 7168, + "k": 16384, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] }, { "type": "gemm", "args": { - "m": 147456, - "n": 16384, + "m": 34688, + "n": 24576, "k": 1536, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/q_b_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/q_b_proj" + ] }, { "type": "gemm", "args": { - "m": 147456, - "n": 1088, - "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 34688, + "n": 24576, + "k": 1536, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 32 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 1, + 32 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "amd/DeepSeek-R1-0528-MXFP4-Preview/q_b_proj", + "amd/DeepSeek-R1-0528-MXFP4-v2/q_b_proj", + "amd/DeepSeek-R1-0528-MXFP4/q_b_proj" + ] }, { "type": "gemm", "args": { - "m": 147456, - "n": 30720, - "k": 1024, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 34688, + "n": 32768, + "k": 512, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/kv_b_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/kv_b_proj" + ] }, { "type": "gemm", "args": { - "m": 147456, - "n": 7168, - "k": 16384, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 34688, + "n": 32768, + "k": 512, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 32 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 1, + 32 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "amd/DeepSeek-R1-0528-MXFP4-Preview/kv_b_proj", + "amd/DeepSeek-R1-0528-MXFP4-v2/kv_b_proj", + "amd/DeepSeek-R1-0528-MXFP4/kv_b_proj" + ] }, { "type": "gemm", "args": { - "m": 18, - "n": 1536, + "m": 36064, + "n": 576, "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/kv_a_proj_with_mqa" + ] }, { "type": "gemm", "args": { - "m": 18, - "n": 16384, - "k": 1536, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 36064, + "n": 1536, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/gate_up_proj", + "deepseek-ai/DeepSeek-R1-0528/q_a_proj", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w3" + ] }, { "type": "gemm", "args": { - "m": 18, - "n": 1088, + "m": 36064, + "n": 1536, "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w3" + ] }, { "type": "gemm", "args": { - "m": 18, - "n": 30720, - "k": 1024, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 36064, + "n": 7168, + "k": 576, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/down_proj" + ] }, { "type": "gemm", "args": { - "m": 18, + "m": 36064, "n": 7168, "k": 16384, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/o_proj", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] }, { "type": "gemm", "args": { - "m": 6291456, - "n": 65536, + "m": 36064, + "n": 24576, "k": 1536, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/q_b_proj" + ] }, { "type": "gemm", "args": { - "m": 6291456, - "n": 1088, - "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 36064, + "n": 32768, + "k": 512, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/kv_b_proj" + ] }, { "type": "gemm", "args": { - "m": 6291456, - "n": 122880, - "k": 1024, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 36864, + "n": 576, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/kv_a_proj_with_mqa" + ] }, { "type": "gemm", "args": { - "m": 6291456, - "n": 7168, - "k": 65536, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 36864, + "n": 1536, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/gate_up_proj", + "deepseek-ai/DeepSeek-R1-0528/q_a_proj", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w3" + ] }, { "type": "gemm", "args": { - "m": 768, - "n": 65536, - "k": 1536, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 36864, + "n": 1536, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w3" + ] }, { "type": "gemm", "args": { - "m": 768, - "n": 1088, + "m": 36864, + "n": 2304, "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/gate_up_proj" + ] }, { "type": "gemm", "args": { - "m": 768, - "n": 122880, - "k": 1024, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 36864, + "n": 3072, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "MiniMaxAI/MiniMax-M2.5/experts.down_proj", + "MiniMaxAI/MiniMax-M2.5/o_proj", + "deepseek-ai/DeepSeek-R1-0528/q_b_proj" + ] }, { "type": "gemm", "args": { - "m": 768, + "m": 36864, + "n": 4096, + "k": 512, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "Qwen/Qwen3.5-397B-A17B-FP8/o_proj", + "Qwen/Qwen3.5-397B-A17B-FP8/shared_expert.down_proj", + "deepseek-ai/DeepSeek-R1-0528/kv_b_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 36864, "n": 7168, - "k": 65536, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "k": 2048, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/o_proj", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] }, { "type": "gemm", "args": { - "m": 1572864, - "n": 65536, - "k": 1536, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 36864, + "n": 7168, + "k": 2304, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/down_proj" + ] }, { "type": "gemm", "args": { - "m": 1572864, - "n": 122880, - "k": 1024, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 39328, + "n": 576, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/Kimi-K2.5-MXFP4/kv_a_proj_with_mqa", + "moonshotai/Kimi-K2.5/kv_a_proj_with_mqa", + "nvidia/DeepSeek-R1-0528-FP4-V2/kv_a_proj_with_mqa", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/kv_a_proj_with_mqa", + "nvidia/Kimi-K2.5-NVFP4/kv_a_proj_with_mqa", + "nvidia/Kimi-K2.6-NVFP4/kv_a_proj_with_mqa" + ] }, { "type": "gemm", "args": { - "m": 1572864, - "n": 7168, - "k": 65536, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 39328, + "n": 576, + "k": 7168, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 32 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 1, + 32 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "amd/DeepSeek-R1-0528-MXFP4-Preview/kv_a_proj_with_mqa", + "amd/DeepSeek-R1-0528-MXFP4-v2/kv_a_proj_with_mqa", + "amd/DeepSeek-R1-0528-MXFP4/kv_a_proj_with_mqa" + ] }, { "type": "gemm", "args": { - "m": 192, - "n": 65536, - "k": 1536, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 39328, + "n": 576, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/kv_a_proj_with_mqa" + ] }, { "type": "gemm", "args": { - "m": 192, - "n": 122880, - "k": 1024, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 39328, + "n": 1536, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/Kimi-K2.5-MXFP4/q_a_proj", + "moonshotai/Kimi-K2.5/q_a_proj", + "nvidia/DeepSeek-R1-0528-FP4-V2/q_a_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/q_a_proj", + "nvidia/Kimi-K2.5-NVFP4/q_a_proj", + "nvidia/Kimi-K2.6-NVFP4/q_a_proj" + ] }, { "type": "gemm", "args": { - "m": 192, - "n": 7168, - "k": 65536, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 39328, + "n": 1536, + "k": 7168, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 32 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 1, + 32 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "amd/DeepSeek-R1-0528-MXFP4-Preview/q_a_proj", + "amd/DeepSeek-R1-0528-MXFP4-v2/q_a_proj", + "amd/DeepSeek-R1-0528-MXFP4/q_a_proj" + ] }, { "type": "gemm", "args": { - "m": 16777216, - "n": 65536, - "k": 1536, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 39328, + "n": 1536, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/gate_up_proj", + "deepseek-ai/DeepSeek-R1-0528/q_a_proj", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w3" + ] }, { "type": "gemm", "args": { - "m": 16777216, - "n": 1088, + "m": 39328, + "n": 1536, "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w3" + ] }, { "type": "gemm", "args": { - "m": 16777216, - "n": 122880, - "k": 1024, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 39328, + "n": 7168, + "k": 576, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-NVFP4-v2/down_proj" + ] }, { "type": "gemm", "args": { - "m": 16777216, + "m": 39328, "n": 7168, - "k": 65536, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "k": 576, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/down_proj" + ] }, { "type": "gemm", "args": { - "m": 2048, - "n": 65536, - "k": 1536, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 39328, + "n": 7168, + "k": 16384, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/o_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/o_proj" + ] }, { "type": "gemm", "args": { - "m": 2048, - "n": 122880, - "k": 1024, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 39328, + "n": 7168, + "k": 16384, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 32 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 1, + 32 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "amd/DeepSeek-R1-0528-MXFP4-Preview/o_proj", + "amd/DeepSeek-R1-0528-MXFP4-v2/o_proj", + "amd/DeepSeek-R1-0528-MXFP4/o_proj" + ] }, { "type": "gemm", "args": { - "m": 2048, + "m": 39328, "n": 7168, - "k": 65536, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "k": 16384, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/o_proj", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] }, { "type": "gemm", "args": { - "m": 44040192, - "n": 1536, - "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 39328, + "n": 7168, + "k": 16384, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] }, { "type": "gemm", "args": { - "m": 44040192, - "n": 65536, + "m": 39328, + "n": 24576, "k": 1536, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/q_b_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/q_b_proj" + ] }, { "type": "gemm", "args": { - "m": 44040192, - "n": 1088, - "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 39328, + "n": 24576, + "k": 1536, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 32 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 1, + 32 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "amd/DeepSeek-R1-0528-MXFP4-Preview/q_b_proj", + "amd/DeepSeek-R1-0528-MXFP4-v2/q_b_proj", + "amd/DeepSeek-R1-0528-MXFP4/q_b_proj" + ] }, { "type": "gemm", "args": { - "m": 44040192, - "n": 122880, - "k": 1024, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 39328, + "n": 24576, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/q_b_proj" + ] }, { "type": "gemm", "args": { - "m": 44040192, - "n": 7168, - "k": 65536, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 39328, + "n": 32768, + "k": 512, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/kv_b_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/kv_b_proj" + ] }, { "type": "gemm", "args": { - "m": 5376, - "n": 1536, - "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 39328, + "n": 32768, + "k": 512, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 32 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 1, + 32 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "amd/DeepSeek-R1-0528-MXFP4-Preview/kv_b_proj", + "amd/DeepSeek-R1-0528-MXFP4-v2/kv_b_proj", + "amd/DeepSeek-R1-0528-MXFP4/kv_b_proj" + ] }, { "type": "gemm", "args": { - "m": 5376, - "n": 65536, - "k": 1536, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 39328, + "n": 32768, + "k": 512, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/kv_b_proj" + ] }, { "type": "gemm", "args": { - "m": 5376, - "n": 1088, + "m": 39360, + "n": 576, "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 5376, - "n": 122880, - "k": 1024, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/kv_a_proj_with_mqa" + ] }, { "type": "gemm", "args": { - "m": 5376, - "n": 7168, - "k": 65536, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 39360, + "n": 1152, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/gate_up_proj" + ] }, { "type": "gemm", "args": { - "m": 14680064, + "m": 39360, "n": 1536, "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/gate_up_proj", + "deepseek-ai/DeepSeek-R1-0528/q_a_proj", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w3" + ] }, { "type": "gemm", "args": { - "m": 14680064, - "n": 65536, - "k": 1536, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 39360, + "n": 7168, + "k": 1152, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/down_proj" + ] }, { "type": "gemm", "args": { - "m": 14680064, - "n": 1088, - "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 39360, + "n": 7168, + "k": 16384, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/o_proj", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] }, { "type": "gemm", "args": { - "m": 14680064, - "n": 122880, - "k": 1024, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 39360, + "n": 24576, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/q_b_proj" + ] }, { "type": "gemm", "args": { - "m": 14680064, - "n": 7168, - "k": 65536, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 39360, + "n": 32768, + "k": 512, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/kv_b_proj" + ] }, { "type": "gemm", "args": { - "m": 1792, - "n": 1536, + "m": 39424, + "n": 576, "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/kv_a_proj_with_mqa" + ] }, { "type": "gemm", "args": { - "m": 1792, - "n": 65536, - "k": 1536, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 39424, + "n": 1152, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/gate_up_proj" + ] }, { "type": "gemm", "args": { - "m": 1792, - "n": 1088, + "m": 39424, + "n": 1536, "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/gate_up_proj", + "deepseek-ai/DeepSeek-R1-0528/q_a_proj", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w3" + ] }, { "type": "gemm", "args": { - "m": 1792, - "n": 122880, - "k": 1024, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 39424, + "n": 7168, + "k": 1152, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/down_proj" + ] }, { "type": "gemm", "args": { - "m": 1792, + "m": 39424, "n": 7168, - "k": 65536, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "k": 16384, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/o_proj", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] }, { "type": "gemm", "args": { - "m": 524288, - "n": 8192, + "m": 39424, + "n": 24576, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/q_b_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 39424, + "n": 32768, "k": 512, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/kv_b_proj" + ] }, { "type": "gemm", "args": { - "m": 524288, - "n": 7168, - "k": 4096, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 40960, + "n": 576, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/kv_a_proj_with_mqa" + ] }, { "type": "gemm", "args": { - "m": 64, - "n": 8192, - "k": 512, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 40960, + "n": 1536, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/gate_up_proj", + "deepseek-ai/DeepSeek-R1-0528/q_a_proj", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w3" + ] }, { "type": "gemm", "args": { - "m": 64, - "n": 7168, - "k": 4096, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 40960, + "n": 2304, + "k": 7168, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/gate_up_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/gate_up_proj", + "nvidia/Kimi-K2.5-NVFP4/gate_up_proj" + ] }, { "type": "gemm", "args": { - "m": 256, - "n": 1024, - "k": 4096, - "dtype_a": "bf16", - "dtype_b": "bf16", - "dtype_out": "bf16" - } + "m": 40960, + "n": 3072, + "k": 1536, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/Kimi-K2.5-MXFP4/q_b_proj", + "amd/MiniMax-M2.5-MXFP4/o_proj", + "moonshotai/Kimi-K2.5/q_b_proj", + "nvidia/DeepSeek-R1-0528-FP4-V2/q_b_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/q_b_proj", + "nvidia/Kimi-K2.5-NVFP4/q_b_proj", + "nvidia/Kimi-K2.6-NVFP4/q_b_proj", + "nvidia/MiniMax-M2.5-NVFP4/o_proj" + ] }, { "type": "gemm", "args": { - "m": 256, - "n": 256, - "k": 4096, - "dtype_a": "bf16", - "dtype_b": "bf16", - "dtype_out": "bf16" - } + "m": 40960, + "n": 3072, + "k": 1536, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/MiniMax-M2.5-NVFP4/experts.down_proj" + ] }, { "type": "gemm", "args": { - "m": 256, - "n": 4096, - "k": 1024, - "dtype_a": "bf16", - "dtype_b": "bf16", - "dtype_out": "bf16" - } + "m": 40960, + "n": 3072, + "k": 1536, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 32 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 1, + 32 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "amd/DeepSeek-R1-0528-MXFP4-Preview/q_b_proj", + "amd/DeepSeek-R1-0528-MXFP4-v2/q_b_proj", + "amd/DeepSeek-R1-0528-MXFP4/q_b_proj", + "amd/MiniMax-M2.5-MXFP4/experts.down_proj" + ] }, { "type": "gemm", "args": { - "m": 256, - "n": 128, - "k": 4096, - "dtype_a": "bf16", - "dtype_b": "bf16", - "dtype_out": "bf16" - } + "m": 40960, + "n": 4096, + "k": 512, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "Qwen/Qwen3.5-397B-A17B/o_proj", + "Qwen/Qwen3.5-397B-A17B/shared_expert.down_proj", + "amd/Kimi-K2.5-MXFP4/kv_b_proj", + "amd/Qwen3.5-397B-A17B-MXFP4/o_proj", + "amd/Qwen3.5-397B-A17B-MXFP4/shared_expert.down_proj", + "moonshotai/Kimi-K2.5/kv_b_proj", + "nvidia/DeepSeek-R1-0528-FP4-V2/kv_b_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/kv_b_proj", + "nvidia/Kimi-K2.5-NVFP4/kv_b_proj", + "nvidia/Kimi-K2.6-NVFP4/kv_b_proj", + "nvidia/Qwen3.5-397B-A17B-NVFP4/o_proj", + "nvidia/Qwen3.5-397B-A17B-NVFP4/shared_expert.down_proj" + ] }, { "type": "gemm", "args": { - "m": 256, + "m": 40960, "n": 4096, - "k": 128, - "dtype_a": "bf16", - "dtype_b": "bf16", - "dtype_out": "bf16" - } + "k": 512, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 32 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 1, + 32 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "amd/DeepSeek-R1-0528-MXFP4-Preview/kv_b_proj", + "amd/DeepSeek-R1-0528-MXFP4-v2/kv_b_proj", + "amd/DeepSeek-R1-0528-MXFP4/kv_b_proj", + "amd/Qwen3.5-397B-A17B-MXFP4-AttnFP8-V2/shared_expert.down_proj" + ] }, { "type": "gemm", "args": { - "m": 1048576, - "n": 1024, - "k": 4096, - "dtype_a": "bf16", - "dtype_b": "bf16", - "dtype_out": "bf16" - } + "m": 40960, + "n": 4608, + "k": 7168, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/gate_up_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/gate_up_proj", + "nvidia/Kimi-K2.5-NVFP4/gate_up_proj" + ] }, { "type": "gemm", "args": { - "m": 1048576, - "n": 256, - "k": 4096, - "dtype_a": "bf16", - "dtype_b": "bf16", - "dtype_out": "bf16" - } + "m": 40960, + "n": 4608, + "k": 7168, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 32 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 1, + 32 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "amd/DeepSeek-R1-0528-MXFP4-Preview/gate_up_proj", + "amd/DeepSeek-R1-0528-MXFP4-v2/gate_up_proj", + "amd/DeepSeek-R1-0528-MXFP4/gate_up_proj", + "amd/Kimi-K2.5-MXFP4/gate_up_proj" + ] }, { "type": "gemm", "args": { - "m": 1048576, - "n": 4096, - "k": 1024, - "dtype_a": "bf16", - "dtype_b": "bf16", - "dtype_out": "bf16" - } + "m": 40960, + "n": 4608, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/gate_up_proj" + ] }, { "type": "gemm", "args": { - "m": 1048576, - "n": 128, - "k": 4096, - "dtype_a": "bf16", - "dtype_b": "bf16", - "dtype_out": "bf16" - } + "m": 40960, + "n": 6144, + "k": 1536, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/GLM-5.1-MXFP4/down_proj", + "amd/GLM-5.2-MXFP4/down_proj", + "amd/Kimi-K2.5-MXFP4/q_b_proj", + "amd/MiniMax-M3-MXFP4/down_proj", + "moonshotai/Kimi-K2.5/q_b_proj", + "nvidia/DeepSeek-R1-0528-FP4-V2/q_b_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/q_b_proj", + "nvidia/GLM-5.1-NVFP4/down_proj", + "nvidia/GLM-5.2-NVFP4/down_proj", + "nvidia/Kimi-K2.5-NVFP4/q_b_proj", + "nvidia/Kimi-K2.6-NVFP4/q_b_proj" + ] }, { "type": "gemm", "args": { - "m": 1048576, - "n": 4096, - "k": 128, - "dtype_a": "bf16", - "dtype_b": "bf16", - "dtype_out": "bf16" - } + "m": 40960, + "n": 6144, + "k": 1536, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/GLM-5-NVFP4/down_proj" + ] }, { "type": "gemm", "args": { - "m": 2097152, - "n": 1024, - "k": 4096, - "dtype_a": "bf16", - "dtype_b": "bf16", - "dtype_out": "bf16" - } + "m": 40960, + "n": 6144, + "k": 1536, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 32 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 1, + 32 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "amd/DeepSeek-R1-0528-MXFP4-Preview/q_b_proj", + "amd/DeepSeek-R1-0528-MXFP4-v2/q_b_proj", + "amd/DeepSeek-R1-0528-MXFP4/q_b_proj" + ] }, { "type": "gemm", "args": { - "m": 2097152, - "n": 256, - "k": 4096, - "dtype_a": "bf16", - "dtype_b": "bf16", - "dtype_out": "bf16" - } + "m": 40960, + "n": 7168, + "k": 2048, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/Kimi-K2.5-MXFP4/o_proj", + "moonshotai/Kimi-K2.5/o_proj", + "nvidia/Kimi-K2.5-NVFP4/o_proj", + "nvidia/Kimi-K2.6-NVFP4/o_proj" + ] }, { "type": "gemm", "args": { - "m": 2097152, - "n": 4096, - "k": 1024, - "dtype_a": "bf16", - "dtype_b": "bf16", - "dtype_out": "bf16" - } + "m": 40960, + "n": 7168, + "k": 2048, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/o_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/o_proj" + ] }, { "type": "gemm", "args": { - "m": 2097152, - "n": 128, - "k": 4096, - "dtype_a": "bf16", - "dtype_b": "bf16", - "dtype_out": "bf16" - } + "m": 40960, + "n": 7168, + "k": 2048, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 32 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 1, + 32 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "amd/DeepSeek-R1-0528-MXFP4-Preview/o_proj", + "amd/DeepSeek-R1-0528-MXFP4-v2/o_proj", + "amd/DeepSeek-R1-0528-MXFP4/o_proj" + ] }, { "type": "gemm", "args": { - "m": 2097152, - "n": 4096, - "k": 128, - "dtype_a": "bf16", - "dtype_b": "bf16", - "dtype_out": "bf16" - } + "m": 40960, + "n": 7168, + "k": 2304, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/down_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/down_proj", + "nvidia/Kimi-K2.5-NVFP4/down_proj" + ] }, { "type": "gemm", "args": { - "m": 512, - "n": 1024, - "k": 4096, - "dtype_a": "bf16", - "dtype_b": "bf16", - "dtype_out": "bf16" - } + "m": 40960, + "n": 7168, + "k": 2304, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 32 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 1, + 32 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "amd/DeepSeek-R1-0528-MXFP4-Preview/down_proj", + "amd/DeepSeek-R1-0528-MXFP4-v2/down_proj", + "amd/DeepSeek-R1-0528-MXFP4/down_proj", + "amd/Kimi-K2.5-MXFP4/down_proj" + ] }, { "type": "gemm", "args": { - "m": 512, - "n": 256, + "m": 40960, + "n": 7168, "k": 4096, - "dtype_a": "bf16", - "dtype_b": "bf16", - "dtype_out": "bf16" - } + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/o_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/o_proj" + ] }, { "type": "gemm", "args": { - "m": 512, - "n": 4096, - "k": 1024, - "dtype_a": "bf16", - "dtype_b": "bf16", - "dtype_out": "bf16" - } + "m": 40960, + "n": 7168, + "k": 4096, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 32 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 1, + 32 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "amd/DeepSeek-R1-0528-MXFP4-Preview/o_proj", + "amd/DeepSeek-R1-0528-MXFP4-v2/o_proj", + "amd/DeepSeek-R1-0528-MXFP4/o_proj" + ] }, { "type": "gemm", "args": { - "m": 512, - "n": 128, - "k": 4096, - "dtype_a": "bf16", - "dtype_b": "bf16", - "dtype_out": "bf16" - } + "m": 40960, + "n": 7168, + "k": 4608, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/down_proj" + ] }, { "type": "gemm", "args": { - "m": 512, - "n": 4096, - "k": 128, - "dtype_a": "bf16", - "dtype_b": "bf16", - "dtype_out": "bf16" - } + "m": 40960, + "n": 7168, + "k": 16384, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/o_proj", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] }, { "type": "gemm", "args": { - "m": 131072, - "n": 4096, - "k": 4096, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 40960, + "n": 8192, + "k": 512, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/Kimi-K2.5-MXFP4/kv_b_proj", + "moonshotai/Kimi-K2.5/kv_b_proj", + "nvidia/DeepSeek-R1-0528-FP4-V2/kv_b_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/kv_b_proj", + "nvidia/Kimi-K2.5-NVFP4/kv_b_proj", + "nvidia/Kimi-K2.6-NVFP4/kv_b_proj" + ] }, { "type": "gemm", "args": { - "m": 131072, - "n": 512, - "k": 4096, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 40960, + "n": 8192, + "k": 512, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 32 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 1, + 32 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "amd/DeepSeek-R1-0528-MXFP4-Preview/kv_b_proj", + "amd/DeepSeek-R1-0528-MXFP4-v2/kv_b_proj", + "amd/DeepSeek-R1-0528-MXFP4/kv_b_proj" + ] }, { "type": "gemm", "args": { - "m": 128, - "n": 4096, - "k": 4096, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 40960, + "n": 24576, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/q_b_proj" + ] }, { "type": "gemm", "args": { - "m": 128, - "n": 512, - "k": 4096, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 40960, + "n": 32768, + "k": 512, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/kv_b_proj" + ] }, { "type": "gemm", "args": { - "m": 262144, - "n": 4096, - "k": 4096, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 42624, + "n": 576, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/Kimi-K2.5-MXFP4/kv_a_proj_with_mqa", + "moonshotai/Kimi-K2.5/kv_a_proj_with_mqa", + "nvidia/DeepSeek-R1-0528-FP4-V2/kv_a_proj_with_mqa", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/kv_a_proj_with_mqa", + "nvidia/Kimi-K2.5-NVFP4/kv_a_proj_with_mqa", + "nvidia/Kimi-K2.6-NVFP4/kv_a_proj_with_mqa" + ] }, { "type": "gemm", "args": { - "m": 262144, - "n": 512, - "k": 4096, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 42624, + "n": 576, + "k": 7168, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 32 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 1, + 32 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "amd/DeepSeek-R1-0528-MXFP4-Preview/kv_a_proj_with_mqa", + "amd/DeepSeek-R1-0528-MXFP4-v2/kv_a_proj_with_mqa", + "amd/DeepSeek-R1-0528-MXFP4/kv_a_proj_with_mqa" + ] }, { "type": "gemm", "args": { - "m": 256, - "n": 4096, - "k": 4096, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 42624, + "n": 576, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/kv_a_proj_with_mqa" + ] }, { "type": "gemm", "args": { - "m": 256, - "n": 512, - "k": 4096, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 42624, + "n": 1152, + "k": 7168, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-NVFP4-v2/gate_up_proj" + ] }, { "type": "gemm", "args": { - "m": 32768, - "n": 4096, - "k": 4096, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 42624, + "n": 1152, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/gate_up_proj" + ] }, { "type": "gemm", "args": { - "m": 32768, - "n": 512, - "k": 4096, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 42624, + "n": 1536, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/Kimi-K2.5-MXFP4/q_a_proj", + "moonshotai/Kimi-K2.5/q_a_proj", + "nvidia/DeepSeek-R1-0528-FP4-V2/q_a_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/q_a_proj", + "nvidia/Kimi-K2.5-NVFP4/q_a_proj", + "nvidia/Kimi-K2.6-NVFP4/q_a_proj" + ] }, { "type": "gemm", "args": { - "m": 4, - "n": 4096, - "k": 4096, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 42624, + "n": 1536, + "k": 7168, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 32 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 1, + 32 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "amd/DeepSeek-R1-0528-MXFP4-Preview/q_a_proj", + "amd/DeepSeek-R1-0528-MXFP4-v2/q_a_proj", + "amd/DeepSeek-R1-0528-MXFP4/q_a_proj" + ] }, { "type": "gemm", "args": { - "m": 4, - "n": 512, - "k": 4096, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 42624, + "n": 1536, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/gate_up_proj", + "deepseek-ai/DeepSeek-R1-0528/q_a_proj", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w3" + ] }, { "type": "gemm", "args": { - "m": 65536, - "n": 4096, - "k": 4096, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 42624, + "n": 1536, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w3" + ] }, { "type": "gemm", "args": { - "m": 65536, - "n": 512, - "k": 4096, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 42624, + "n": 7168, + "k": 1152, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/down_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/down_proj", + "nvidia/Kimi-K2.5-NVFP4/down_proj" + ] }, { "type": "gemm", "args": { - "m": 8, - "n": 4096, - "k": 4096, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 42624, + "n": 7168, + "k": 1152, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/down_proj" + ] }, { "type": "gemm", "args": { - "m": 8, - "n": 512, - "k": 4096, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 42624, + "n": 7168, + "k": 8192, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/Kimi-K2.5-MXFP4/o_proj", + "moonshotai/Kimi-K2.5/o_proj", + "nvidia/Kimi-K2.5-NVFP4/o_proj", + "nvidia/Kimi-K2.6-NVFP4/o_proj" + ] }, { "type": "gemm", "args": { - "m": 16, - "n": 4096, - "k": 4096, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 42624, + "n": 7168, + "k": 8192, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] }, { "type": "gemm", "args": { - "m": 16, - "n": 512, - "k": 4096, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 42624, + "n": 7168, + "k": 16384, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/o_proj", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] }, { "type": "gemm", "args": { - "m": 32, - "n": 4096, - "k": 4096, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 42624, + "n": 12288, + "k": 1536, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/Kimi-K2.5-MXFP4/q_b_proj", + "moonshotai/Kimi-K2.5/q_b_proj", + "nvidia/DeepSeek-R1-0528-FP4-V2/q_b_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/q_b_proj", + "nvidia/Kimi-K2.5-NVFP4/q_b_proj", + "nvidia/Kimi-K2.6-NVFP4/q_b_proj" + ] }, { "type": "gemm", "args": { - "m": 32, - "n": 512, - "k": 4096, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 42624, + "n": 16384, + "k": 512, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/Kimi-K2.5-MXFP4/kv_b_proj", + "moonshotai/Kimi-K2.5/kv_b_proj", + "nvidia/DeepSeek-R1-0528-FP4-V2/kv_b_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/kv_b_proj", + "nvidia/Kimi-K2.5-NVFP4/kv_b_proj", + "nvidia/Kimi-K2.6-NVFP4/kv_b_proj" + ] }, { "type": "gemm", "args": { - "m": 512, - "n": 1024, - "k": 4096, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 42624, + "n": 24576, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/q_b_proj" + ] }, { "type": "gemm", "args": { - "m": 512, - "n": 4096, - "k": 1024, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 42624, + "n": 32768, + "k": 512, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/kv_b_proj" + ] }, { "type": "gemm", "args": { - "m": 512, - "n": 128, - "k": 4096, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 43690, + "n": 384, + "k": 7168, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [] }, { "type": "gemm", "args": { - "m": 512, - "n": 4096, - "k": 128, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 43690, + "n": 384, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w3" + ] }, { "type": "gemm", "args": { - "m": 512, - "n": 4096, - "k": 4096, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 43690, + "n": 576, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/Kimi-K2.5-MXFP4/kv_a_proj_with_mqa", + "moonshotai/Kimi-K2.5/kv_a_proj_with_mqa", + "nvidia/DeepSeek-R1-0528-FP4-V2/kv_a_proj_with_mqa", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/kv_a_proj_with_mqa", + "nvidia/Kimi-K2.5-NVFP4/kv_a_proj_with_mqa", + "nvidia/Kimi-K2.6-NVFP4/kv_a_proj_with_mqa" + ] }, { "type": "gemm", "args": { - "m": 512, - "n": 512, - "k": 4096, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 43690, + "n": 576, + "k": 7168, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 32 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 1, + 32 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "amd/DeepSeek-R1-0528-MXFP4-Preview/kv_a_proj_with_mqa", + "amd/DeepSeek-R1-0528-MXFP4-v2/kv_a_proj_with_mqa", + "amd/DeepSeek-R1-0528-MXFP4/kv_a_proj_with_mqa" + ] }, { "type": "gemm", "args": { - "m": 64, - "n": 4096, - "k": 4096, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 43690, + "n": 576, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/kv_a_proj_with_mqa" + ] }, { "type": "gemm", "args": { - "m": 64, - "n": 512, - "k": 4096, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 43690, + "n": 1536, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/Kimi-K2.5-MXFP4/q_a_proj", + "moonshotai/Kimi-K2.5/q_a_proj", + "nvidia/DeepSeek-R1-0528-FP4-V2/q_a_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/q_a_proj", + "nvidia/Kimi-K2.5-NVFP4/q_a_proj", + "nvidia/Kimi-K2.6-NVFP4/q_a_proj" + ] }, { "type": "gemm", "args": { - "m": 4096, - "n": 4096, - "k": 4096, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 43690, + "n": 1536, + "k": 7168, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 32 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 1, + 32 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "amd/DeepSeek-R1-0528-MXFP4-Preview/q_a_proj", + "amd/DeepSeek-R1-0528-MXFP4-v2/q_a_proj", + "amd/DeepSeek-R1-0528-MXFP4/q_a_proj" + ] }, { "type": "gemm", "args": { - "m": 4096, - "n": 512, - "k": 4096, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 43690, + "n": 1536, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/gate_up_proj", + "deepseek-ai/DeepSeek-R1-0528/q_a_proj", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w3" + ] }, { "type": "gemm", "args": { - "m": 8192, - "n": 4096, - "k": 4096, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 43690, + "n": 1536, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w3" + ] }, { "type": "gemm", "args": { - "m": 8192, - "n": 512, - "k": 4096, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 43690, + "n": 7168, + "k": 384, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-NVFP4-v2/down_proj" + ] }, { "type": "gemm", "args": { - "m": 16384, - "n": 4096, - "k": 4096, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 43690, + "n": 7168, + "k": 384, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w2" + ] }, { "type": "gemm", "args": { - "m": 16384, - "n": 512, - "k": 4096, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 43690, + "n": 7168, + "k": 16384, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/o_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/o_proj" + ] }, { "type": "gemm", "args": { - "m": 524288, - "n": 4096, - "k": 4096, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 43690, + "n": 7168, + "k": 16384, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 32 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 1, + 32 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "amd/DeepSeek-R1-0528-MXFP4-Preview/o_proj", + "amd/DeepSeek-R1-0528-MXFP4-v2/o_proj", + "amd/DeepSeek-R1-0528-MXFP4/o_proj" + ] }, { "type": "gemm", "args": { - "m": 524288, - "n": 512, - "k": 4096, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 43690, + "n": 7168, + "k": 16384, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/o_proj", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] }, { "type": "gemm", "args": { - "m": 1048576, - "n": 4096, - "k": 4096, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 43690, + "n": 7168, + "k": 16384, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] }, { "type": "gemm", "args": { - "m": 1048576, - "n": 512, - "k": 4096, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 43690, + "n": 24576, + "k": 1536, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/q_b_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/q_b_proj" + ] }, { "type": "gemm", "args": { - "m": 2097152, - "n": 4096, - "k": 4096, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 43690, + "n": 24576, + "k": 1536, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 32 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 1, + 32 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "amd/DeepSeek-R1-0528-MXFP4-Preview/q_b_proj", + "amd/DeepSeek-R1-0528-MXFP4-v2/q_b_proj", + "amd/DeepSeek-R1-0528-MXFP4/q_b_proj" + ] }, { "type": "gemm", "args": { - "m": 2097152, - "n": 512, - "k": 4096, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 43690, + "n": 24576, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/q_b_proj" + ] }, { "type": "gemm", "args": { - "m": 2097152, - "n": 1024, - "k": 4096, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 43690, + "n": 32768, + "k": 512, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/kv_b_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/kv_b_proj" + ] }, { "type": "gemm", "args": { - "m": 2097152, - "n": 4096, - "k": 1024, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 43690, + "n": 32768, + "k": 512, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 32 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 1, + 32 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "amd/DeepSeek-R1-0528-MXFP4-Preview/kv_b_proj", + "amd/DeepSeek-R1-0528-MXFP4-v2/kv_b_proj", + "amd/DeepSeek-R1-0528-MXFP4/kv_b_proj" + ] }, { "type": "gemm", "args": { - "m": 2097152, - "n": 128, - "k": 4096, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 43690, + "n": 32768, + "k": 512, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/kv_b_proj" + ] }, { "type": "gemm", "args": { - "m": 2097152, - "n": 4096, - "k": 128, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 46080, + "n": 576, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/Kimi-K2.5-MXFP4/kv_a_proj_with_mqa", + "moonshotai/Kimi-K2.5/kv_a_proj_with_mqa", + "nvidia/DeepSeek-R1-0528-FP4-V2/kv_a_proj_with_mqa", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/kv_a_proj_with_mqa", + "nvidia/Kimi-K2.5-NVFP4/kv_a_proj_with_mqa", + "nvidia/Kimi-K2.6-NVFP4/kv_a_proj_with_mqa" + ] }, { "type": "gemm", "args": { - "m": 2097152, - "n": 4096, - "k": 4096, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 46080, + "n": 576, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/kv_a_proj_with_mqa" + ] }, { "type": "gemm", "args": { - "m": 2097152, - "n": 256, - "k": 4096, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 46080, + "n": 1536, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/Kimi-K2.5-MXFP4/q_a_proj", + "moonshotai/Kimi-K2.5/q_a_proj", + "nvidia/DeepSeek-R1-0528-FP4-V2/q_a_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/q_a_proj", + "nvidia/Kimi-K2.5-NVFP4/q_a_proj", + "nvidia/Kimi-K2.6-NVFP4/q_a_proj" + ] }, { "type": "gemm", "args": { - "m": 2097152, - "n": 512, - "k": 4096, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 46080, + "n": 1536, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/gate_up_proj", + "deepseek-ai/DeepSeek-R1-0528/q_a_proj", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w3" + ] }, { "type": "gemm", "args": { - "m": 4096, - "n": 7168, - "k": 512, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 46080, + "n": 2304, + "k": 7168, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/gate_up_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/gate_up_proj", + "nvidia/Kimi-K2.5-NVFP4/gate_up_proj" + ] }, { "type": "gemm", "args": { - "m": 4096, - "n": 6144, - "k": 4096, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 46080, + "n": 3072, + "k": 1536, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/Kimi-K2.5-MXFP4/q_b_proj", + "amd/MiniMax-M2.5-MXFP4/o_proj", + "moonshotai/Kimi-K2.5/q_b_proj", + "nvidia/DeepSeek-R1-0528-FP4-V2/q_b_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/q_b_proj", + "nvidia/Kimi-K2.5-NVFP4/q_b_proj", + "nvidia/Kimi-K2.6-NVFP4/q_b_proj", + "nvidia/MiniMax-M2.5-NVFP4/o_proj" + ] }, { "type": "gemm", "args": { - "m": 4096, + "m": 46080, "n": 3072, - "k": 6144, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "k": 1536, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/MiniMax-M2.5-NVFP4/experts.down_proj" + ] }, { "type": "gemm", "args": { - "m": 4096, - "n": 6144, - "k": 3072, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 46080, + "n": 3072, + "k": 1536, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 32 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 1, + 32 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "amd/DeepSeek-R1-0528-MXFP4-Preview/q_b_proj", + "amd/DeepSeek-R1-0528-MXFP4-v2/q_b_proj", + "amd/DeepSeek-R1-0528-MXFP4/q_b_proj", + "amd/MiniMax-M2.5-MXFP4/experts.down_proj" + ] }, { "type": "gemm", "args": { - "m": 8, - "n": 7168, + "m": 46080, + "n": 4096, "k": 512, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } - }, - { - "type": "gemm", - "args": { - "m": 8, - "n": 6144, - "k": 4096, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "Qwen/Qwen3.5-397B-A17B/o_proj", + "Qwen/Qwen3.5-397B-A17B/shared_expert.down_proj", + "amd/Kimi-K2.5-MXFP4/kv_b_proj", + "amd/Qwen3.5-397B-A17B-MXFP4/o_proj", + "amd/Qwen3.5-397B-A17B-MXFP4/shared_expert.down_proj", + "moonshotai/Kimi-K2.5/kv_b_proj", + "nvidia/DeepSeek-R1-0528-FP4-V2/kv_b_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/kv_b_proj", + "nvidia/Kimi-K2.5-NVFP4/kv_b_proj", + "nvidia/Kimi-K2.6-NVFP4/kv_b_proj", + "nvidia/Qwen3.5-397B-A17B-NVFP4/o_proj", + "nvidia/Qwen3.5-397B-A17B-NVFP4/shared_expert.down_proj" + ] }, { "type": "gemm", "args": { - "m": 8192, - "n": 7168, + "m": 46080, + "n": 4096, "k": 512, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 32 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 1, + 32 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "amd/DeepSeek-R1-0528-MXFP4-Preview/kv_b_proj", + "amd/DeepSeek-R1-0528-MXFP4-v2/kv_b_proj", + "amd/DeepSeek-R1-0528-MXFP4/kv_b_proj", + "amd/Qwen3.5-397B-A17B-MXFP4-AttnFP8-V2/shared_expert.down_proj" + ] }, { "type": "gemm", "args": { - "m": 8192, - "n": 6144, - "k": 4096, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 46080, + "n": 4608, + "k": 7168, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/gate_up_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/gate_up_proj", + "nvidia/Kimi-K2.5-NVFP4/gate_up_proj" + ] }, { "type": "gemm", "args": { - "m": 8192, - "n": 3072, - "k": 6144, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 46080, + "n": 4608, + "k": 7168, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 32 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 1, + 32 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "amd/DeepSeek-R1-0528-MXFP4-Preview/gate_up_proj", + "amd/DeepSeek-R1-0528-MXFP4-v2/gate_up_proj", + "amd/DeepSeek-R1-0528-MXFP4/gate_up_proj", + "amd/Kimi-K2.5-MXFP4/gate_up_proj" + ] }, { "type": "gemm", "args": { - "m": 8192, - "n": 6144, - "k": 3072, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 46080, + "n": 4608, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/gate_up_proj" + ] }, { "type": "gemm", "args": { - "m": 16, + "m": 46080, "n": 7168, - "k": 512, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "k": 576, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-NVFP4-v2/down_proj" + ] }, { "type": "gemm", "args": { - "m": 16, - "n": 6144, - "k": 4096, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 46080, + "n": 7168, + "k": 2048, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/Kimi-K2.5-MXFP4/o_proj", + "moonshotai/Kimi-K2.5/o_proj", + "nvidia/Kimi-K2.5-NVFP4/o_proj", + "nvidia/Kimi-K2.6-NVFP4/o_proj" + ] }, { "type": "gemm", "args": { - "m": 16384, + "m": 46080, "n": 7168, - "k": 512, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "k": 2048, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/o_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/o_proj" + ] }, { "type": "gemm", "args": { - "m": 16384, - "n": 6144, - "k": 4096, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 46080, + "n": 7168, + "k": 2048, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 32 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 1, + 32 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "amd/DeepSeek-R1-0528-MXFP4-Preview/o_proj", + "amd/DeepSeek-R1-0528-MXFP4-v2/o_proj", + "amd/DeepSeek-R1-0528-MXFP4/o_proj" + ] }, { "type": "gemm", "args": { - "m": 16384, - "n": 3072, - "k": 6144, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 46080, + "n": 7168, + "k": 2304, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/down_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/down_proj", + "nvidia/Kimi-K2.5-NVFP4/down_proj" + ] }, { "type": "gemm", "args": { - "m": 16384, - "n": 6144, - "k": 3072, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 46080, + "n": 7168, + "k": 2304, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 32 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 1, + 32 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "amd/DeepSeek-R1-0528-MXFP4-Preview/down_proj", + "amd/DeepSeek-R1-0528-MXFP4-v2/down_proj", + "amd/DeepSeek-R1-0528-MXFP4/down_proj", + "amd/Kimi-K2.5-MXFP4/down_proj" + ] }, { "type": "gemm", "args": { - "m": 32, + "m": 46080, "n": 7168, - "k": 512, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "k": 4608, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/down_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/down_proj", + "nvidia/Kimi-K2.5-NVFP4/down_proj" + ] }, { "type": "gemm", "args": { - "m": 32, - "n": 6144, - "k": 4096, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 46080, + "n": 7168, + "k": 4608, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/down_proj" + ] }, { "type": "gemm", "args": { - "m": 32, - "n": 3072, - "k": 6144, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 46080, + "n": 7168, + "k": 16384, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/o_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/o_proj" + ] }, { "type": "gemm", "args": { - "m": 32, - "n": 6144, - "k": 3072, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 46080, + "n": 7168, + "k": 16384, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/o_proj", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] }, { "type": "gemm", "args": { - "m": 32768, - "n": 7168, - "k": 512, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 46080, + "n": 24576, + "k": 1536, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/q_b_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/q_b_proj" + ] }, { "type": "gemm", "args": { - "m": 32768, - "n": 6144, - "k": 4096, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 46080, + "n": 24576, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/q_b_proj" + ] }, { "type": "gemm", "args": { - "m": 64, - "n": 7168, + "m": 46080, + "n": 32768, "k": 512, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/kv_b_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/kv_b_proj" + ] }, { "type": "gemm", "args": { - "m": 64, - "n": 6144, - "k": 4096, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 46080, + "n": 32768, + "k": 512, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/kv_b_proj" + ] }, { "type": "gemm", "args": { - "m": 64, - "n": 3072, - "k": 6144, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 49152, + "n": 576, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/Kimi-K2.5-MXFP4/kv_a_proj_with_mqa", + "moonshotai/Kimi-K2.5/kv_a_proj_with_mqa", + "nvidia/DeepSeek-R1-0528-FP4-V2/kv_a_proj_with_mqa", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/kv_a_proj_with_mqa", + "nvidia/Kimi-K2.5-NVFP4/kv_a_proj_with_mqa", + "nvidia/Kimi-K2.6-NVFP4/kv_a_proj_with_mqa" + ] }, { "type": "gemm", "args": { - "m": 64, - "n": 6144, - "k": 3072, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 49152, + "n": 576, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/kv_a_proj_with_mqa" + ] }, { "type": "gemm", "args": { - "m": 65536, - "n": 7168, - "k": 512, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 49152, + "n": 1152, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/gate_up_proj" + ] }, { "type": "gemm", "args": { - "m": 65536, - "n": 6144, - "k": 4096, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 49152, + "n": 1536, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/q_b_proj" + ] }, { "type": "gemm", "args": { - "m": 128, - "n": 7168, - "k": 512, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 49152, + "n": 1536, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/Kimi-K2.5-MXFP4/q_a_proj", + "moonshotai/Kimi-K2.5/q_a_proj", + "nvidia/DeepSeek-R1-0528-FP4-V2/q_a_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/q_a_proj", + "nvidia/Kimi-K2.5-NVFP4/q_a_proj", + "nvidia/Kimi-K2.6-NVFP4/q_a_proj" + ] }, { "type": "gemm", "args": { - "m": 128, - "n": 6144, - "k": 4096, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 49152, + "n": 1536, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/gate_up_proj", + "deepseek-ai/DeepSeek-R1-0528/q_a_proj", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w3" + ] }, { "type": "gemm", "args": { - "m": 128, - "n": 3072, - "k": 6144, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 49152, + "n": 2048, + "k": 512, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/kv_b_proj" + ] }, { "type": "gemm", "args": { - "m": 128, - "n": 6144, - "k": 3072, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 49152, + "n": 2304, + "k": 7168, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/gate_up_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/gate_up_proj", + "nvidia/Kimi-K2.5-NVFP4/gate_up_proj" + ] }, { "type": "gemm", "args": { - "m": 131072, - "n": 7168, - "k": 512, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 49152, + "n": 2304, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/gate_up_proj" + ] }, { "type": "gemm", "args": { - "m": 131072, - "n": 6144, - "k": 4096, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 49152, + "n": 3072, + "k": 1536, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/Kimi-K2.5-MXFP4/q_b_proj", + "amd/MiniMax-M2.5-MXFP4/o_proj", + "moonshotai/Kimi-K2.5/q_b_proj", + "nvidia/DeepSeek-R1-0528-FP4-V2/q_b_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/q_b_proj", + "nvidia/Kimi-K2.5-NVFP4/q_b_proj", + "nvidia/Kimi-K2.6-NVFP4/q_b_proj", + "nvidia/MiniMax-M2.5-NVFP4/o_proj" + ] }, { "type": "gemm", "args": { - "m": 256, - "n": 7168, - "k": 512, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 49152, + "n": 3072, + "k": 1536, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/MiniMax-M2.5-NVFP4/experts.down_proj" + ] }, { "type": "gemm", "args": { - "m": 256, - "n": 6144, - "k": 4096, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 49152, + "n": 3072, + "k": 1536, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 32 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 1, + 32 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "amd/DeepSeek-R1-0528-MXFP4-Preview/q_b_proj", + "amd/DeepSeek-R1-0528-MXFP4-v2/q_b_proj", + "amd/DeepSeek-R1-0528-MXFP4/q_b_proj", + "amd/MiniMax-M2.5-MXFP4/experts.down_proj" + ] }, { "type": "gemm", "args": { - "m": 256, + "m": 49152, "n": 3072, - "k": 6144, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "MiniMaxAI/MiniMax-M2.5/experts.down_proj", + "MiniMaxAI/MiniMax-M2.5/o_proj", + "deepseek-ai/DeepSeek-R1-0528/q_b_proj" + ] }, { "type": "gemm", "args": { - "m": 256, - "n": 6144, - "k": 3072, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 49152, + "n": 4096, + "k": 512, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "Qwen/Qwen3.5-397B-A17B/o_proj", + "Qwen/Qwen3.5-397B-A17B/shared_expert.down_proj", + "amd/Kimi-K2.5-MXFP4/kv_b_proj", + "amd/Qwen3.5-397B-A17B-MXFP4/o_proj", + "amd/Qwen3.5-397B-A17B-MXFP4/shared_expert.down_proj", + "moonshotai/Kimi-K2.5/kv_b_proj", + "nvidia/DeepSeek-R1-0528-FP4-V2/kv_b_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/kv_b_proj", + "nvidia/Kimi-K2.5-NVFP4/kv_b_proj", + "nvidia/Kimi-K2.6-NVFP4/kv_b_proj", + "nvidia/Qwen3.5-397B-A17B-NVFP4/o_proj", + "nvidia/Qwen3.5-397B-A17B-NVFP4/shared_expert.down_proj" + ] }, { "type": "gemm", "args": { - "m": 262144, - "n": 7168, + "m": 49152, + "n": 4096, "k": 512, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 32 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 1, + 32 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "amd/DeepSeek-R1-0528-MXFP4-Preview/kv_b_proj", + "amd/DeepSeek-R1-0528-MXFP4-v2/kv_b_proj", + "amd/DeepSeek-R1-0528-MXFP4/kv_b_proj", + "amd/Qwen3.5-397B-A17B-MXFP4-AttnFP8-V2/shared_expert.down_proj" + ] }, { "type": "gemm", "args": { - "m": 262144, - "n": 6144, - "k": 4096, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 49152, + "n": 4096, + "k": 512, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "Qwen/Qwen3.5-397B-A17B-FP8/o_proj", + "Qwen/Qwen3.5-397B-A17B-FP8/shared_expert.down_proj", + "deepseek-ai/DeepSeek-R1-0528/kv_b_proj" + ] }, { "type": "gemm", "args": { - "m": 262144, - "n": 3072, - "k": 6144, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 49152, + "n": 4608, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/gate_up_proj" + ] }, { "type": "gemm", "args": { - "m": 262144, - "n": 6144, - "k": 3072, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 49152, + "n": 7168, + "k": 1024, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/o_proj", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] }, { "type": "gemm", "args": { - "m": 524288, + "m": 49152, "n": 7168, - "k": 512, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "k": 1152, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/down_proj" + ] }, { "type": "gemm", "args": { - "m": 524288, - "n": 6144, - "k": 4096, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 49152, + "n": 7168, + "k": 2048, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/Kimi-K2.5-MXFP4/o_proj", + "moonshotai/Kimi-K2.5/o_proj", + "nvidia/Kimi-K2.5-NVFP4/o_proj", + "nvidia/Kimi-K2.6-NVFP4/o_proj" + ] }, { "type": "gemm", "args": { - "m": 1048576, + "m": 49152, "n": 7168, - "k": 512, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "k": 2048, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/o_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/o_proj" + ] }, { "type": "gemm", "args": { - "m": 1048576, - "n": 6144, - "k": 4096, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 49152, + "n": 7168, + "k": 2048, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 32 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 1, + 32 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "amd/DeepSeek-R1-0528-MXFP4-Preview/o_proj", + "amd/DeepSeek-R1-0528-MXFP4-v2/o_proj", + "amd/DeepSeek-R1-0528-MXFP4/o_proj" + ] }, { "type": "gemm", "args": { - "m": 1048576, - "n": 3072, - "k": 6144, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 49152, + "n": 7168, + "k": 2048, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/o_proj", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] }, { "type": "gemm", "args": { - "m": 1048576, - "n": 6144, - "k": 3072, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 49152, + "n": 7168, + "k": 2304, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/down_proj" + ] }, { "type": "gemm", "args": { - "m": 4, + "m": 49152, "n": 7168, - "k": 512, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "k": 4608, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/down_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/down_proj", + "nvidia/Kimi-K2.5-NVFP4/down_proj" + ] }, { "type": "gemm", "args": { - "m": 4, - "n": 6144, - "k": 4096, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 49152, + "n": 7168, + "k": 4608, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/down_proj" + ] }, { "type": "gemm", "args": { - "m": 2097152, + "m": 49152, "n": 7168, - "k": 512, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "k": 8192, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/Kimi-K2.5-MXFP4/o_proj", + "moonshotai/Kimi-K2.5/o_proj", + "nvidia/Kimi-K2.5-NVFP4/o_proj", + "nvidia/Kimi-K2.6-NVFP4/o_proj" + ] }, { "type": "gemm", "args": { - "m": 2097152, - "n": 6144, - "k": 4096, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 49152, + "n": 7168, + "k": 9216, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/down_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/down_proj", + "nvidia/Kimi-K2.5-NVFP4/down_proj" + ] }, { "type": "gemm", "args": { - "m": 2097152, - "n": 3072, - "k": 6144, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 49152, + "n": 7168, + "k": 16384, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/o_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/o_proj" + ] }, { "type": "gemm", "args": { - "m": 2097152, - "n": 6144, - "k": 3072, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 49152, + "n": 7168, + "k": 16384, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/o_proj", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] }, { "type": "gemm", "args": { - "m": 4096, - "n": 8192, - "k": 2048, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 49152, + "n": 9216, + "k": 7168, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/gate_up_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/gate_up_proj", + "nvidia/Kimi-K2.5-NVFP4/gate_up_proj" + ] }, { "type": "gemm", "args": { - "m": 4096, - "n": 14336, - "k": 512, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 49152, + "n": 9216, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/gate_up_proj" + ] }, { "type": "gemm", "args": { - "m": 4096, - "n": 6144, - "k": 8192, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 49152, + "n": 12288, + "k": 1536, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/Kimi-K2.5-MXFP4/q_b_proj", + "moonshotai/Kimi-K2.5/q_b_proj", + "nvidia/DeepSeek-R1-0528-FP4-V2/q_b_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/q_b_proj", + "nvidia/Kimi-K2.5-NVFP4/q_b_proj", + "nvidia/Kimi-K2.6-NVFP4/q_b_proj" + ] }, { "type": "gemm", "args": { - "m": 4096, - "n": 6144, - "k": 6144, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 49152, + "n": 16384, + "k": 512, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/Kimi-K2.5-MXFP4/kv_b_proj", + "moonshotai/Kimi-K2.5/kv_b_proj", + "nvidia/DeepSeek-R1-0528-FP4-V2/kv_b_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/kv_b_proj", + "nvidia/Kimi-K2.5-NVFP4/kv_b_proj", + "nvidia/Kimi-K2.6-NVFP4/kv_b_proj" + ] }, { "type": "gemm", "args": { - "m": 4, - "n": 8192, - "k": 2048, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 49152, + "n": 24576, + "k": 1536, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/q_b_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/q_b_proj" + ] }, { "type": "gemm", "args": { - "m": 4, - "n": 14336, - "k": 512, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 49152, + "n": 24576, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/q_b_proj" + ] }, { "type": "gemm", "args": { - "m": 4, - "n": 6144, - "k": 8192, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 49152, + "n": 32768, + "k": 512, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/kv_b_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/kv_b_proj" + ] }, { "type": "gemm", "args": { - "m": 4, - "n": 6144, - "k": 6144, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 49152, + "n": 32768, + "k": 512, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/kv_b_proj" + ] }, { "type": "gemm", "args": { - "m": 8192, - "n": 8192, - "k": 2048, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 51200, + "n": 576, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/Kimi-K2.5-MXFP4/kv_a_proj_with_mqa", + "moonshotai/Kimi-K2.5/kv_a_proj_with_mqa", + "nvidia/DeepSeek-R1-0528-FP4-V2/kv_a_proj_with_mqa", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/kv_a_proj_with_mqa", + "nvidia/Kimi-K2.5-NVFP4/kv_a_proj_with_mqa", + "nvidia/Kimi-K2.6-NVFP4/kv_a_proj_with_mqa" + ] }, { "type": "gemm", "args": { - "m": 8192, - "n": 14336, - "k": 512, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 51200, + "n": 1536, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/Kimi-K2.5-MXFP4/q_a_proj", + "moonshotai/Kimi-K2.5/q_a_proj", + "nvidia/DeepSeek-R1-0528-FP4-V2/q_a_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/q_a_proj", + "nvidia/Kimi-K2.5-NVFP4/q_a_proj", + "nvidia/Kimi-K2.6-NVFP4/q_a_proj" + ] }, { "type": "gemm", "args": { - "m": 8192, - "n": 6144, - "k": 8192, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 51200, + "n": 4608, + "k": 7168, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/gate_up_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/gate_up_proj", + "nvidia/Kimi-K2.5-NVFP4/gate_up_proj" + ] }, { "type": "gemm", "args": { - "m": 8192, - "n": 6144, - "k": 6144, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 51200, + "n": 4608, + "k": 7168, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 32 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 1, + 32 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "amd/DeepSeek-R1-0528-MXFP4-Preview/gate_up_proj", + "amd/DeepSeek-R1-0528-MXFP4-v2/gate_up_proj", + "amd/DeepSeek-R1-0528-MXFP4/gate_up_proj", + "amd/Kimi-K2.5-MXFP4/gate_up_proj" + ] }, { "type": "gemm", "args": { - "m": 8, - "n": 8192, - "k": 2048, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 51200, + "n": 7168, + "k": 4608, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/down_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/down_proj", + "nvidia/Kimi-K2.5-NVFP4/down_proj" + ] }, { "type": "gemm", "args": { - "m": 8, - "n": 14336, - "k": 512, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 51200, + "n": 7168, + "k": 16384, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/o_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/o_proj" + ] }, { "type": "gemm", "args": { - "m": 8, - "n": 6144, - "k": 8192, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 51200, + "n": 24576, + "k": 1536, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/q_b_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/q_b_proj" + ] }, { "type": "gemm", "args": { - "m": 8, - "n": 6144, - "k": 6144, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 51200, + "n": 32768, + "k": 512, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/kv_b_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/kv_b_proj" + ] }, { "type": "gemm", "args": { - "m": 16384, - "n": 8192, - "k": 2048, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 56320, + "n": 576, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/Kimi-K2.5-MXFP4/kv_a_proj_with_mqa", + "moonshotai/Kimi-K2.5/kv_a_proj_with_mqa", + "nvidia/DeepSeek-R1-0528-FP4-V2/kv_a_proj_with_mqa", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/kv_a_proj_with_mqa", + "nvidia/Kimi-K2.5-NVFP4/kv_a_proj_with_mqa", + "nvidia/Kimi-K2.6-NVFP4/kv_a_proj_with_mqa" + ] }, { "type": "gemm", "args": { - "m": 16384, - "n": 14336, - "k": 512, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 56320, + "n": 1536, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/Kimi-K2.5-MXFP4/q_a_proj", + "moonshotai/Kimi-K2.5/q_a_proj", + "nvidia/DeepSeek-R1-0528-FP4-V2/q_a_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/q_a_proj", + "nvidia/Kimi-K2.5-NVFP4/q_a_proj", + "nvidia/Kimi-K2.6-NVFP4/q_a_proj" + ] }, { "type": "gemm", "args": { - "m": 16384, - "n": 6144, - "k": 8192, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 56320, + "n": 3072, + "k": 1536, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/Kimi-K2.5-MXFP4/q_b_proj", + "amd/MiniMax-M2.5-MXFP4/o_proj", + "moonshotai/Kimi-K2.5/q_b_proj", + "nvidia/DeepSeek-R1-0528-FP4-V2/q_b_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/q_b_proj", + "nvidia/Kimi-K2.5-NVFP4/q_b_proj", + "nvidia/Kimi-K2.6-NVFP4/q_b_proj", + "nvidia/MiniMax-M2.5-NVFP4/o_proj" + ] }, { "type": "gemm", "args": { - "m": 16384, - "n": 6144, - "k": 6144, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 56320, + "n": 3072, + "k": 1536, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/MiniMax-M2.5-NVFP4/experts.down_proj" + ] }, { "type": "gemm", "args": { - "m": 16, - "n": 8192, - "k": 2048, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 56320, + "n": 3072, + "k": 1536, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 32 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 1, + 32 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "amd/DeepSeek-R1-0528-MXFP4-Preview/q_b_proj", + "amd/DeepSeek-R1-0528-MXFP4-v2/q_b_proj", + "amd/DeepSeek-R1-0528-MXFP4/q_b_proj", + "amd/MiniMax-M2.5-MXFP4/experts.down_proj" + ] }, { "type": "gemm", "args": { - "m": 16, - "n": 14336, + "m": 56320, + "n": 4096, "k": 512, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "Qwen/Qwen3.5-397B-A17B/o_proj", + "Qwen/Qwen3.5-397B-A17B/shared_expert.down_proj", + "amd/Kimi-K2.5-MXFP4/kv_b_proj", + "amd/Qwen3.5-397B-A17B-MXFP4/o_proj", + "amd/Qwen3.5-397B-A17B-MXFP4/shared_expert.down_proj", + "moonshotai/Kimi-K2.5/kv_b_proj", + "nvidia/DeepSeek-R1-0528-FP4-V2/kv_b_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/kv_b_proj", + "nvidia/Kimi-K2.5-NVFP4/kv_b_proj", + "nvidia/Kimi-K2.6-NVFP4/kv_b_proj", + "nvidia/Qwen3.5-397B-A17B-NVFP4/o_proj", + "nvidia/Qwen3.5-397B-A17B-NVFP4/shared_expert.down_proj" + ] }, { "type": "gemm", "args": { - "m": 16, - "n": 6144, - "k": 8192, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 56320, + "n": 4096, + "k": 512, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 32 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 1, + 32 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "amd/DeepSeek-R1-0528-MXFP4-Preview/kv_b_proj", + "amd/DeepSeek-R1-0528-MXFP4-v2/kv_b_proj", + "amd/DeepSeek-R1-0528-MXFP4/kv_b_proj", + "amd/Qwen3.5-397B-A17B-MXFP4-AttnFP8-V2/shared_expert.down_proj" + ] }, { "type": "gemm", "args": { - "m": 16, - "n": 6144, - "k": 6144, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 56320, + "n": 4608, + "k": 7168, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/gate_up_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/gate_up_proj", + "nvidia/Kimi-K2.5-NVFP4/gate_up_proj" + ] }, { "type": "gemm", "args": { - "m": 32768, - "n": 8192, - "k": 2048, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 56320, + "n": 4608, + "k": 7168, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 32 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 1, + 32 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "amd/DeepSeek-R1-0528-MXFP4-Preview/gate_up_proj", + "amd/DeepSeek-R1-0528-MXFP4-v2/gate_up_proj", + "amd/DeepSeek-R1-0528-MXFP4/gate_up_proj", + "amd/Kimi-K2.5-MXFP4/gate_up_proj" + ] }, { "type": "gemm", "args": { - "m": 32768, - "n": 14336, - "k": 512, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 56320, + "n": 7168, + "k": 2048, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/Kimi-K2.5-MXFP4/o_proj", + "moonshotai/Kimi-K2.5/o_proj", + "nvidia/Kimi-K2.5-NVFP4/o_proj", + "nvidia/Kimi-K2.6-NVFP4/o_proj" + ] }, { "type": "gemm", "args": { - "m": 32768, - "n": 6144, - "k": 8192, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 56320, + "n": 7168, + "k": 2048, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/o_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/o_proj" + ] }, { "type": "gemm", "args": { - "m": 32768, - "n": 6144, - "k": 6144, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 56320, + "n": 7168, + "k": 2048, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 32 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 1, + 32 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "amd/DeepSeek-R1-0528-MXFP4-Preview/o_proj", + "amd/DeepSeek-R1-0528-MXFP4-v2/o_proj", + "amd/DeepSeek-R1-0528-MXFP4/o_proj" + ] }, { "type": "gemm", "args": { - "m": 32, - "n": 8192, - "k": 2048, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 56320, + "n": 7168, + "k": 4608, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/down_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/down_proj", + "nvidia/Kimi-K2.5-NVFP4/down_proj" + ] }, { "type": "gemm", "args": { - "m": 32, - "n": 14336, - "k": 512, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 56320, + "n": 7168, + "k": 4608, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 32 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 1, + 32 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "amd/DeepSeek-R1-0528-MXFP4-Preview/down_proj", + "amd/DeepSeek-R1-0528-MXFP4-v2/down_proj", + "amd/DeepSeek-R1-0528-MXFP4/down_proj", + "amd/Kimi-K2.5-MXFP4/down_proj" + ] }, { "type": "gemm", "args": { - "m": 32, - "n": 6144, - "k": 8192, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 56320, + "n": 7168, + "k": 9216, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/down_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/down_proj", + "nvidia/Kimi-K2.5-NVFP4/down_proj" + ] }, { "type": "gemm", "args": { - "m": 32, - "n": 6144, - "k": 6144, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 56320, + "n": 7168, + "k": 16384, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/o_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/o_proj" + ] }, { "type": "gemm", "args": { - "m": 65536, - "n": 8192, - "k": 2048, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 56320, + "n": 9216, + "k": 7168, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/gate_up_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/gate_up_proj", + "nvidia/Kimi-K2.5-NVFP4/gate_up_proj" + ] }, { "type": "gemm", "args": { - "m": 65536, - "n": 14336, - "k": 512, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 56320, + "n": 9216, + "k": 7168, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 32 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 1, + 32 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "amd/DeepSeek-R1-0528-MXFP4-Preview/gate_up_proj", + "amd/DeepSeek-R1-0528-MXFP4-v2/gate_up_proj", + "amd/DeepSeek-R1-0528-MXFP4/gate_up_proj", + "amd/Kimi-K2.5-MXFP4/gate_up_proj" + ] }, { "type": "gemm", "args": { - "m": 65536, - "n": 6144, - "k": 8192, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 56320, + "n": 24576, + "k": 1536, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/q_b_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/q_b_proj" + ] }, { "type": "gemm", "args": { - "m": 65536, - "n": 6144, - "k": 6144, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 56320, + "n": 32768, + "k": 512, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/kv_b_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/kv_b_proj" + ] }, { "type": "gemm", "args": { - "m": 64, - "n": 8192, - "k": 2048, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 59136, + "n": 576, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/kv_a_proj_with_mqa" + ] }, { "type": "gemm", "args": { - "m": 64, - "n": 14336, - "k": 512, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 59136, + "n": 1152, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/gate_up_proj" + ] }, { "type": "gemm", "args": { - "m": 64, - "n": 6144, - "k": 8192, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 59136, + "n": 1536, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/gate_up_proj", + "deepseek-ai/DeepSeek-R1-0528/q_a_proj", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w3" + ] }, { "type": "gemm", "args": { - "m": 64, - "n": 6144, - "k": 6144, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 59136, + "n": 7168, + "k": 1152, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/down_proj" + ] }, { "type": "gemm", "args": { - "m": 131072, - "n": 8192, - "k": 2048, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 59136, + "n": 7168, + "k": 16384, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/o_proj", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] }, { "type": "gemm", "args": { - "m": 131072, - "n": 14336, - "k": 512, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 59136, + "n": 24576, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/q_b_proj" + ] }, { "type": "gemm", "args": { - "m": 131072, - "n": 6144, - "k": 8192, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 59136, + "n": 32768, + "k": 512, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/kv_b_proj" + ] }, { "type": "gemm", "args": { - "m": 131072, - "n": 6144, - "k": 6144, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 61440, + "n": 576, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/Kimi-K2.5-MXFP4/kv_a_proj_with_mqa", + "moonshotai/Kimi-K2.5/kv_a_proj_with_mqa", + "nvidia/DeepSeek-R1-0528-FP4-V2/kv_a_proj_with_mqa", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/kv_a_proj_with_mqa", + "nvidia/Kimi-K2.5-NVFP4/kv_a_proj_with_mqa", + "nvidia/Kimi-K2.6-NVFP4/kv_a_proj_with_mqa" + ] }, { "type": "gemm", "args": { - "m": 128, - "n": 8192, - "k": 2048, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 61440, + "n": 1152, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/gate_up_proj" + ] }, { "type": "gemm", "args": { - "m": 128, - "n": 14336, - "k": 512, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 61440, + "n": 1536, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/q_b_proj" + ] }, { "type": "gemm", "args": { - "m": 128, - "n": 6144, - "k": 8192, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 61440, + "n": 1536, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/Kimi-K2.5-MXFP4/q_a_proj", + "moonshotai/Kimi-K2.5/q_a_proj", + "nvidia/DeepSeek-R1-0528-FP4-V2/q_a_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/q_a_proj", + "nvidia/Kimi-K2.5-NVFP4/q_a_proj", + "nvidia/Kimi-K2.6-NVFP4/q_a_proj" + ] }, { "type": "gemm", "args": { - "m": 128, - "n": 6144, - "k": 6144, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 61440, + "n": 2048, + "k": 512, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/kv_b_proj" + ] }, { "type": "gemm", "args": { - "m": 262144, - "n": 8192, - "k": 2048, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 61440, + "n": 2304, + "k": 7168, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/gate_up_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/gate_up_proj", + "nvidia/Kimi-K2.5-NVFP4/gate_up_proj" + ] }, { "type": "gemm", "args": { - "m": 262144, - "n": 14336, - "k": 512, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 61440, + "n": 3072, + "k": 1536, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/Kimi-K2.5-MXFP4/q_b_proj", + "amd/MiniMax-M2.5-MXFP4/o_proj", + "moonshotai/Kimi-K2.5/q_b_proj", + "nvidia/DeepSeek-R1-0528-FP4-V2/q_b_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/q_b_proj", + "nvidia/Kimi-K2.5-NVFP4/q_b_proj", + "nvidia/Kimi-K2.6-NVFP4/q_b_proj", + "nvidia/MiniMax-M2.5-NVFP4/o_proj" + ] }, { "type": "gemm", "args": { - "m": 262144, - "n": 6144, - "k": 8192, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 61440, + "n": 3072, + "k": 1536, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/MiniMax-M2.5-NVFP4/experts.down_proj" + ] }, { "type": "gemm", "args": { - "m": 262144, - "n": 6144, - "k": 6144, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 61440, + "n": 3072, + "k": 1536, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 32 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 1, + 32 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "amd/DeepSeek-R1-0528-MXFP4-Preview/q_b_proj", + "amd/DeepSeek-R1-0528-MXFP4-v2/q_b_proj", + "amd/DeepSeek-R1-0528-MXFP4/q_b_proj", + "amd/MiniMax-M2.5-MXFP4/experts.down_proj" + ] }, { "type": "gemm", "args": { - "m": 256, - "n": 8192, - "k": 2048, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 61440, + "n": 4096, + "k": 512, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "Qwen/Qwen3.5-397B-A17B/o_proj", + "Qwen/Qwen3.5-397B-A17B/shared_expert.down_proj", + "amd/Kimi-K2.5-MXFP4/kv_b_proj", + "amd/Qwen3.5-397B-A17B-MXFP4/o_proj", + "amd/Qwen3.5-397B-A17B-MXFP4/shared_expert.down_proj", + "moonshotai/Kimi-K2.5/kv_b_proj", + "nvidia/DeepSeek-R1-0528-FP4-V2/kv_b_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/kv_b_proj", + "nvidia/Kimi-K2.5-NVFP4/kv_b_proj", + "nvidia/Kimi-K2.6-NVFP4/kv_b_proj", + "nvidia/Qwen3.5-397B-A17B-NVFP4/o_proj", + "nvidia/Qwen3.5-397B-A17B-NVFP4/shared_expert.down_proj" + ] }, { "type": "gemm", "args": { - "m": 256, - "n": 14336, + "m": 61440, + "n": 4096, "k": 512, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 32 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 1, + 32 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "amd/DeepSeek-R1-0528-MXFP4-Preview/kv_b_proj", + "amd/DeepSeek-R1-0528-MXFP4-v2/kv_b_proj", + "amd/DeepSeek-R1-0528-MXFP4/kv_b_proj", + "amd/Qwen3.5-397B-A17B-MXFP4-AttnFP8-V2/shared_expert.down_proj" + ] }, { "type": "gemm", "args": { - "m": 256, - "n": 6144, - "k": 8192, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 61440, + "n": 4608, + "k": 7168, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/gate_up_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/gate_up_proj", + "nvidia/Kimi-K2.5-NVFP4/gate_up_proj" + ] }, { "type": "gemm", "args": { - "m": 256, - "n": 6144, - "k": 6144, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 61440, + "n": 4608, + "k": 7168, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 32 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 1, + 32 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "amd/DeepSeek-R1-0528-MXFP4-Preview/gate_up_proj", + "amd/DeepSeek-R1-0528-MXFP4-v2/gate_up_proj", + "amd/DeepSeek-R1-0528-MXFP4/gate_up_proj", + "amd/Kimi-K2.5-MXFP4/gate_up_proj" + ] }, { "type": "gemm", "args": { - "m": 524288, - "n": 8192, - "k": 2048, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 61440, + "n": 7168, + "k": 1024, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/o_proj", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] }, { "type": "gemm", "args": { - "m": 524288, - "n": 14336, - "k": 512, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 61440, + "n": 7168, + "k": 1152, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/down_proj" + ] }, { "type": "gemm", "args": { - "m": 524288, - "n": 6144, - "k": 8192, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 61440, + "n": 7168, + "k": 2048, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/Kimi-K2.5-MXFP4/o_proj", + "moonshotai/Kimi-K2.5/o_proj", + "nvidia/Kimi-K2.5-NVFP4/o_proj", + "nvidia/Kimi-K2.6-NVFP4/o_proj" + ] }, { "type": "gemm", "args": { - "m": 524288, - "n": 6144, - "k": 6144, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 61440, + "n": 7168, + "k": 2048, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/o_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/o_proj" + ] }, { "type": "gemm", "args": { - "m": 1048576, - "n": 8192, + "m": 61440, + "n": 7168, "k": 2048, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 32 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 1, + 32 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "amd/DeepSeek-R1-0528-MXFP4-Preview/o_proj", + "amd/DeepSeek-R1-0528-MXFP4-v2/o_proj", + "amd/DeepSeek-R1-0528-MXFP4/o_proj" + ] }, { "type": "gemm", "args": { - "m": 1048576, - "n": 14336, - "k": 512, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 61440, + "n": 7168, + "k": 2304, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/down_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/down_proj", + "nvidia/Kimi-K2.5-NVFP4/down_proj" + ] }, { "type": "gemm", "args": { - "m": 1048576, - "n": 6144, - "k": 8192, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 61440, + "n": 7168, + "k": 2304, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 32 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 1, + 32 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "amd/DeepSeek-R1-0528-MXFP4-Preview/down_proj", + "amd/DeepSeek-R1-0528-MXFP4-v2/down_proj", + "amd/DeepSeek-R1-0528-MXFP4/down_proj", + "amd/Kimi-K2.5-MXFP4/down_proj" + ] }, { "type": "gemm", "args": { - "m": 1048576, - "n": 6144, - "k": 6144, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 61440, + "n": 7168, + "k": 4608, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/down_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/down_proj", + "nvidia/Kimi-K2.5-NVFP4/down_proj" + ] }, { "type": "gemm", "args": { - "m": 2097152, - "n": 8192, - "k": 2048, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 61440, + "n": 7168, + "k": 8192, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/Kimi-K2.5-MXFP4/o_proj", + "moonshotai/Kimi-K2.5/o_proj", + "nvidia/Kimi-K2.5-NVFP4/o_proj", + "nvidia/Kimi-K2.6-NVFP4/o_proj" + ] }, { "type": "gemm", "args": { - "m": 2097152, - "n": 14336, - "k": 512, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 61440, + "n": 7168, + "k": 9216, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/down_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/down_proj", + "nvidia/Kimi-K2.5-NVFP4/down_proj" + ] }, { "type": "gemm", "args": { - "m": 2097152, - "n": 6144, - "k": 8192, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 61440, + "n": 7168, + "k": 16384, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/o_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/o_proj" + ] }, { "type": "gemm", "args": { - "m": 2097152, - "n": 6144, - "k": 6144, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 61440, + "n": 9216, + "k": 7168, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/gate_up_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/gate_up_proj", + "nvidia/Kimi-K2.5-NVFP4/gate_up_proj" + ] }, { "type": "gemm", "args": { - "m": 8192, - "n": 1536, + "m": 61440, + "n": 12288, "k": 1536, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/Kimi-K2.5-MXFP4/q_b_proj", + "moonshotai/Kimi-K2.5/q_b_proj", + "nvidia/DeepSeek-R1-0528-FP4-V2/q_b_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/q_b_proj", + "nvidia/Kimi-K2.5-NVFP4/q_b_proj", + "nvidia/Kimi-K2.6-NVFP4/q_b_proj" + ] }, { "type": "gemm", "args": { - "m": 8192, - "n": 2048, + "m": 61440, + "n": 16384, "k": 512, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/Kimi-K2.5-MXFP4/kv_b_proj", + "moonshotai/Kimi-K2.5/kv_b_proj", + "nvidia/DeepSeek-R1-0528-FP4-V2/kv_b_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/kv_b_proj", + "nvidia/Kimi-K2.5-NVFP4/kv_b_proj", + "nvidia/Kimi-K2.6-NVFP4/kv_b_proj" + ] }, { "type": "gemm", "args": { - "m": 8192, - "n": 7168, - "k": 1024, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 61440, + "n": 24576, + "k": 1536, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/q_b_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/q_b_proj" + ] }, { "type": "gemm", "args": { - "m": 8, - "n": 1536, - "k": 1536, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 61440, + "n": 32768, + "k": 512, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/kv_b_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/kv_b_proj" + ] }, { "type": "gemm", "args": { - "m": 8, - "n": 2048, - "k": 512, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 64512, + "n": 576, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/kv_a_proj_with_mqa" + ] }, { "type": "gemm", "args": { - "m": 8, - "n": 7168, - "k": 1024, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 64512, + "n": 1536, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/gate_up_proj", + "deepseek-ai/DeepSeek-R1-0528/q_a_proj", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w3" + ] }, { "type": "gemm", "args": { - "m": 16384, - "n": 1536, - "k": 1536, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 64512, + "n": 2304, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/gate_up_proj" + ] }, { "type": "gemm", "args": { - "m": 16384, - "n": 2048, - "k": 512, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 64512, + "n": 7168, + "k": 2304, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/down_proj" + ] }, { "type": "gemm", "args": { - "m": 16384, + "m": 64512, "n": 7168, - "k": 1024, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "k": 16384, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/o_proj", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] }, { "type": "gemm", "args": { - "m": 16, - "n": 1536, + "m": 64512, + "n": 24576, "k": 1536, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/q_b_proj" + ] }, { "type": "gemm", "args": { - "m": 16, - "n": 2048, + "m": 64512, + "n": 32768, "k": 512, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/kv_b_proj" + ] }, { "type": "gemm", "args": { - "m": 16, - "n": 7168, - "k": 1024, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 65536, + "n": 64, + "k": 2880, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/gpt-oss-120b-w-mxfp4-a-fp8/k_proj", + "amd/gpt-oss-120b-w-mxfp4-a-fp8/v_proj", + "openai/gpt-oss-120b/k_proj", + "openai/gpt-oss-120b/v_proj" + ] }, { "type": "gemm", "args": { - "m": 32, - "n": 1536, - "k": 1536, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 65536, + "n": 128, + "k": 2880, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/gpt-oss-120b-w-mxfp4-a-fp8/k_proj", + "amd/gpt-oss-120b-w-mxfp4-a-fp8/v_proj", + "openai/gpt-oss-120b/k_proj", + "openai/gpt-oss-120b/v_proj" + ] }, { "type": "gemm", "args": { - "m": 32, - "n": 2048, - "k": 512, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 65536, + "n": 128, + "k": 3072, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/MiniMax-M2.5-MXFP4/k_proj", + "amd/MiniMax-M2.5-MXFP4/v_proj", + "nvidia/MiniMax-M2.5-NVFP4/k_proj", + "nvidia/MiniMax-M2.5-NVFP4/v_proj" + ] }, { "type": "gemm", "args": { - "m": 32, - "n": 7168, - "k": 1024, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 65536, + "n": 128, + "k": 3072, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "MiniMaxAI/MiniMax-M2.5/k_proj", + "MiniMaxAI/MiniMax-M2.5/v_proj" + ] }, { "type": "gemm", "args": { "m": 65536, - "n": 1536, - "k": 1536, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "n": 128, + "k": 4096, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "Qwen/Qwen3.5-397B-A17B/shared_expert.gate_up_proj", + "amd/Qwen3.5-397B-A17B-MXFP4/shared_expert.gate_up_proj", + "nvidia/Qwen3.5-397B-A17B-NVFP4/shared_expert.gate_up_proj" + ] }, { "type": "gemm", "args": { "m": 65536, - "n": 2048, - "k": 512, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "n": 128, + "k": 4096, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "Qwen/Qwen3.5-397B-A17B-FP8/shared_expert.gate_up_proj" + ] }, { "type": "gemm", "args": { "m": 65536, - "n": 7168, - "k": 1024, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "n": 192, + "k": 3072, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/MiniMax-M2.5-NVFP4/experts.gate_up_proj" + ] }, { "type": "gemm", "args": { - "m": 64, - "n": 1536, - "k": 1536, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 65536, + "n": 192, + "k": 3072, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 32 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 1, + 32 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "amd/MiniMax-M2.5-MXFP4/experts.gate_up_proj" + ] }, { "type": "gemm", "args": { - "m": 64, - "n": 2048, - "k": 512, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 65536, + "n": 192, + "k": 3072, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "MiniMaxAI/MiniMax-M2.5/experts.gate_up_proj" + ] }, { "type": "gemm", "args": { - "m": 64, - "n": 7168, - "k": 1024, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 65536, + "n": 256, + "k": 2880, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/gpt-oss-120b-w-mxfp4-a-fp8/k_proj", + "amd/gpt-oss-120b-w-mxfp4-a-fp8/q_proj", + "amd/gpt-oss-120b-w-mxfp4-a-fp8/v_proj", + "openai/gpt-oss-120b/k_proj", + "openai/gpt-oss-120b/q_proj", + "openai/gpt-oss-120b/v_proj" + ] }, { "type": "gemm", "args": { - "m": 131072, - "n": 1536, - "k": 1536, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 65536, + "n": 256, + "k": 3072, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/MiniMax-M2.5-MXFP4/k_proj", + "amd/MiniMax-M2.5-MXFP4/v_proj", + "nvidia/MiniMax-M2.5-NVFP4/k_proj", + "nvidia/MiniMax-M2.5-NVFP4/v_proj" + ] }, { "type": "gemm", "args": { - "m": 131072, - "n": 2048, - "k": 512, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 65536, + "n": 256, + "k": 3072, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "MiniMaxAI/MiniMax-M2.5/k_proj", + "MiniMaxAI/MiniMax-M2.5/v_proj" + ] }, { "type": "gemm", "args": { - "m": 131072, - "n": 7168, - "k": 1024, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 65536, + "n": 256, + "k": 4096, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "Qwen/Qwen3.5-397B-A17B/k_proj", + "Qwen/Qwen3.5-397B-A17B/shared_expert.gate_up_proj", + "Qwen/Qwen3.5-397B-A17B/v_proj", + "amd/Qwen3.5-397B-A17B-MXFP4/k_proj", + "amd/Qwen3.5-397B-A17B-MXFP4/shared_expert.gate_up_proj", + "amd/Qwen3.5-397B-A17B-MXFP4/v_proj", + "nvidia/Qwen3.5-397B-A17B-NVFP4/k_proj", + "nvidia/Qwen3.5-397B-A17B-NVFP4/shared_expert.gate_up_proj", + "nvidia/Qwen3.5-397B-A17B-NVFP4/v_proj" + ] }, { "type": "gemm", "args": { - "m": 262144, - "n": 1536, - "k": 1536, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 65536, + "n": 256, + "k": 4096, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "Qwen/Qwen3.5-397B-A17B-FP8/k_proj", + "Qwen/Qwen3.5-397B-A17B-FP8/shared_expert.gate_up_proj", + "Qwen/Qwen3.5-397B-A17B-FP8/v_proj" + ] }, { "type": "gemm", "args": { - "m": 262144, - "n": 2048, - "k": 512, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 65536, + "n": 384, + "k": 3072, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 32 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 1, + 32 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "amd/MiniMax-M2.5-MXFP4/experts.gate_up_proj" + ] }, { "type": "gemm", "args": { - "m": 262144, - "n": 7168, - "k": 1024, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 65536, + "n": 384, + "k": 3072, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "MiniMaxAI/MiniMax-M2.5/experts.gate_up_proj", + "MiniMaxAI/MiniMax-M2.5/q_proj" + ] }, { "type": "gemm", "args": { - "m": 524288, - "n": 1536, - "k": 1536, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 65536, + "n": 512, + "k": 2880, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/gpt-oss-120b-w-mxfp4-a-fp8/k_proj", + "amd/gpt-oss-120b-w-mxfp4-a-fp8/q_proj", + "amd/gpt-oss-120b-w-mxfp4-a-fp8/v_proj", + "openai/gpt-oss-120b/k_proj", + "openai/gpt-oss-120b/q_proj", + "openai/gpt-oss-120b/v_proj" + ] }, { "type": "gemm", "args": { - "m": 524288, - "n": 2048, - "k": 512, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 65536, + "n": 512, + "k": 3072, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/MiniMax-M2.5-MXFP4/k_proj", + "amd/MiniMax-M2.5-MXFP4/v_proj", + "nvidia/MiniMax-M2.5-NVFP4/k_proj", + "nvidia/MiniMax-M2.5-NVFP4/v_proj" + ] }, { "type": "gemm", "args": { - "m": 524288, - "n": 7168, - "k": 1024, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 65536, + "n": 512, + "k": 3072, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "MiniMaxAI/MiniMax-M2.5/k_proj", + "MiniMaxAI/MiniMax-M2.5/v_proj" + ] }, { "type": "gemm", "args": { - "m": 128, - "n": 1536, - "k": 1536, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 65536, + "n": 512, + "k": 4096, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "Qwen/Qwen3.5-397B-A17B/k_proj", + "Qwen/Qwen3.5-397B-A17B/q_proj", + "Qwen/Qwen3.5-397B-A17B/shared_expert.gate_up_proj", + "Qwen/Qwen3.5-397B-A17B/v_proj", + "amd/Qwen3.5-397B-A17B-MXFP4/k_proj", + "amd/Qwen3.5-397B-A17B-MXFP4/q_proj", + "amd/Qwen3.5-397B-A17B-MXFP4/shared_expert.gate_up_proj", + "amd/Qwen3.5-397B-A17B-MXFP4/v_proj", + "nvidia/Qwen3.5-397B-A17B-NVFP4/k_proj", + "nvidia/Qwen3.5-397B-A17B-NVFP4/q_proj", + "nvidia/Qwen3.5-397B-A17B-NVFP4/shared_expert.gate_up_proj", + "nvidia/Qwen3.5-397B-A17B-NVFP4/v_proj" + ] }, { "type": "gemm", "args": { - "m": 128, - "n": 2048, - "k": 512, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 65536, + "n": 512, + "k": 4096, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "Qwen/Qwen3.5-397B-A17B-FP8/k_proj", + "Qwen/Qwen3.5-397B-A17B-FP8/q_proj", + "Qwen/Qwen3.5-397B-A17B-FP8/shared_expert.gate_up_proj", + "Qwen/Qwen3.5-397B-A17B-FP8/v_proj", + "deepseek-ai/DeepSeek-V4.1-Flash/attn.wo_a" + ] }, { "type": "gemm", "args": { - "m": 128, - "n": 7168, - "k": 1024, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 65536, + "n": 576, + "k": 6144, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/GLM-5.1-MXFP4/kv_a_proj_with_mqa", + "amd/GLM-5.2-MXFP4/kv_a_proj_with_mqa", + "nvidia/GLM-5-NVFP4/kv_a_proj_with_mqa", + "nvidia/GLM-5.1-NVFP4/kv_a_proj_with_mqa", + "nvidia/GLM-5.2-NVFP4/kv_a_proj_with_mqa" + ] }, { "type": "gemm", "args": { - "m": 1048576, - "n": 1536, - "k": 1536, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 65536, + "n": 576, + "k": 6144, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "zai-org/GLM-5-FP8/kv_a_proj_with_mqa", + "zai-org/GLM-5.1-FP8/kv_a_proj_with_mqa", + "zai-org/GLM-5.2-FP8/kv_a_proj_with_mqa" + ] }, { "type": "gemm", "args": { - "m": 1048576, - "n": 2048, - "k": 512, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 65536, + "n": 576, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/Kimi-K2.5-MXFP4/kv_a_proj_with_mqa", + "moonshotai/Kimi-K2.5/kv_a_proj_with_mqa", + "nvidia/DeepSeek-R1-0528-FP4-V2/kv_a_proj_with_mqa", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/kv_a_proj_with_mqa", + "nvidia/Kimi-K2.5-NVFP4/kv_a_proj_with_mqa", + "nvidia/Kimi-K2.6-NVFP4/kv_a_proj_with_mqa" + ] }, { "type": "gemm", "args": { - "m": 1048576, - "n": 7168, - "k": 1024, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 65536, + "n": 576, + "k": 7168, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 32 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 1, + 32 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "amd/DeepSeek-R1-0528-MXFP4-Preview/kv_a_proj_with_mqa", + "amd/DeepSeek-R1-0528-MXFP4-v2/kv_a_proj_with_mqa", + "amd/DeepSeek-R1-0528-MXFP4/kv_a_proj_with_mqa" + ] }, { "type": "gemm", "args": { - "m": 2048, - "n": 3072, - "k": 3072, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 65536, + "n": 576, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/kv_a_proj_with_mqa" + ] }, { "type": "gemm", "args": { - "m": 2048, - "n": 512, + "m": 65536, + "n": 768, "k": 3072, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/MiniMax-M2.5-MXFP4/q_proj", + "nvidia/MiniMax-M2.5-NVFP4/q_proj" + ] }, { "type": "gemm", "args": { - "m": 2048, + "m": 65536, "n": 768, "k": 3072, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/MiniMax-M2.5-NVFP4/experts.gate_up_proj" + ] }, { "type": "gemm", "args": { - "m": 2048, - "n": 3072, - "k": 768, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 65536, + "n": 768, + "k": 3072, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 32 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 1, + 32 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "amd/MiniMax-M2.5-MXFP4/experts.gate_up_proj" + ] }, { "type": "gemm", "args": { - "m": 2, - "n": 3072, + "m": 65536, + "n": 768, "k": 3072, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "MiniMaxAI/MiniMax-M2.5/experts.gate_up_proj", + "MiniMaxAI/MiniMax-M2.5/q_proj" + ] }, { "type": "gemm", "args": { - "m": 2, - "n": 512, - "k": 3072, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 65536, + "n": 1024, + "k": 2880, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/gpt-oss-120b-w-mxfp4-a-fp8/q_proj", + "openai/gpt-oss-120b/q_proj" + ] }, { "type": "gemm", "args": { - "m": 2, - "n": 768, + "m": 65536, + "n": 1024, "k": 3072, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/MiniMax-M2.5-MXFP4/k_proj", + "amd/MiniMax-M2.5-MXFP4/qkv_proj", + "amd/MiniMax-M2.5-MXFP4/v_proj", + "nvidia/MiniMax-M2.5-NVFP4/k_proj", + "nvidia/MiniMax-M2.5-NVFP4/qkv_proj", + "nvidia/MiniMax-M2.5-NVFP4/v_proj" + ] }, { "type": "gemm", "args": { - "m": 2, - "n": 3072, - "k": 768, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 65536, + "n": 1024, + "k": 3072, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "MiniMaxAI/MiniMax-M2.5/k_proj", + "MiniMaxAI/MiniMax-M2.5/qkv_proj", + "MiniMaxAI/MiniMax-M2.5/v_proj" + ] }, { "type": "gemm", "args": { - "m": 4096, - "n": 3072, - "k": 3072, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 65536, + "n": 1024, + "k": 4096, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "Qwen/Qwen3.5-397B-A17B/q_proj", + "Qwen/Qwen3.5-397B-A17B/shared_expert.gate_up_proj", + "amd/Qwen3.5-397B-A17B-MXFP4/q_proj", + "amd/Qwen3.5-397B-A17B-MXFP4/shared_expert.gate_up_proj", + "nvidia/Qwen3.5-397B-A17B-NVFP4/q_proj", + "nvidia/Qwen3.5-397B-A17B-NVFP4/shared_expert.gate_up_proj" + ] }, { "type": "gemm", "args": { - "m": 4096, - "n": 512, - "k": 3072, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 65536, + "n": 1024, + "k": 4096, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "Qwen/Qwen3.5-397B-A17B-FP8/q_proj", + "Qwen/Qwen3.5-397B-A17B-FP8/shared_expert.gate_up_proj", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_a", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_a", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_a" + ] }, { "type": "gemm", "args": { - "m": 8192, - "n": 3072, - "k": 3072, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 65536, + "n": 1088, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] }, { "type": "gemm", "args": { - "m": 8192, - "n": 512, - "k": 3072, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 65536, + "n": 1152, + "k": 7168, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-NVFP4-v2/gate_up_proj" + ] }, { "type": "gemm", "args": { - "m": 16384, - "n": 3072, - "k": 3072, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 65536, + "n": 1152, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/gate_up_proj" + ] }, { "type": "gemm", "args": { - "m": 16384, - "n": 512, - "k": 3072, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 65536, + "n": 1536, + "k": 1536, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/Kimi-K2.5-MXFP4/q_b_proj", + "moonshotai/Kimi-K2.5/q_b_proj", + "nvidia/DeepSeek-R1-0528-FP4-V2/q_b_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/q_b_proj", + "nvidia/Kimi-K2.5-NVFP4/q_b_proj", + "nvidia/Kimi-K2.6-NVFP4/q_b_proj" + ] }, { "type": "gemm", "args": { - "m": 2048, - "n": 128, - "k": 3072, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 65536, + "n": 1536, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/q_b_proj" + ] }, { "type": "gemm", "args": { - "m": 2048, - "n": 192, + "m": 65536, + "n": 1536, "k": 3072, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/MiniMax-M2.5-MXFP4/q_proj", + "nvidia/MiniMax-M2.5-NVFP4/q_proj" + ] }, { "type": "gemm", "args": { - "m": 2048, - "n": 3072, - "k": 192, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 65536, + "n": 1536, + "k": 3072, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 32 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 1, + 32 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "amd/MiniMax-M2.5-MXFP4/experts.gate_up_proj" + ] }, { "type": "gemm", "args": { - "m": 2, - "n": 128, + "m": 65536, + "n": 1536, "k": 3072, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "MiniMaxAI/MiniMax-M2.5/experts.gate_up_proj", + "MiniMaxAI/MiniMax-M2.5/q_proj" + ] }, { "type": "gemm", "args": { - "m": 2, - "n": 192, - "k": 3072, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 65536, + "n": 1536, + "k": 6144, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "zai-org/GLM-5-FP8/gate_up_proj", + "zai-org/GLM-5.1-FP8/gate_up_proj", + "zai-org/GLM-5.2-FP8/gate_up_proj" + ] }, { "type": "gemm", "args": { - "m": 2, - "n": 3072, - "k": 192, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 65536, + "n": 1536, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/Kimi-K2.5-MXFP4/q_a_proj", + "moonshotai/Kimi-K2.5/q_a_proj", + "nvidia/DeepSeek-R1-0528-FP4-V2/q_a_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/q_a_proj", + "nvidia/Kimi-K2.5-NVFP4/q_a_proj", + "nvidia/Kimi-K2.6-NVFP4/q_a_proj" + ] }, { "type": "gemm", "args": { - "m": 512, + "m": 65536, "n": 1536, - "k": 3072, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "k": 7168, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 32 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 1, + 32 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "amd/DeepSeek-R1-0528-MXFP4-Preview/q_a_proj", + "amd/DeepSeek-R1-0528-MXFP4-v2/q_a_proj", + "amd/DeepSeek-R1-0528-MXFP4/q_a_proj" + ] }, { "type": "gemm", "args": { - "m": 1024, + "m": 65536, "n": 1536, - "k": 3072, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/gate_up_proj", + "deepseek-ai/DeepSeek-R1-0528/q_a_proj", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w3" + ] }, { "type": "gemm", "args": { - "m": 4096, - "n": 3072, - "k": 3072, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 65536, + "n": 1536, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w3" + ] }, { "type": "gemm", "args": { - "m": 4096, - "n": 512, - "k": 3072, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 65536, + "n": 2048, + "k": 512, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/Kimi-K2.5-MXFP4/kv_b_proj", + "moonshotai/Kimi-K2.5/kv_b_proj", + "nvidia/DeepSeek-R1-0528-FP4-V2/kv_b_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/kv_b_proj", + "nvidia/Kimi-K2.5-NVFP4/kv_b_proj", + "nvidia/Kimi-K2.6-NVFP4/kv_b_proj" + ] }, { "type": "gemm", "args": { - "m": 4, - "n": 3072, - "k": 3072, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 65536, + "n": 2048, + "k": 512, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/kv_b_proj" + ] }, { "type": "gemm", "args": { - "m": 4, - "n": 512, - "k": 3072, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 65536, + "n": 2048, + "k": 2048, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "zai-org/GLM-5-FP8/q_b_proj", + "zai-org/GLM-5.1-FP8/q_b_proj", + "zai-org/GLM-5.2-FP8/q_b_proj" + ] }, { "type": "gemm", "args": { - "m": 8192, - "n": 3072, - "k": 3072, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 65536, + "n": 2048, + "k": 2880, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/gpt-oss-120b-w-mxfp4-a-fp8/q_proj", + "openai/gpt-oss-120b/q_proj" + ] }, { "type": "gemm", "args": { - "m": 8192, - "n": 512, - "k": 3072, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 65536, + "n": 2048, + "k": 4096, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "Qwen/Qwen3.5-397B-A17B/q_proj", + "Qwen/Qwen3.5-397B-A17B/shared_expert.gate_up_proj", + "amd/Qwen3.5-397B-A17B-MXFP4/q_proj", + "amd/Qwen3.5-397B-A17B-MXFP4/shared_expert.gate_up_proj", + "nvidia/Qwen3.5-397B-A17B-NVFP4/q_proj", + "nvidia/Qwen3.5-397B-A17B-NVFP4/shared_expert.gate_up_proj" + ] }, { "type": "gemm", "args": { - "m": 8, - "n": 3072, - "k": 3072, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 65536, + "n": 2048, + "k": 4096, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "Qwen/Qwen3.5-397B-A17B-FP8/q_proj", + "Qwen/Qwen3.5-397B-A17B-FP8/shared_expert.gate_up_proj" + ] }, { "type": "gemm", "args": { - "m": 8, - "n": 512, - "k": 3072, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 65536, + "n": 2048, + "k": 6144, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/GLM-5.1-MXFP4/q_a_proj", + "amd/GLM-5.2-MXFP4/q_a_proj", + "amd/MiniMax-M3-MXFP4/q_proj", + "nvidia/GLM-5-NVFP4/q_a_proj", + "nvidia/GLM-5.1-NVFP4/q_a_proj", + "nvidia/GLM-5.2-NVFP4/q_a_proj" + ] }, { "type": "gemm", "args": { - "m": 32768, - "n": 3072, - "k": 3072, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 65536, + "n": 2048, + "k": 6144, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "zai-org/GLM-5-FP8/q_a_proj", + "zai-org/GLM-5.1-FP8/q_a_proj", + "zai-org/GLM-5.2-FP8/q_a_proj" + ] }, { "type": "gemm", "args": { - "m": 32768, - "n": 512, - "k": 3072, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 65536, + "n": 2304, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/gate_up_proj" + ] }, { "type": "gemm", "args": { - "m": 32, - "n": 3072, - "k": 3072, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 65536, + "n": 2880, + "k": 512, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/gpt-oss-120b-w-mxfp4-a-fp8/o_proj", + "openai/gpt-oss-120b/o_proj" + ] }, { "type": "gemm", "args": { - "m": 32, - "n": 512, - "k": 3072, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 65536, + "n": 2880, + "k": 1024, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/gpt-oss-120b-w-mxfp4-a-fp8/o_proj", + "openai/gpt-oss-120b/o_proj" + ] }, { "type": "gemm", "args": { - "m": 32, - "n": 768, - "k": 3072, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 65536, + "n": 2880, + "k": 2048, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/gpt-oss-120b-w-mxfp4-a-fp8/o_proj", + "openai/gpt-oss-120b/o_proj" + ] }, { "type": "gemm", "args": { - "m": 32, - "n": 3072, - "k": 768, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 65536, + "n": 2880, + "k": 4096, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/gpt-oss-120b-w-mxfp4-a-fp8/o_proj", + "openai/gpt-oss-120b/o_proj" + ] }, { "type": "gemm", "args": { "m": 65536, "n": 3072, - "k": 3072, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "k": 192, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 32 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 1, + 32 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "amd/MiniMax-M2.5-MXFP4/experts.down_proj" + ] }, { "type": "gemm", "args": { "m": 65536, - "n": 512, - "k": 3072, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "n": 3072, + "k": 192, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "MiniMaxAI/MiniMax-M2.5/experts.down_proj" + ] }, { "type": "gemm", "args": { "m": 65536, - "n": 768, - "k": 3072, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "n": 3072, + "k": 384, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/MiniMax-M2.5-NVFP4/experts.down_proj" + ] }, { "type": "gemm", "args": { "m": 65536, "n": 3072, - "k": 768, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "k": 384, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 32 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 1, + 32 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "amd/MiniMax-M2.5-MXFP4/experts.down_proj" + ] }, { "type": "gemm", "args": { - "m": 64, + "m": 65536, "n": 3072, - "k": 3072, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "k": 384, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "MiniMaxAI/MiniMax-M2.5/experts.down_proj", + "MiniMaxAI/MiniMax-M2.5/o_proj" + ] }, { "type": "gemm", "args": { - "m": 64, - "n": 512, - "k": 3072, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 65536, + "n": 3072, + "k": 768, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/MiniMax-M2.5-MXFP4/o_proj", + "nvidia/MiniMax-M2.5-NVFP4/o_proj" + ] }, { "type": "gemm", "args": { - "m": 64, - "n": 768, - "k": 3072, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 65536, + "n": 3072, + "k": 768, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 32 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 1, + 32 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "amd/MiniMax-M2.5-MXFP4/experts.down_proj" + ] }, { "type": "gemm", "args": { - "m": 64, + "m": 65536, "n": 3072, "k": 768, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "MiniMaxAI/MiniMax-M2.5/experts.down_proj", + "MiniMaxAI/MiniMax-M2.5/o_proj" + ] }, { "type": "gemm", "args": { - "m": 262144, + "m": 65536, "n": 3072, - "k": 3072, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "k": 1536, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/Kimi-K2.5-MXFP4/q_b_proj", + "amd/MiniMax-M2.5-MXFP4/o_proj", + "moonshotai/Kimi-K2.5/q_b_proj", + "nvidia/DeepSeek-R1-0528-FP4-V2/q_b_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/q_b_proj", + "nvidia/Kimi-K2.5-NVFP4/q_b_proj", + "nvidia/Kimi-K2.6-NVFP4/q_b_proj", + "nvidia/MiniMax-M2.5-NVFP4/o_proj" + ] }, { "type": "gemm", "args": { - "m": 262144, - "n": 512, - "k": 3072, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 65536, + "n": 3072, + "k": 1536, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/MiniMax-M2.5-NVFP4/experts.down_proj" + ] }, { "type": "gemm", "args": { - "m": 524288, + "m": 65536, "n": 3072, - "k": 3072, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "k": 1536, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 32 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 1, + 32 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "amd/DeepSeek-R1-0528-MXFP4-Preview/q_b_proj", + "amd/DeepSeek-R1-0528-MXFP4-v2/q_b_proj", + "amd/DeepSeek-R1-0528-MXFP4/q_b_proj", + "amd/MiniMax-M2.5-MXFP4/experts.down_proj" + ] }, { "type": "gemm", "args": { - "m": 524288, - "n": 512, - "k": 3072, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 65536, + "n": 3072, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "MiniMaxAI/MiniMax-M2.5/experts.down_proj", + "MiniMaxAI/MiniMax-M2.5/o_proj", + "deepseek-ai/DeepSeek-R1-0528/q_b_proj" + ] }, { "type": "gemm", "args": { - "m": 1024, + "m": 65536, "n": 3072, "k": 3072, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/MiniMax-M2.5-MXFP4/o_proj", + "amd/MiniMax-M2.5-MXFP4/q_proj", + "nvidia/MiniMax-M2.5-NVFP4/o_proj", + "nvidia/MiniMax-M2.5-NVFP4/q_proj" + ] }, { "type": "gemm", "args": { - "m": 1024, - "n": 512, + "m": 65536, + "n": 3072, "k": 3072, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/MiniMax-M2.5-NVFP4/experts.gate_up_proj" + ] }, { "type": "gemm", "args": { - "m": 32, - "n": 256, + "m": 65536, + "n": 3072, "k": 3072, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 32 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 1, + 32 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "amd/MiniMax-M2.5-MXFP4/experts.gate_up_proj" + ] }, { "type": "gemm", "args": { - "m": 32, - "n": 384, + "m": 65536, + "n": 3072, "k": 3072, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "MiniMaxAI/MiniMax-M2.5/experts.gate_up_proj", + "MiniMaxAI/MiniMax-M2.5/o_proj", + "MiniMaxAI/MiniMax-M2.5/q_proj" + ] }, { "type": "gemm", "args": { - "m": 32, + "m": 65536, "n": 3072, - "k": 384, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "k": 6144, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/GLM-5.1-MXFP4/gate_up_proj", + "amd/GLM-5.2-MXFP4/gate_up_proj", + "amd/MiniMax-M2.5-MXFP4/o_proj", + "amd/MiniMax-M3-MXFP4/gate_up_proj", + "nvidia/GLM-5.1-NVFP4/gate_up_proj", + "nvidia/GLM-5.2-NVFP4/gate_up_proj", + "nvidia/MiniMax-M2.5-NVFP4/o_proj" + ] }, { "type": "gemm", "args": { - "m": 16384, - "n": 128, - "k": 3072, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 65536, + "n": 3072, + "k": 6144, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "MiniMaxAI/MiniMax-M2.5/o_proj", + "zai-org/GLM-5-FP8/gate_up_proj", + "zai-org/GLM-5.1-FP8/gate_up_proj", + "zai-org/GLM-5.2-FP8/gate_up_proj" + ] }, { "type": "gemm", "args": { - "m": 16384, - "n": 192, - "k": 3072, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 65536, + "n": 3584, + "k": 512, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "zai-org/GLM-5-FP8/kv_b_proj", + "zai-org/GLM-5.1-FP8/kv_b_proj", + "zai-org/GLM-5.2-FP8/kv_b_proj" + ] }, { "type": "gemm", "args": { - "m": 16384, - "n": 3072, - "k": 192, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 65536, + "n": 4096, + "k": 128, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "Qwen/Qwen3.5-397B-A17B/shared_expert.down_proj", + "amd/Qwen3.5-397B-A17B-MXFP4/shared_expert.down_proj", + "nvidia/Qwen3.5-397B-A17B-NVFP4/shared_expert.down_proj" + ] }, { "type": "gemm", "args": { - "m": 16, - "n": 128, - "k": 3072, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 65536, + "n": 4096, + "k": 128, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "Qwen/Qwen3.5-397B-A17B-FP8/shared_expert.down_proj" + ] }, { "type": "gemm", "args": { - "m": 16, - "n": 192, - "k": 3072, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 65536, + "n": 4096, + "k": 256, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "Qwen/Qwen3.5-397B-A17B/shared_expert.down_proj", + "amd/Qwen3.5-397B-A17B-MXFP4/shared_expert.down_proj", + "nvidia/Qwen3.5-397B-A17B-NVFP4/shared_expert.down_proj" + ] }, { "type": "gemm", "args": { - "m": 16, - "n": 3072, - "k": 192, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 65536, + "n": 4096, + "k": 256, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "Qwen/Qwen3.5-397B-A17B-FP8/shared_expert.down_proj" + ] }, { "type": "gemm", "args": { - "m": 4194304, - "n": 6144, - "k": 3072, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 65536, + "n": 4096, + "k": 512, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "Qwen/Qwen3.5-397B-A17B/o_proj", + "Qwen/Qwen3.5-397B-A17B/shared_expert.down_proj", + "amd/Kimi-K2.5-MXFP4/kv_b_proj", + "amd/Qwen3.5-397B-A17B-MXFP4/o_proj", + "amd/Qwen3.5-397B-A17B-MXFP4/shared_expert.down_proj", + "moonshotai/Kimi-K2.5/kv_b_proj", + "nvidia/DeepSeek-R1-0528-FP4-V2/kv_b_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/kv_b_proj", + "nvidia/Kimi-K2.5-NVFP4/kv_b_proj", + "nvidia/Kimi-K2.6-NVFP4/kv_b_proj", + "nvidia/Qwen3.5-397B-A17B-NVFP4/o_proj", + "nvidia/Qwen3.5-397B-A17B-NVFP4/shared_expert.down_proj" + ] }, { "type": "gemm", "args": { - "m": 4194304, - "n": 1024, - "k": 3072, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 65536, + "n": 4096, + "k": 512, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 32 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 1, + 32 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "amd/DeepSeek-R1-0528-MXFP4-Preview/kv_b_proj", + "amd/DeepSeek-R1-0528-MXFP4-v2/kv_b_proj", + "amd/DeepSeek-R1-0528-MXFP4/kv_b_proj", + "amd/Qwen3.5-397B-A17B-MXFP4-AttnFP8-V2/shared_expert.down_proj" + ] }, { "type": "gemm", "args": { - "m": 4194304, - "n": 3072, - "k": 6144, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 65536, + "n": 4096, + "k": 512, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "Qwen/Qwen3.5-397B-A17B-FP8/o_proj", + "Qwen/Qwen3.5-397B-A17B-FP8/shared_expert.down_proj", + "deepseek-ai/DeepSeek-R1-0528/kv_b_proj" + ] }, { "type": "gemm", "args": { - "m": 4194304, - "n": 1536, - "k": 3072, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 65536, + "n": 4096, + "k": 1024, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "Qwen/Qwen3.5-397B-A17B/o_proj", + "Qwen/Qwen3.5-397B-A17B/shared_expert.down_proj", + "amd/Qwen3.5-397B-A17B-MXFP4/o_proj", + "amd/Qwen3.5-397B-A17B-MXFP4/shared_expert.down_proj", + "nvidia/Qwen3.5-397B-A17B-NVFP4/o_proj", + "nvidia/Qwen3.5-397B-A17B-NVFP4/shared_expert.down_proj" + ] }, { "type": "gemm", "args": { - "m": 8388608, - "n": 6144, - "k": 3072, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 65536, + "n": 4096, + "k": 1024, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "Qwen/Qwen3.5-397B-A17B-FP8/o_proj", + "Qwen/Qwen3.5-397B-A17B-FP8/shared_expert.down_proj" + ] }, { "type": "gemm", "args": { - "m": 8388608, - "n": 1024, - "k": 3072, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 65536, + "n": 4096, + "k": 2048, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "Qwen/Qwen3.5-397B-A17B/o_proj", + "amd/GLM-5.1-MXFP4/q_b_proj", + "amd/GLM-5.2-MXFP4/q_b_proj", + "amd/Qwen3.5-397B-A17B-MXFP4/o_proj", + "nvidia/GLM-5-NVFP4/q_b_proj", + "nvidia/GLM-5.1-NVFP4/q_b_proj", + "nvidia/GLM-5.2-NVFP4/q_b_proj", + "nvidia/Qwen3.5-397B-A17B-NVFP4/o_proj" + ] }, { "type": "gemm", "args": { - "m": 8388608, - "n": 3072, - "k": 6144, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 65536, + "n": 4096, + "k": 2048, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "Qwen/Qwen3.5-397B-A17B-FP8/o_proj", + "zai-org/GLM-5-FP8/q_b_proj", + "zai-org/GLM-5.1-FP8/q_b_proj", + "zai-org/GLM-5.2-FP8/q_b_proj" + ] }, { "type": "gemm", "args": { - "m": 8388608, - "n": 1536, - "k": 3072, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 65536, + "n": 4096, + "k": 2880, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/gpt-oss-120b-w-mxfp4-a-fp8/q_proj", + "openai/gpt-oss-120b/q_proj" + ] }, { "type": "gemm", "args": { - "m": 8388608, - "n": 3072, - "k": 1536, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 65536, + "n": 4096, + "k": 4096, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "Qwen/Qwen3.5-397B-A17B/o_proj", + "Qwen/Qwen3.5-397B-A17B/q_proj", + "amd/Qwen3.5-397B-A17B-MXFP4/o_proj", + "amd/Qwen3.5-397B-A17B-MXFP4/q_proj", + "nvidia/Qwen3.5-397B-A17B-NVFP4/o_proj", + "nvidia/Qwen3.5-397B-A17B-NVFP4/q_proj" + ] }, { "type": "gemm", "args": { - "m": 8388608, - "n": 3072, - "k": 3072, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 65536, + "n": 4096, + "k": 4096, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "Qwen/Qwen3.5-397B-A17B-FP8/o_proj", + "Qwen/Qwen3.5-397B-A17B-FP8/q_proj" + ] }, { "type": "gemm", "args": { - "m": 8388608, - "n": 512, - "k": 3072, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 65536, + "n": 4608, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "moonshotai/Kimi-K2.5/gate_up_proj", + "nvidia/Kimi-K2.6-NVFP4/gate_up_proj" + ] }, { "type": "gemm", "args": { - "m": 8388608, - "n": 768, - "k": 3072, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 65536, + "n": 4608, + "k": 7168, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/gate_up_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/gate_up_proj", + "nvidia/Kimi-K2.5-NVFP4/gate_up_proj" + ] }, { "type": "gemm", "args": { - "m": 8388608, - "n": 3072, - "k": 768, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 65536, + "n": 4608, + "k": 7168, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 32 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 1, + 32 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "amd/DeepSeek-R1-0528-MXFP4-Preview/gate_up_proj", + "amd/DeepSeek-R1-0528-MXFP4-v2/gate_up_proj", + "amd/DeepSeek-R1-0528-MXFP4/gate_up_proj", + "amd/Kimi-K2.5-MXFP4/gate_up_proj" + ] }, { "type": "gemm", "args": { - "m": 262144, - "n": 256, - "k": 3072, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 65536, + "n": 4608, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/gate_up_proj" + ] }, { "type": "gemm", "args": { - "m": 262144, - "n": 384, - "k": 3072, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 65536, + "n": 6144, + "k": 1536, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/GLM-5.1-MXFP4/down_proj", + "amd/GLM-5.2-MXFP4/down_proj", + "amd/Kimi-K2.5-MXFP4/q_b_proj", + "amd/MiniMax-M3-MXFP4/down_proj", + "moonshotai/Kimi-K2.5/q_b_proj", + "nvidia/DeepSeek-R1-0528-FP4-V2/q_b_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/q_b_proj", + "nvidia/GLM-5.1-NVFP4/down_proj", + "nvidia/GLM-5.2-NVFP4/down_proj", + "nvidia/Kimi-K2.5-NVFP4/q_b_proj", + "nvidia/Kimi-K2.6-NVFP4/q_b_proj" + ] }, { "type": "gemm", "args": { - "m": 262144, - "n": 3072, - "k": 384, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 65536, + "n": 6144, + "k": 1536, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 32 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 1, + 32 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "amd/DeepSeek-R1-0528-MXFP4-Preview/q_b_proj", + "amd/DeepSeek-R1-0528-MXFP4-v2/q_b_proj", + "amd/DeepSeek-R1-0528-MXFP4/q_b_proj" + ] }, { "type": "gemm", "args": { - "m": 524288, - "n": 256, - "k": 3072, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 65536, + "n": 6144, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/q_b_proj", + "zai-org/GLM-5-FP8/down_proj", + "zai-org/GLM-5.1-FP8/down_proj", + "zai-org/GLM-5.2-FP8/down_proj" + ] }, { "type": "gemm", "args": { - "m": 524288, - "n": 384, - "k": 3072, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 65536, + "n": 6144, + "k": 2048, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "zai-org/GLM-5-FP8/o_proj", + "zai-org/GLM-5.1-FP8/o_proj", + "zai-org/GLM-5.2-FP8/o_proj" + ] }, { "type": "gemm", "args": { - "m": 524288, - "n": 3072, - "k": 384, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 65536, + "n": 6144, + "k": 3072, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/GLM-5.1-MXFP4/down_proj", + "amd/GLM-5.2-MXFP4/down_proj", + "amd/MiniMax-M2.5-MXFP4/q_proj", + "amd/MiniMax-M3-MXFP4/down_proj", + "nvidia/GLM-5.1-NVFP4/down_proj", + "nvidia/GLM-5.2-NVFP4/down_proj", + "nvidia/MiniMax-M2.5-NVFP4/q_proj" + ] }, { "type": "gemm", "args": { - "m": 1048576, - "n": 256, + "m": 65536, + "n": 6144, "k": 3072, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/GLM-5-NVFP4/down_proj" + ] }, { "type": "gemm", "args": { - "m": 1048576, - "n": 384, + "m": 65536, + "n": 6144, "k": 3072, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "MiniMaxAI/MiniMax-M2.5/q_proj", + "zai-org/GLM-5-FP8/down_proj", + "zai-org/GLM-5.1-FP8/down_proj", + "zai-org/GLM-5.2-FP8/down_proj" + ] }, { "type": "gemm", "args": { - "m": 1048576, - "n": 3072, - "k": 384, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 65536, + "n": 6144, + "k": 4096, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/GLM-5.1-MXFP4/o_proj", + "amd/GLM-5.2-MXFP4/o_proj", + "amd/MiniMax-M3-MXFP4/o_proj", + "nvidia/GLM-5-NVFP4/o_proj", + "nvidia/GLM-5.1-NVFP4/o_proj", + "nvidia/GLM-5.2-NVFP4/o_proj" + ] }, { "type": "gemm", "args": { "m": 65536, - "n": 128, - "k": 3072, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "n": 6144, + "k": 4096, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "zai-org/GLM-5-FP8/o_proj", + "zai-org/GLM-5.1-FP8/o_proj", + "zai-org/GLM-5.2-FP8/o_proj" + ] }, { "type": "gemm", "args": { "m": 65536, - "n": 192, - "k": 3072, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "n": 6144, + "k": 6144, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/GLM-5.1-MXFP4/down_proj", + "amd/GLM-5.1-MXFP4/gate_up_proj", + "amd/GLM-5.2-MXFP4/down_proj", + "amd/GLM-5.2-MXFP4/gate_up_proj", + "amd/MiniMax-M3-MXFP4/gate_up_proj", + "nvidia/GLM-5.1-NVFP4/down_proj", + "nvidia/GLM-5.1-NVFP4/gate_up_proj", + "nvidia/GLM-5.2-NVFP4/down_proj", + "nvidia/GLM-5.2-NVFP4/gate_up_proj" + ] }, { "type": "gemm", "args": { "m": 65536, - "n": 3072, - "k": 192, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "n": 6144, + "k": 6144, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/GLM-5-NVFP4/down_proj", + "nvidia/GLM-5-NVFP4/gate_up_proj" + ] }, { "type": "gemm", "args": { - "m": 131072, - "n": 128, - "k": 3072, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 65536, + "n": 6144, + "k": 6144, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "zai-org/GLM-5-FP8/down_proj", + "zai-org/GLM-5-FP8/gate_up_proj", + "zai-org/GLM-5.1-FP8/down_proj", + "zai-org/GLM-5.1-FP8/gate_up_proj", + "zai-org/GLM-5.2-FP8/down_proj", + "zai-org/GLM-5.2-FP8/gate_up_proj" + ] }, { "type": "gemm", "args": { - "m": 131072, - "n": 192, - "k": 3072, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 65536, + "n": 6144, + "k": 8192, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/GLM-5.1-MXFP4/o_proj", + "amd/GLM-5.2-MXFP4/o_proj", + "amd/MiniMax-M3-MXFP4/o_proj", + "nvidia/GLM-5-NVFP4/o_proj", + "nvidia/GLM-5.1-NVFP4/o_proj", + "nvidia/GLM-5.2-NVFP4/o_proj" + ] }, { "type": "gemm", "args": { - "m": 131072, - "n": 3072, - "k": 192, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 65536, + "n": 7168, + "k": 512, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/GLM-5.1-MXFP4/kv_b_proj", + "amd/GLM-5.2-MXFP4/kv_b_proj", + "amd/Kimi-K2.5-MXFP4/o_proj", + "moonshotai/Kimi-K2.5/o_proj", + "nvidia/GLM-5-NVFP4/kv_b_proj", + "nvidia/GLM-5.1-NVFP4/kv_b_proj", + "nvidia/GLM-5.2-NVFP4/kv_b_proj", + "nvidia/Kimi-K2.5-NVFP4/o_proj", + "nvidia/Kimi-K2.6-NVFP4/o_proj" + ] }, { "type": "gemm", "args": { - "m": 256, - "n": 128, - "k": 3072, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 65536, + "n": 7168, + "k": 512, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "zai-org/GLM-5-FP8/kv_b_proj", + "zai-org/GLM-5.1-FP8/kv_b_proj", + "zai-org/GLM-5.2-FP8/kv_b_proj" + ] }, { "type": "gemm", "args": { - "m": 256, - "n": 192, - "k": 3072, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 65536, + "n": 7168, + "k": 576, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-NVFP4-v2/down_proj" + ] }, { "type": "gemm", "args": { - "m": 256, - "n": 3072, - "k": 192, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 65536, + "n": 7168, + "k": 576, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/down_proj" + ] }, { "type": "gemm", "args": { - "m": 512, - "n": 128, - "k": 3072, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 65536, + "n": 7168, + "k": 1024, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/Kimi-K2.5-MXFP4/o_proj", + "moonshotai/Kimi-K2.5/o_proj", + "nvidia/Kimi-K2.5-NVFP4/o_proj", + "nvidia/Kimi-K2.6-NVFP4/o_proj" + ] }, { "type": "gemm", "args": { - "m": 512, - "n": 192, - "k": 3072, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 65536, + "n": 7168, + "k": 1024, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/o_proj", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] }, { "type": "gemm", "args": { - "m": 512, - "n": 3072, - "k": 192, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 65536, + "n": 7168, + "k": 1152, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/down_proj" + ] }, { "type": "gemm", "args": { - "m": 2097152, - "n": 128, - "k": 3072, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 65536, + "n": 7168, + "k": 2048, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/Kimi-K2.5-MXFP4/o_proj", + "moonshotai/Kimi-K2.5/o_proj", + "nvidia/Kimi-K2.5-NVFP4/o_proj", + "nvidia/Kimi-K2.6-NVFP4/o_proj" + ] }, { "type": "gemm", "args": { - "m": 2097152, - "n": 192, - "k": 3072, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 65536, + "n": 7168, + "k": 2048, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/o_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/o_proj" + ] }, { "type": "gemm", "args": { - "m": 2097152, - "n": 3072, - "k": 192, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 65536, + "n": 7168, + "k": 2048, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 32 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 1, + 32 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "amd/DeepSeek-R1-0528-MXFP4-Preview/o_proj", + "amd/DeepSeek-R1-0528-MXFP4-v2/o_proj", + "amd/DeepSeek-R1-0528-MXFP4/o_proj" + ] }, { "type": "gemm", "args": { - "m": 1024, - "n": 64, - "k": 2880, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 65536, + "n": 7168, + "k": 2048, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/o_proj", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] }, { "type": "gemm", "args": { - "m": 1024, - "n": 2880, - "k": 512, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 65536, + "n": 7168, + "k": 2304, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "moonshotai/Kimi-K2.5/down_proj", + "nvidia/Kimi-K2.6-NVFP4/down_proj" + ] }, { "type": "gemm", "args": { - "m": 2048, - "n": 64, - "k": 2880, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 65536, + "n": 7168, + "k": 2304, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/down_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/down_proj", + "nvidia/Kimi-K2.5-NVFP4/down_proj" + ] }, { "type": "gemm", "args": { - "m": 2048, - "n": 2880, - "k": 512, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 65536, + "n": 7168, + "k": 2304, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 32 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 1, + 32 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "amd/DeepSeek-R1-0528-MXFP4-Preview/down_proj", + "amd/DeepSeek-R1-0528-MXFP4-v2/down_proj", + "amd/DeepSeek-R1-0528-MXFP4/down_proj", + "amd/Kimi-K2.5-MXFP4/down_proj" + ] }, { "type": "gemm", "args": { - "m": 2, - "n": 64, - "k": 2880, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 65536, + "n": 7168, + "k": 2304, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/down_proj" + ] }, { "type": "gemm", "args": { - "m": 2, - "n": 2880, - "k": 512, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 65536, + "n": 7168, + "k": 4096, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/o_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/o_proj" + ] }, { "type": "gemm", "args": { - "m": 1572864, - "n": 3072, - "k": 1536, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 65536, + "n": 7168, + "k": 4096, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 32 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 1, + 32 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "amd/DeepSeek-R1-0528-MXFP4-Preview/o_proj", + "amd/DeepSeek-R1-0528-MXFP4-v2/o_proj", + "amd/DeepSeek-R1-0528-MXFP4/o_proj" + ] }, { "type": "gemm", "args": { - "m": 1572864, - "n": 4096, - "k": 512, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 65536, + "n": 7168, + "k": 4096, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/o_proj", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] }, { "type": "gemm", "args": { - "m": 1572864, + "m": 65536, "n": 7168, - "k": 2048, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "k": 4608, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "moonshotai/Kimi-K2.5/down_proj", + "nvidia/Kimi-K2.6-NVFP4/down_proj" + ] }, { "type": "gemm", "args": { - "m": 1536, - "n": 3072, - "k": 1536, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 65536, + "n": 7168, + "k": 4608, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/down_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/down_proj", + "nvidia/Kimi-K2.5-NVFP4/down_proj" + ] }, { "type": "gemm", "args": { - "m": 1536, - "n": 4096, - "k": 512, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 65536, + "n": 7168, + "k": 4608, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 32 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 1, + 32 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "amd/DeepSeek-R1-0528-MXFP4-Preview/down_proj", + "amd/DeepSeek-R1-0528-MXFP4-v2/down_proj", + "amd/DeepSeek-R1-0528-MXFP4/down_proj", + "amd/Kimi-K2.5-MXFP4/down_proj" + ] }, { "type": "gemm", "args": { - "m": 1536, + "m": 65536, "n": 7168, - "k": 2048, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "k": 4608, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/down_proj" + ] }, { "type": "gemm", "args": { - "m": 192, - "n": 2304, - "k": 7168, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 65536, + "n": 7168, + "k": 8192, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/Kimi-K2.5-MXFP4/o_proj", + "moonshotai/Kimi-K2.5/o_proj", + "nvidia/Kimi-K2.5-NVFP4/o_proj", + "nvidia/Kimi-K2.6-NVFP4/o_proj" + ] }, { "type": "gemm", "args": { - "m": 192, + "m": 65536, "n": 7168, - "k": 2304, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "k": 8192, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] }, { "type": "gemm", "args": { - "m": 2048, - "n": 6144, - "k": 1536, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 65536, + "n": 7168, + "k": 16384, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/o_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/o_proj" + ] }, { "type": "gemm", "args": { - "m": 2048, - "n": 8192, - "k": 512, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 65536, + "n": 7168, + "k": 16384, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/o_proj", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] }, { "type": "gemm", "args": { - "m": 2048, + "m": 65536, "n": 7168, - "k": 4096, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "k": 16384, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] }, { "type": "gemm", "args": { - "m": 2, - "n": 6144, - "k": 1536, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 65536, + "n": 7168, + "k": 65536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] }, { "type": "gemm", "args": { - "m": 2, + "m": 65536, "n": 8192, "k": 512, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/Kimi-K2.5-MXFP4/kv_b_proj", + "moonshotai/Kimi-K2.5/kv_b_proj", + "nvidia/DeepSeek-R1-0528-FP4-V2/kv_b_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/kv_b_proj", + "nvidia/Kimi-K2.5-NVFP4/kv_b_proj", + "nvidia/Kimi-K2.6-NVFP4/kv_b_proj" + ] }, { "type": "gemm", "args": { - "m": 2, - "n": 7168, - "k": 4096, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 65536, + "n": 8192, + "k": 512, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 32 + ] + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 1, + 32 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "amd/DeepSeek-R1-0528-MXFP4-Preview/kv_b_proj", + "amd/DeepSeek-R1-0528-MXFP4-v2/kv_b_proj", + "amd/DeepSeek-R1-0528-MXFP4/kv_b_proj" + ] }, { "type": "gemm", "args": { - "m": 2, - "n": 4608, - "k": 7168, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 65536, + "n": 8192, + "k": 512, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/kv_b_proj" + ] }, { "type": "gemm", "args": { - "m": 2, - "n": 7168, - "k": 4608, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 65536, + "n": 8192, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_b" + ] }, { "type": "gemm", "args": { - "m": 1024, - "n": 6144, - "k": 1536, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 65536, + "n": 8192, + "k": 2048, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/GLM-5.1-MXFP4/q_b_proj", + "amd/GLM-5.2-MXFP4/q_b_proj", + "nvidia/GLM-5-NVFP4/q_b_proj", + "nvidia/GLM-5.1-NVFP4/q_b_proj", + "nvidia/GLM-5.2-NVFP4/q_b_proj" + ] }, { "type": "gemm", "args": { - "m": 1024, - "n": 8192, - "k": 512, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 65536, + "n": 12288, + "k": 1536, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/Kimi-K2.5-MXFP4/q_b_proj", + "moonshotai/Kimi-K2.5/q_b_proj", + "nvidia/DeepSeek-R1-0528-FP4-V2/q_b_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/q_b_proj", + "nvidia/Kimi-K2.5-NVFP4/q_b_proj", + "nvidia/Kimi-K2.6-NVFP4/q_b_proj" + ] }, { "type": "gemm", "args": { - "m": 1024, - "n": 7168, - "k": 4096, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 65536, + "n": 14336, + "k": 512, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/GLM-5.1-MXFP4/kv_b_proj", + "amd/GLM-5.2-MXFP4/kv_b_proj", + "nvidia/GLM-5-NVFP4/kv_b_proj", + "nvidia/GLM-5.1-NVFP4/kv_b_proj", + "nvidia/GLM-5.2-NVFP4/kv_b_proj" + ] }, { "type": "gemm", "args": { - "m": 1, - "n": 6144, - "k": 1536, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 65536, + "n": 15360, + "k": 1024, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] }, { "type": "gemm", "args": { - "m": 1, - "n": 8192, + "m": 65536, + "n": 16384, "k": 512, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "amd/Kimi-K2.5-MXFP4/kv_b_proj", + "moonshotai/Kimi-K2.5/kv_b_proj", + "nvidia/DeepSeek-R1-0528-FP4-V2/kv_b_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/kv_b_proj", + "nvidia/Kimi-K2.5-NVFP4/kv_b_proj", + "nvidia/Kimi-K2.6-NVFP4/kv_b_proj" + ] }, { "type": "gemm", "args": { - "m": 1, - "n": 7168, - "k": 4096, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 65536, + "n": 16384, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_b" + ] }, { "type": "gemm", "args": { - "m": 128, - "n": 8192, - "k": 512, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 65536, + "n": 24576, + "k": 1536, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/q_b_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/q_b_proj" + ] }, { "type": "gemm", "args": { - "m": 128, - "n": 7168, - "k": 4096, - "dtype_a": "fp8", - "dtype_b": "fp8", - "dtype_out": "bf16" - } + "m": 65536, + "n": 24576, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/q_b_proj" + ] }, { "type": "gemm", "args": { - "m": 2048, - "n": 3072, - "k": 1536, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 65536, + "n": 30720, + "k": 1024, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] }, { "type": "gemm", "args": { - "m": 2048, - "n": 4096, + "m": 65536, + "n": 32768, "k": 512, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-FP4-V2/kv_b_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/kv_b_proj" + ] }, { "type": "gemm", "args": { - "m": 2048, - "n": 7168, - "k": 2048, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 65536, + "n": 32768, + "k": 512, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/kv_b_proj" + ] }, { "type": "gemm", "args": { - "m": 2048, - "n": 2304, - "k": 7168, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 65536, + "n": 65536, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_b" + ] }, { "type": "gemm", "args": { - "m": 2048, - "n": 7168, - "k": 2304, - "dtype_a": "nvfp4", - "dtype_b": "nvfp4", - "dtype_out": "bf16" - } + "m": 65536, + "n": 122880, + "k": 1024, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] } -] \ No newline at end of file +] diff --git a/operatorx/testlists/gemm_deepseek_v4.json b/operatorx/testlists/gemm_deepseek_v4.json new file mode 100644 index 0000000000..daa638d745 --- /dev/null +++ b/operatorx/testlists/gemm_deepseek_v4.json @@ -0,0 +1,26818 @@ +[ + { + "type": "gemm", + "args": { + "m": 1, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 7168, + "n": 1024, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.compressor.wkv+attn.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro/attn.compressor.wkv+attn.compressor.wgate" + ] + }, + { + "type": "gemm", + "args": { + "m": 2, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 7168, + "n": 1024, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.compressor.wkv+attn.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro/attn.compressor.wkv+attn.compressor.wgate" + ] + }, + { + "type": "gemm", + "args": { + "m": 3, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 7168, + "n": 1024, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.compressor.wkv+attn.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro/attn.compressor.wkv+attn.compressor.wgate" + ] + }, + { + "type": "gemm", + "args": { + "m": 4, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 7168, + "n": 1024, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.compressor.wkv+attn.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro/attn.compressor.wkv+attn.compressor.wgate" + ] + }, + { + "type": "gemm", + "args": { + "m": 6, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 7168, + "n": 1024, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.compressor.wkv+attn.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro/attn.compressor.wkv+attn.compressor.wgate" + ] + }, + { + "type": "gemm", + "args": { + "m": 8, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 7168, + "n": 1024, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.compressor.wkv+attn.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro/attn.compressor.wkv+attn.compressor.wgate" + ] + }, + { + "type": "gemm", + "args": { + "m": 12, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 7168, + "n": 1024, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.compressor.wkv+attn.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro/attn.compressor.wkv+attn.compressor.wgate" + ] + }, + { + "type": "gemm", + "args": { + "m": 16, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 7168, + "n": 1024, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.compressor.wkv+attn.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro/attn.compressor.wkv+attn.compressor.wgate" + ] + }, + { + "type": "gemm", + "args": { + "m": 24, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 7168, + "n": 1024, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.compressor.wkv+attn.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro/attn.compressor.wkv+attn.compressor.wgate" + ] + }, + { + "type": "gemm", + "args": { + "m": 32, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 7168, + "n": 1024, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.compressor.wkv+attn.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro/attn.compressor.wkv+attn.compressor.wgate" + ] + }, + { + "type": "gemm", + "args": { + "m": 48, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 7168, + "n": 1024, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.compressor.wkv+attn.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro/attn.compressor.wkv+attn.compressor.wgate" + ] + }, + { + "type": "gemm", + "args": { + "m": 64, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 7168, + "n": 1024, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.compressor.wkv+attn.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro/attn.compressor.wkv+attn.compressor.wgate" + ] + }, + { + "type": "gemm", + "args": { + "m": 96, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 7168, + "n": 1024, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.compressor.wkv+attn.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro/attn.compressor.wkv+attn.compressor.wgate" + ] + }, + { + "type": "gemm", + "args": { + "m": 128, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 7168, + "n": 1024, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.compressor.wkv+attn.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro/attn.compressor.wkv+attn.compressor.wgate" + ] + }, + { + "type": "gemm", + "args": { + "m": 192, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 7168, + "n": 1024, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.compressor.wkv+attn.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro/attn.compressor.wkv+attn.compressor.wgate" + ] + }, + { + "type": "gemm", + "args": { + "m": 256, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 7168, + "n": 1024, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.compressor.wkv+attn.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro/attn.compressor.wkv+attn.compressor.wgate" + ] + }, + { + "type": "gemm", + "args": { + "m": 384, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 7168, + "n": 1024, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.compressor.wkv+attn.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro/attn.compressor.wkv+attn.compressor.wgate" + ] + }, + { + "type": "gemm", + "args": { + "m": 512, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 7168, + "n": 1024, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.compressor.wkv+attn.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro/attn.compressor.wkv+attn.compressor.wgate" + ] + }, + { + "type": "gemm", + "args": { + "m": 768, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 7168, + "n": 1024, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.compressor.wkv+attn.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro/attn.compressor.wkv+attn.compressor.wgate" + ] + }, + { + "type": "gemm", + "args": { + "m": 1024, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 7168, + "n": 1024, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.compressor.wkv+attn.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro/attn.compressor.wkv+attn.compressor.wgate" + ] + }, + { + "type": "gemm", + "args": { + "m": 1536, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 7168, + "n": 1024, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.compressor.wkv+attn.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro/attn.compressor.wkv+attn.compressor.wgate" + ] + }, + { + "type": "gemm", + "args": { + "m": 2048, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 7168, + "n": 1024, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.compressor.wkv+attn.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro/attn.compressor.wkv+attn.compressor.wgate" + ] + }, + { + "type": "gemm", + "args": { + "m": 3072, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 7168, + "n": 1024, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.compressor.wkv+attn.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro/attn.compressor.wkv+attn.compressor.wgate" + ] + }, + { + "type": "gemm", + "args": { + "m": 4096, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 7168, + "n": 1024, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.compressor.wkv+attn.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro/attn.compressor.wkv+attn.compressor.wgate" + ] + }, + { + "type": "gemm", + "args": { + "m": 6144, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 7168, + "n": 1024, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.compressor.wkv+attn.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro/attn.compressor.wkv+attn.compressor.wgate" + ] + }, + { + "type": "gemm", + "args": { + "m": 8192, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 7168, + "n": 1024, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.compressor.wkv+attn.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro/attn.compressor.wkv+attn.compressor.wgate" + ] + }, + { + "type": "gemm", + "args": { + "m": 12288, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 7168, + "n": 1024, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.compressor.wkv+attn.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro/attn.compressor.wkv+attn.compressor.wgate" + ] + }, + { + "type": "gemm", + "args": { + "m": 16384, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 7168, + "n": 1024, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.compressor.wkv+attn.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro/attn.compressor.wkv+attn.compressor.wgate" + ] + }, + { + "type": "gemm", + "args": { + "m": 24576, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 7168, + "n": 1024, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.compressor.wkv+attn.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro/attn.compressor.wkv+attn.compressor.wgate" + ] + }, + { + "type": "gemm", + "args": { + "m": 32768, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 7168, + "n": 1024, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.compressor.wkv+attn.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro/attn.compressor.wkv+attn.compressor.wgate" + ] + }, + { + "type": "gemm", + "args": { + "m": 49152, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 7168, + "n": 1024, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.compressor.wkv+attn.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro/attn.compressor.wkv+attn.compressor.wgate" + ] + }, + { + "type": "gemm", + "args": { + "m": 65536, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 7168, + "n": 1024, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.compressor.wkv+attn.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro/attn.compressor.wkv+attn.compressor.wgate" + ] + }, + { + "type": "gemm", + "args": { + "m": 1, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 7168, + "n": 1024, + "out": "fp32" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.compressor.wkv+attn.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro/attn.compressor.wkv+attn.compressor.wgate" + ] + }, + { + "type": "gemm", + "args": { + "m": 2, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 7168, + "n": 1024, + "out": "fp32" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.compressor.wkv+attn.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro/attn.compressor.wkv+attn.compressor.wgate" + ] + }, + { + "type": "gemm", + "args": { + "m": 3, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 7168, + "n": 1024, + "out": "fp32" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.compressor.wkv+attn.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro/attn.compressor.wkv+attn.compressor.wgate" + ] + }, + { + "type": "gemm", + "args": { + "m": 4, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 7168, + "n": 1024, + "out": "fp32" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.compressor.wkv+attn.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro/attn.compressor.wkv+attn.compressor.wgate" + ] + }, + { + "type": "gemm", + "args": { + "m": 6, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 7168, + "n": 1024, + "out": "fp32" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.compressor.wkv+attn.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro/attn.compressor.wkv+attn.compressor.wgate" + ] + }, + { + "type": "gemm", + "args": { + "m": 8, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 7168, + "n": 1024, + "out": "fp32" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.compressor.wkv+attn.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro/attn.compressor.wkv+attn.compressor.wgate" + ] + }, + { + "type": "gemm", + "args": { + "m": 12, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 7168, + "n": 1024, + "out": "fp32" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.compressor.wkv+attn.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro/attn.compressor.wkv+attn.compressor.wgate" + ] + }, + { + "type": "gemm", + "args": { + "m": 16, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 7168, + "n": 1024, + "out": "fp32" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.compressor.wkv+attn.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro/attn.compressor.wkv+attn.compressor.wgate" + ] + }, + { + "type": "gemm", + "args": { + "m": 24, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 7168, + "n": 1024, + "out": "fp32" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.compressor.wkv+attn.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro/attn.compressor.wkv+attn.compressor.wgate" + ] + }, + { + "type": "gemm", + "args": { + "m": 32, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 7168, + "n": 1024, + "out": "fp32" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.compressor.wkv+attn.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro/attn.compressor.wkv+attn.compressor.wgate" + ] + }, + { + "type": "gemm", + "args": { + "m": 48, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 7168, + "n": 1024, + "out": "fp32" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.compressor.wkv+attn.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro/attn.compressor.wkv+attn.compressor.wgate" + ] + }, + { + "type": "gemm", + "args": { + "m": 64, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 7168, + "n": 1024, + "out": "fp32" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.compressor.wkv+attn.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro/attn.compressor.wkv+attn.compressor.wgate" + ] + }, + { + "type": "gemm", + "args": { + "m": 96, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 7168, + "n": 1024, + "out": "fp32" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.compressor.wkv+attn.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro/attn.compressor.wkv+attn.compressor.wgate" + ] + }, + { + "type": "gemm", + "args": { + "m": 128, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 7168, + "n": 1024, + "out": "fp32" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.compressor.wkv+attn.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro/attn.compressor.wkv+attn.compressor.wgate" + ] + }, + { + "type": "gemm", + "args": { + "m": 192, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 7168, + "n": 1024, + "out": "fp32" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.compressor.wkv+attn.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro/attn.compressor.wkv+attn.compressor.wgate" + ] + }, + { + "type": "gemm", + "args": { + "m": 256, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 7168, + "n": 1024, + "out": "fp32" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.compressor.wkv+attn.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro/attn.compressor.wkv+attn.compressor.wgate" + ] + }, + { + "type": "gemm", + "args": { + "m": 384, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 7168, + "n": 1024, + "out": "fp32" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.compressor.wkv+attn.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro/attn.compressor.wkv+attn.compressor.wgate" + ] + }, + { + "type": "gemm", + "args": { + "m": 512, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 7168, + "n": 1024, + "out": "fp32" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.compressor.wkv+attn.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro/attn.compressor.wkv+attn.compressor.wgate" + ] + }, + { + "type": "gemm", + "args": { + "m": 768, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 7168, + "n": 1024, + "out": "fp32" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.compressor.wkv+attn.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro/attn.compressor.wkv+attn.compressor.wgate" + ] + }, + { + "type": "gemm", + "args": { + "m": 1024, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 7168, + "n": 1024, + "out": "fp32" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.compressor.wkv+attn.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro/attn.compressor.wkv+attn.compressor.wgate" + ] + }, + { + "type": "gemm", + "args": { + "m": 1536, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 7168, + "n": 1024, + "out": "fp32" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.compressor.wkv+attn.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro/attn.compressor.wkv+attn.compressor.wgate" + ] + }, + { + "type": "gemm", + "args": { + "m": 2048, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 7168, + "n": 1024, + "out": "fp32" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.compressor.wkv+attn.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro/attn.compressor.wkv+attn.compressor.wgate" + ] + }, + { + "type": "gemm", + "args": { + "m": 3072, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 7168, + "n": 1024, + "out": "fp32" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.compressor.wkv+attn.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro/attn.compressor.wkv+attn.compressor.wgate" + ] + }, + { + "type": "gemm", + "args": { + "m": 4096, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 7168, + "n": 1024, + "out": "fp32" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.compressor.wkv+attn.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro/attn.compressor.wkv+attn.compressor.wgate" + ] + }, + { + "type": "gemm", + "args": { + "m": 6144, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 7168, + "n": 1024, + "out": "fp32" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.compressor.wkv+attn.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro/attn.compressor.wkv+attn.compressor.wgate" + ] + }, + { + "type": "gemm", + "args": { + "m": 8192, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 7168, + "n": 1024, + "out": "fp32" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.compressor.wkv+attn.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro/attn.compressor.wkv+attn.compressor.wgate" + ] + }, + { + "type": "gemm", + "args": { + "m": 12288, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 7168, + "n": 1024, + "out": "fp32" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.compressor.wkv+attn.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro/attn.compressor.wkv+attn.compressor.wgate" + ] + }, + { + "type": "gemm", + "args": { + "m": 16384, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 7168, + "n": 1024, + "out": "fp32" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.compressor.wkv+attn.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro/attn.compressor.wkv+attn.compressor.wgate" + ] + }, + { + "type": "gemm", + "args": { + "m": 24576, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 7168, + "n": 1024, + "out": "fp32" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.compressor.wkv+attn.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro/attn.compressor.wkv+attn.compressor.wgate" + ] + }, + { + "type": "gemm", + "args": { + "m": 32768, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 7168, + "n": 1024, + "out": "fp32" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.compressor.wkv+attn.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro/attn.compressor.wkv+attn.compressor.wgate" + ] + }, + { + "type": "gemm", + "args": { + "m": 49152, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 7168, + "n": 1024, + "out": "fp32" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.compressor.wkv+attn.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro/attn.compressor.wkv+attn.compressor.wgate" + ] + }, + { + "type": "gemm", + "args": { + "m": 65536, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 7168, + "n": 1024, + "out": "fp32" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.compressor.wkv+attn.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro/attn.compressor.wkv+attn.compressor.wgate" + ] + }, + { + "type": "gemm", + "args": { + "m": 1, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 7168, + "n": 2048, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.compressor.wkv+attn.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro/attn.compressor.wkv+attn.compressor.wgate" + ] + }, + { + "type": "gemm", + "args": { + "m": 2, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 7168, + "n": 2048, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.compressor.wkv+attn.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro/attn.compressor.wkv+attn.compressor.wgate" + ] + }, + { + "type": "gemm", + "args": { + "m": 3, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 7168, + "n": 2048, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.compressor.wkv+attn.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro/attn.compressor.wkv+attn.compressor.wgate" + ] + }, + { + "type": "gemm", + "args": { + "m": 4, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 7168, + "n": 2048, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.compressor.wkv+attn.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro/attn.compressor.wkv+attn.compressor.wgate" + ] + }, + { + "type": "gemm", + "args": { + "m": 6, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 7168, + "n": 2048, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.compressor.wkv+attn.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro/attn.compressor.wkv+attn.compressor.wgate" + ] + }, + { + "type": "gemm", + "args": { + "m": 8, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 7168, + "n": 2048, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.compressor.wkv+attn.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro/attn.compressor.wkv+attn.compressor.wgate" + ] + }, + { + "type": "gemm", + "args": { + "m": 12, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 7168, + "n": 2048, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.compressor.wkv+attn.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro/attn.compressor.wkv+attn.compressor.wgate" + ] + }, + { + "type": "gemm", + "args": { + "m": 16, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 7168, + "n": 2048, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.compressor.wkv+attn.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro/attn.compressor.wkv+attn.compressor.wgate" + ] + }, + { + "type": "gemm", + "args": { + "m": 24, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 7168, + "n": 2048, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.compressor.wkv+attn.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro/attn.compressor.wkv+attn.compressor.wgate" + ] + }, + { + "type": "gemm", + "args": { + "m": 32, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 7168, + "n": 2048, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.compressor.wkv+attn.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro/attn.compressor.wkv+attn.compressor.wgate" + ] + }, + { + "type": "gemm", + "args": { + "m": 48, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 7168, + "n": 2048, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.compressor.wkv+attn.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro/attn.compressor.wkv+attn.compressor.wgate" + ] + }, + { + "type": "gemm", + "args": { + "m": 64, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 7168, + "n": 2048, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.compressor.wkv+attn.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro/attn.compressor.wkv+attn.compressor.wgate" + ] + }, + { + "type": "gemm", + "args": { + "m": 96, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 7168, + "n": 2048, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.compressor.wkv+attn.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro/attn.compressor.wkv+attn.compressor.wgate" + ] + }, + { + "type": "gemm", + "args": { + "m": 128, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 7168, + "n": 2048, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.compressor.wkv+attn.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro/attn.compressor.wkv+attn.compressor.wgate" + ] + }, + { + "type": "gemm", + "args": { + "m": 192, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 7168, + "n": 2048, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.compressor.wkv+attn.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro/attn.compressor.wkv+attn.compressor.wgate" + ] + }, + { + "type": "gemm", + "args": { + "m": 256, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 7168, + "n": 2048, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.compressor.wkv+attn.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro/attn.compressor.wkv+attn.compressor.wgate" + ] + }, + { + "type": "gemm", + "args": { + "m": 384, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 7168, + "n": 2048, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.compressor.wkv+attn.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro/attn.compressor.wkv+attn.compressor.wgate" + ] + }, + { + "type": "gemm", + "args": { + "m": 512, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 7168, + "n": 2048, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.compressor.wkv+attn.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro/attn.compressor.wkv+attn.compressor.wgate" + ] + }, + { + "type": "gemm", + "args": { + "m": 768, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 7168, + "n": 2048, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.compressor.wkv+attn.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro/attn.compressor.wkv+attn.compressor.wgate" + ] + }, + { + "type": "gemm", + "args": { + "m": 1024, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 7168, + "n": 2048, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.compressor.wkv+attn.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro/attn.compressor.wkv+attn.compressor.wgate" + ] + }, + { + "type": "gemm", + "args": { + "m": 1536, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 7168, + "n": 2048, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.compressor.wkv+attn.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro/attn.compressor.wkv+attn.compressor.wgate" + ] + }, + { + "type": "gemm", + "args": { + "m": 2048, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 7168, + "n": 2048, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.compressor.wkv+attn.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro/attn.compressor.wkv+attn.compressor.wgate" + ] + }, + { + "type": "gemm", + "args": { + "m": 3072, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 7168, + "n": 2048, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.compressor.wkv+attn.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro/attn.compressor.wkv+attn.compressor.wgate" + ] + }, + { + "type": "gemm", + "args": { + "m": 4096, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 7168, + "n": 2048, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.compressor.wkv+attn.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro/attn.compressor.wkv+attn.compressor.wgate" + ] + }, + { + "type": "gemm", + "args": { + "m": 6144, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 7168, + "n": 2048, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.compressor.wkv+attn.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro/attn.compressor.wkv+attn.compressor.wgate" + ] + }, + { + "type": "gemm", + "args": { + "m": 8192, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 7168, + "n": 2048, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.compressor.wkv+attn.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro/attn.compressor.wkv+attn.compressor.wgate" + ] + }, + { + "type": "gemm", + "args": { + "m": 12288, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 7168, + "n": 2048, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.compressor.wkv+attn.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro/attn.compressor.wkv+attn.compressor.wgate" + ] + }, + { + "type": "gemm", + "args": { + "m": 16384, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 7168, + "n": 2048, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.compressor.wkv+attn.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro/attn.compressor.wkv+attn.compressor.wgate" + ] + }, + { + "type": "gemm", + "args": { + "m": 24576, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 7168, + "n": 2048, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.compressor.wkv+attn.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro/attn.compressor.wkv+attn.compressor.wgate" + ] + }, + { + "type": "gemm", + "args": { + "m": 32768, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 7168, + "n": 2048, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.compressor.wkv+attn.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro/attn.compressor.wkv+attn.compressor.wgate" + ] + }, + { + "type": "gemm", + "args": { + "m": 49152, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 7168, + "n": 2048, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.compressor.wkv+attn.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro/attn.compressor.wkv+attn.compressor.wgate" + ] + }, + { + "type": "gemm", + "args": { + "m": 65536, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 7168, + "n": 2048, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.compressor.wkv+attn.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro/attn.compressor.wkv+attn.compressor.wgate" + ] + }, + { + "type": "gemm", + "args": { + "m": 1, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 7168, + "n": 2048, + "out": "fp32" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.compressor.wkv+attn.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro/attn.compressor.wkv+attn.compressor.wgate" + ] + }, + { + "type": "gemm", + "args": { + "m": 2, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 7168, + "n": 2048, + "out": "fp32" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.compressor.wkv+attn.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro/attn.compressor.wkv+attn.compressor.wgate" + ] + }, + { + "type": "gemm", + "args": { + "m": 3, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 7168, + "n": 2048, + "out": "fp32" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.compressor.wkv+attn.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro/attn.compressor.wkv+attn.compressor.wgate" + ] + }, + { + "type": "gemm", + "args": { + "m": 4, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 7168, + "n": 2048, + "out": "fp32" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.compressor.wkv+attn.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro/attn.compressor.wkv+attn.compressor.wgate" + ] + }, + { + "type": "gemm", + "args": { + "m": 6, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 7168, + "n": 2048, + "out": "fp32" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.compressor.wkv+attn.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro/attn.compressor.wkv+attn.compressor.wgate" + ] + }, + { + "type": "gemm", + "args": { + "m": 8, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 7168, + "n": 2048, + "out": "fp32" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.compressor.wkv+attn.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro/attn.compressor.wkv+attn.compressor.wgate" + ] + }, + { + "type": "gemm", + "args": { + "m": 12, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 7168, + "n": 2048, + "out": "fp32" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.compressor.wkv+attn.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro/attn.compressor.wkv+attn.compressor.wgate" + ] + }, + { + "type": "gemm", + "args": { + "m": 16, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 7168, + "n": 2048, + "out": "fp32" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.compressor.wkv+attn.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro/attn.compressor.wkv+attn.compressor.wgate" + ] + }, + { + "type": "gemm", + "args": { + "m": 24, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 7168, + "n": 2048, + "out": "fp32" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.compressor.wkv+attn.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro/attn.compressor.wkv+attn.compressor.wgate" + ] + }, + { + "type": "gemm", + "args": { + "m": 32, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 7168, + "n": 2048, + "out": "fp32" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.compressor.wkv+attn.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro/attn.compressor.wkv+attn.compressor.wgate" + ] + }, + { + "type": "gemm", + "args": { + "m": 48, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 7168, + "n": 2048, + "out": "fp32" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.compressor.wkv+attn.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro/attn.compressor.wkv+attn.compressor.wgate" + ] + }, + { + "type": "gemm", + "args": { + "m": 64, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 7168, + "n": 2048, + "out": "fp32" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.compressor.wkv+attn.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro/attn.compressor.wkv+attn.compressor.wgate" + ] + }, + { + "type": "gemm", + "args": { + "m": 96, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 7168, + "n": 2048, + "out": "fp32" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.compressor.wkv+attn.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro/attn.compressor.wkv+attn.compressor.wgate" + ] + }, + { + "type": "gemm", + "args": { + "m": 128, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 7168, + "n": 2048, + "out": "fp32" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.compressor.wkv+attn.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro/attn.compressor.wkv+attn.compressor.wgate" + ] + }, + { + "type": "gemm", + "args": { + "m": 192, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 7168, + "n": 2048, + "out": "fp32" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.compressor.wkv+attn.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro/attn.compressor.wkv+attn.compressor.wgate" + ] + }, + { + "type": "gemm", + "args": { + "m": 256, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 7168, + "n": 2048, + "out": "fp32" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.compressor.wkv+attn.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro/attn.compressor.wkv+attn.compressor.wgate" + ] + }, + { + "type": "gemm", + "args": { + "m": 384, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 7168, + "n": 2048, + "out": "fp32" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.compressor.wkv+attn.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro/attn.compressor.wkv+attn.compressor.wgate" + ] + }, + { + "type": "gemm", + "args": { + "m": 512, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 7168, + "n": 2048, + "out": "fp32" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.compressor.wkv+attn.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro/attn.compressor.wkv+attn.compressor.wgate" + ] + }, + { + "type": "gemm", + "args": { + "m": 768, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 7168, + "n": 2048, + "out": "fp32" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.compressor.wkv+attn.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro/attn.compressor.wkv+attn.compressor.wgate" + ] + }, + { + "type": "gemm", + "args": { + "m": 1024, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 7168, + "n": 2048, + "out": "fp32" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.compressor.wkv+attn.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro/attn.compressor.wkv+attn.compressor.wgate" + ] + }, + { + "type": "gemm", + "args": { + "m": 1536, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 7168, + "n": 2048, + "out": "fp32" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.compressor.wkv+attn.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro/attn.compressor.wkv+attn.compressor.wgate" + ] + }, + { + "type": "gemm", + "args": { + "m": 2048, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 7168, + "n": 2048, + "out": "fp32" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.compressor.wkv+attn.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro/attn.compressor.wkv+attn.compressor.wgate" + ] + }, + { + "type": "gemm", + "args": { + "m": 3072, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 7168, + "n": 2048, + "out": "fp32" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.compressor.wkv+attn.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro/attn.compressor.wkv+attn.compressor.wgate" + ] + }, + { + "type": "gemm", + "args": { + "m": 4096, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 7168, + "n": 2048, + "out": "fp32" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.compressor.wkv+attn.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro/attn.compressor.wkv+attn.compressor.wgate" + ] + }, + { + "type": "gemm", + "args": { + "m": 6144, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 7168, + "n": 2048, + "out": "fp32" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.compressor.wkv+attn.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro/attn.compressor.wkv+attn.compressor.wgate" + ] + }, + { + "type": "gemm", + "args": { + "m": 8192, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 7168, + "n": 2048, + "out": "fp32" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.compressor.wkv+attn.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro/attn.compressor.wkv+attn.compressor.wgate" + ] + }, + { + "type": "gemm", + "args": { + "m": 12288, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 7168, + "n": 2048, + "out": "fp32" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.compressor.wkv+attn.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro/attn.compressor.wkv+attn.compressor.wgate" + ] + }, + { + "type": "gemm", + "args": { + "m": 16384, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 7168, + "n": 2048, + "out": "fp32" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.compressor.wkv+attn.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro/attn.compressor.wkv+attn.compressor.wgate" + ] + }, + { + "type": "gemm", + "args": { + "m": 24576, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 7168, + "n": 2048, + "out": "fp32" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.compressor.wkv+attn.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro/attn.compressor.wkv+attn.compressor.wgate" + ] + }, + { + "type": "gemm", + "args": { + "m": 32768, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 7168, + "n": 2048, + "out": "fp32" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.compressor.wkv+attn.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro/attn.compressor.wkv+attn.compressor.wgate" + ] + }, + { + "type": "gemm", + "args": { + "m": 49152, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 7168, + "n": 2048, + "out": "fp32" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.compressor.wkv+attn.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro/attn.compressor.wkv+attn.compressor.wgate" + ] + }, + { + "type": "gemm", + "args": { + "m": 65536, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 7168, + "n": 2048, + "out": "fp32" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.compressor.wkv+attn.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro/attn.compressor.wkv+attn.compressor.wgate" + ] + }, + { + "type": "gemm", + "args": { + "m": 1, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 7168, + "n": 2560, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.compressor.wkv+attn.compressor.wgate+attn.indexer.compressor.wkv+attn.indexer.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro/attn.compressor.wkv+attn.compressor.wgate+attn.indexer.compressor.wkv+attn.indexer.compressor.wgate" + ] + }, + { + "type": "gemm", + "args": { + "m": 2, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 7168, + "n": 2560, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.compressor.wkv+attn.compressor.wgate+attn.indexer.compressor.wkv+attn.indexer.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro/attn.compressor.wkv+attn.compressor.wgate+attn.indexer.compressor.wkv+attn.indexer.compressor.wgate" + ] + }, + { + "type": "gemm", + "args": { + "m": 3, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 7168, + "n": 2560, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.compressor.wkv+attn.compressor.wgate+attn.indexer.compressor.wkv+attn.indexer.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro/attn.compressor.wkv+attn.compressor.wgate+attn.indexer.compressor.wkv+attn.indexer.compressor.wgate" + ] + }, + { + "type": "gemm", + "args": { + "m": 4, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 7168, + "n": 2560, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.compressor.wkv+attn.compressor.wgate+attn.indexer.compressor.wkv+attn.indexer.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro/attn.compressor.wkv+attn.compressor.wgate+attn.indexer.compressor.wkv+attn.indexer.compressor.wgate" + ] + }, + { + "type": "gemm", + "args": { + "m": 6, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 7168, + "n": 2560, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.compressor.wkv+attn.compressor.wgate+attn.indexer.compressor.wkv+attn.indexer.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro/attn.compressor.wkv+attn.compressor.wgate+attn.indexer.compressor.wkv+attn.indexer.compressor.wgate" + ] + }, + { + "type": "gemm", + "args": { + "m": 8, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 7168, + "n": 2560, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.compressor.wkv+attn.compressor.wgate+attn.indexer.compressor.wkv+attn.indexer.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro/attn.compressor.wkv+attn.compressor.wgate+attn.indexer.compressor.wkv+attn.indexer.compressor.wgate" + ] + }, + { + "type": "gemm", + "args": { + "m": 12, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 7168, + "n": 2560, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.compressor.wkv+attn.compressor.wgate+attn.indexer.compressor.wkv+attn.indexer.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro/attn.compressor.wkv+attn.compressor.wgate+attn.indexer.compressor.wkv+attn.indexer.compressor.wgate" + ] + }, + { + "type": "gemm", + "args": { + "m": 16, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 7168, + "n": 2560, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.compressor.wkv+attn.compressor.wgate+attn.indexer.compressor.wkv+attn.indexer.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro/attn.compressor.wkv+attn.compressor.wgate+attn.indexer.compressor.wkv+attn.indexer.compressor.wgate" + ] + }, + { + "type": "gemm", + "args": { + "m": 24, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 7168, + "n": 2560, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.compressor.wkv+attn.compressor.wgate+attn.indexer.compressor.wkv+attn.indexer.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro/attn.compressor.wkv+attn.compressor.wgate+attn.indexer.compressor.wkv+attn.indexer.compressor.wgate" + ] + }, + { + "type": "gemm", + "args": { + "m": 32, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 7168, + "n": 2560, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.compressor.wkv+attn.compressor.wgate+attn.indexer.compressor.wkv+attn.indexer.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro/attn.compressor.wkv+attn.compressor.wgate+attn.indexer.compressor.wkv+attn.indexer.compressor.wgate" + ] + }, + { + "type": "gemm", + "args": { + "m": 48, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 7168, + "n": 2560, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.compressor.wkv+attn.compressor.wgate+attn.indexer.compressor.wkv+attn.indexer.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro/attn.compressor.wkv+attn.compressor.wgate+attn.indexer.compressor.wkv+attn.indexer.compressor.wgate" + ] + }, + { + "type": "gemm", + "args": { + "m": 64, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 7168, + "n": 2560, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.compressor.wkv+attn.compressor.wgate+attn.indexer.compressor.wkv+attn.indexer.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro/attn.compressor.wkv+attn.compressor.wgate+attn.indexer.compressor.wkv+attn.indexer.compressor.wgate" + ] + }, + { + "type": "gemm", + "args": { + "m": 96, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 7168, + "n": 2560, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.compressor.wkv+attn.compressor.wgate+attn.indexer.compressor.wkv+attn.indexer.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro/attn.compressor.wkv+attn.compressor.wgate+attn.indexer.compressor.wkv+attn.indexer.compressor.wgate" + ] + }, + { + "type": "gemm", + "args": { + "m": 128, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 7168, + "n": 2560, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.compressor.wkv+attn.compressor.wgate+attn.indexer.compressor.wkv+attn.indexer.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro/attn.compressor.wkv+attn.compressor.wgate+attn.indexer.compressor.wkv+attn.indexer.compressor.wgate" + ] + }, + { + "type": "gemm", + "args": { + "m": 192, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 7168, + "n": 2560, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.compressor.wkv+attn.compressor.wgate+attn.indexer.compressor.wkv+attn.indexer.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro/attn.compressor.wkv+attn.compressor.wgate+attn.indexer.compressor.wkv+attn.indexer.compressor.wgate" + ] + }, + { + "type": "gemm", + "args": { + "m": 256, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 7168, + "n": 2560, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.compressor.wkv+attn.compressor.wgate+attn.indexer.compressor.wkv+attn.indexer.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro/attn.compressor.wkv+attn.compressor.wgate+attn.indexer.compressor.wkv+attn.indexer.compressor.wgate" + ] + }, + { + "type": "gemm", + "args": { + "m": 384, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 7168, + "n": 2560, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.compressor.wkv+attn.compressor.wgate+attn.indexer.compressor.wkv+attn.indexer.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro/attn.compressor.wkv+attn.compressor.wgate+attn.indexer.compressor.wkv+attn.indexer.compressor.wgate" + ] + }, + { + "type": "gemm", + "args": { + "m": 512, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 7168, + "n": 2560, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.compressor.wkv+attn.compressor.wgate+attn.indexer.compressor.wkv+attn.indexer.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro/attn.compressor.wkv+attn.compressor.wgate+attn.indexer.compressor.wkv+attn.indexer.compressor.wgate" + ] + }, + { + "type": "gemm", + "args": { + "m": 768, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 7168, + "n": 2560, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.compressor.wkv+attn.compressor.wgate+attn.indexer.compressor.wkv+attn.indexer.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro/attn.compressor.wkv+attn.compressor.wgate+attn.indexer.compressor.wkv+attn.indexer.compressor.wgate" + ] + }, + { + "type": "gemm", + "args": { + "m": 1024, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 7168, + "n": 2560, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.compressor.wkv+attn.compressor.wgate+attn.indexer.compressor.wkv+attn.indexer.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro/attn.compressor.wkv+attn.compressor.wgate+attn.indexer.compressor.wkv+attn.indexer.compressor.wgate" + ] + }, + { + "type": "gemm", + "args": { + "m": 1536, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 7168, + "n": 2560, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.compressor.wkv+attn.compressor.wgate+attn.indexer.compressor.wkv+attn.indexer.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro/attn.compressor.wkv+attn.compressor.wgate+attn.indexer.compressor.wkv+attn.indexer.compressor.wgate" + ] + }, + { + "type": "gemm", + "args": { + "m": 2048, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 7168, + "n": 2560, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.compressor.wkv+attn.compressor.wgate+attn.indexer.compressor.wkv+attn.indexer.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro/attn.compressor.wkv+attn.compressor.wgate+attn.indexer.compressor.wkv+attn.indexer.compressor.wgate" + ] + }, + { + "type": "gemm", + "args": { + "m": 3072, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 7168, + "n": 2560, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.compressor.wkv+attn.compressor.wgate+attn.indexer.compressor.wkv+attn.indexer.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro/attn.compressor.wkv+attn.compressor.wgate+attn.indexer.compressor.wkv+attn.indexer.compressor.wgate" + ] + }, + { + "type": "gemm", + "args": { + "m": 4096, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 7168, + "n": 2560, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.compressor.wkv+attn.compressor.wgate+attn.indexer.compressor.wkv+attn.indexer.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro/attn.compressor.wkv+attn.compressor.wgate+attn.indexer.compressor.wkv+attn.indexer.compressor.wgate" + ] + }, + { + "type": "gemm", + "args": { + "m": 6144, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 7168, + "n": 2560, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.compressor.wkv+attn.compressor.wgate+attn.indexer.compressor.wkv+attn.indexer.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro/attn.compressor.wkv+attn.compressor.wgate+attn.indexer.compressor.wkv+attn.indexer.compressor.wgate" + ] + }, + { + "type": "gemm", + "args": { + "m": 8192, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 7168, + "n": 2560, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.compressor.wkv+attn.compressor.wgate+attn.indexer.compressor.wkv+attn.indexer.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro/attn.compressor.wkv+attn.compressor.wgate+attn.indexer.compressor.wkv+attn.indexer.compressor.wgate" + ] + }, + { + "type": "gemm", + "args": { + "m": 12288, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 7168, + "n": 2560, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.compressor.wkv+attn.compressor.wgate+attn.indexer.compressor.wkv+attn.indexer.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro/attn.compressor.wkv+attn.compressor.wgate+attn.indexer.compressor.wkv+attn.indexer.compressor.wgate" + ] + }, + { + "type": "gemm", + "args": { + "m": 16384, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 7168, + "n": 2560, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.compressor.wkv+attn.compressor.wgate+attn.indexer.compressor.wkv+attn.indexer.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro/attn.compressor.wkv+attn.compressor.wgate+attn.indexer.compressor.wkv+attn.indexer.compressor.wgate" + ] + }, + { + "type": "gemm", + "args": { + "m": 24576, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 7168, + "n": 2560, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.compressor.wkv+attn.compressor.wgate+attn.indexer.compressor.wkv+attn.indexer.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro/attn.compressor.wkv+attn.compressor.wgate+attn.indexer.compressor.wkv+attn.indexer.compressor.wgate" + ] + }, + { + "type": "gemm", + "args": { + "m": 32768, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 7168, + "n": 2560, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.compressor.wkv+attn.compressor.wgate+attn.indexer.compressor.wkv+attn.indexer.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro/attn.compressor.wkv+attn.compressor.wgate+attn.indexer.compressor.wkv+attn.indexer.compressor.wgate" + ] + }, + { + "type": "gemm", + "args": { + "m": 49152, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 7168, + "n": 2560, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.compressor.wkv+attn.compressor.wgate+attn.indexer.compressor.wkv+attn.indexer.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro/attn.compressor.wkv+attn.compressor.wgate+attn.indexer.compressor.wkv+attn.indexer.compressor.wgate" + ] + }, + { + "type": "gemm", + "args": { + "m": 65536, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 7168, + "n": 2560, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.compressor.wkv+attn.compressor.wgate+attn.indexer.compressor.wkv+attn.indexer.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro/attn.compressor.wkv+attn.compressor.wgate+attn.indexer.compressor.wkv+attn.indexer.compressor.wgate" + ] + }, + { + "type": "gemm", + "args": { + "m": 1, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 7168, + "n": 2560, + "out": "fp32" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.compressor.wkv+attn.compressor.wgate+attn.indexer.compressor.wkv+attn.indexer.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro/attn.compressor.wkv+attn.compressor.wgate+attn.indexer.compressor.wkv+attn.indexer.compressor.wgate" + ] + }, + { + "type": "gemm", + "args": { + "m": 2, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 7168, + "n": 2560, + "out": "fp32" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.compressor.wkv+attn.compressor.wgate+attn.indexer.compressor.wkv+attn.indexer.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro/attn.compressor.wkv+attn.compressor.wgate+attn.indexer.compressor.wkv+attn.indexer.compressor.wgate" + ] + }, + { + "type": "gemm", + "args": { + "m": 3, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 7168, + "n": 2560, + "out": "fp32" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.compressor.wkv+attn.compressor.wgate+attn.indexer.compressor.wkv+attn.indexer.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro/attn.compressor.wkv+attn.compressor.wgate+attn.indexer.compressor.wkv+attn.indexer.compressor.wgate" + ] + }, + { + "type": "gemm", + "args": { + "m": 4, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 7168, + "n": 2560, + "out": "fp32" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.compressor.wkv+attn.compressor.wgate+attn.indexer.compressor.wkv+attn.indexer.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro/attn.compressor.wkv+attn.compressor.wgate+attn.indexer.compressor.wkv+attn.indexer.compressor.wgate" + ] + }, + { + "type": "gemm", + "args": { + "m": 6, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 7168, + "n": 2560, + "out": "fp32" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.compressor.wkv+attn.compressor.wgate+attn.indexer.compressor.wkv+attn.indexer.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro/attn.compressor.wkv+attn.compressor.wgate+attn.indexer.compressor.wkv+attn.indexer.compressor.wgate" + ] + }, + { + "type": "gemm", + "args": { + "m": 8, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 7168, + "n": 2560, + "out": "fp32" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.compressor.wkv+attn.compressor.wgate+attn.indexer.compressor.wkv+attn.indexer.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro/attn.compressor.wkv+attn.compressor.wgate+attn.indexer.compressor.wkv+attn.indexer.compressor.wgate" + ] + }, + { + "type": "gemm", + "args": { + "m": 12, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 7168, + "n": 2560, + "out": "fp32" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.compressor.wkv+attn.compressor.wgate+attn.indexer.compressor.wkv+attn.indexer.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro/attn.compressor.wkv+attn.compressor.wgate+attn.indexer.compressor.wkv+attn.indexer.compressor.wgate" + ] + }, + { + "type": "gemm", + "args": { + "m": 16, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 7168, + "n": 2560, + "out": "fp32" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.compressor.wkv+attn.compressor.wgate+attn.indexer.compressor.wkv+attn.indexer.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro/attn.compressor.wkv+attn.compressor.wgate+attn.indexer.compressor.wkv+attn.indexer.compressor.wgate" + ] + }, + { + "type": "gemm", + "args": { + "m": 24, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 7168, + "n": 2560, + "out": "fp32" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.compressor.wkv+attn.compressor.wgate+attn.indexer.compressor.wkv+attn.indexer.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro/attn.compressor.wkv+attn.compressor.wgate+attn.indexer.compressor.wkv+attn.indexer.compressor.wgate" + ] + }, + { + "type": "gemm", + "args": { + "m": 32, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 7168, + "n": 2560, + "out": "fp32" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.compressor.wkv+attn.compressor.wgate+attn.indexer.compressor.wkv+attn.indexer.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro/attn.compressor.wkv+attn.compressor.wgate+attn.indexer.compressor.wkv+attn.indexer.compressor.wgate" + ] + }, + { + "type": "gemm", + "args": { + "m": 48, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 7168, + "n": 2560, + "out": "fp32" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.compressor.wkv+attn.compressor.wgate+attn.indexer.compressor.wkv+attn.indexer.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro/attn.compressor.wkv+attn.compressor.wgate+attn.indexer.compressor.wkv+attn.indexer.compressor.wgate" + ] + }, + { + "type": "gemm", + "args": { + "m": 64, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 7168, + "n": 2560, + "out": "fp32" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.compressor.wkv+attn.compressor.wgate+attn.indexer.compressor.wkv+attn.indexer.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro/attn.compressor.wkv+attn.compressor.wgate+attn.indexer.compressor.wkv+attn.indexer.compressor.wgate" + ] + }, + { + "type": "gemm", + "args": { + "m": 96, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 7168, + "n": 2560, + "out": "fp32" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.compressor.wkv+attn.compressor.wgate+attn.indexer.compressor.wkv+attn.indexer.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro/attn.compressor.wkv+attn.compressor.wgate+attn.indexer.compressor.wkv+attn.indexer.compressor.wgate" + ] + }, + { + "type": "gemm", + "args": { + "m": 128, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 7168, + "n": 2560, + "out": "fp32" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.compressor.wkv+attn.compressor.wgate+attn.indexer.compressor.wkv+attn.indexer.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro/attn.compressor.wkv+attn.compressor.wgate+attn.indexer.compressor.wkv+attn.indexer.compressor.wgate" + ] + }, + { + "type": "gemm", + "args": { + "m": 192, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 7168, + "n": 2560, + "out": "fp32" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.compressor.wkv+attn.compressor.wgate+attn.indexer.compressor.wkv+attn.indexer.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro/attn.compressor.wkv+attn.compressor.wgate+attn.indexer.compressor.wkv+attn.indexer.compressor.wgate" + ] + }, + { + "type": "gemm", + "args": { + "m": 256, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 7168, + "n": 2560, + "out": "fp32" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.compressor.wkv+attn.compressor.wgate+attn.indexer.compressor.wkv+attn.indexer.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro/attn.compressor.wkv+attn.compressor.wgate+attn.indexer.compressor.wkv+attn.indexer.compressor.wgate" + ] + }, + { + "type": "gemm", + "args": { + "m": 384, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 7168, + "n": 2560, + "out": "fp32" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.compressor.wkv+attn.compressor.wgate+attn.indexer.compressor.wkv+attn.indexer.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro/attn.compressor.wkv+attn.compressor.wgate+attn.indexer.compressor.wkv+attn.indexer.compressor.wgate" + ] + }, + { + "type": "gemm", + "args": { + "m": 512, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 7168, + "n": 2560, + "out": "fp32" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.compressor.wkv+attn.compressor.wgate+attn.indexer.compressor.wkv+attn.indexer.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro/attn.compressor.wkv+attn.compressor.wgate+attn.indexer.compressor.wkv+attn.indexer.compressor.wgate" + ] + }, + { + "type": "gemm", + "args": { + "m": 768, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 7168, + "n": 2560, + "out": "fp32" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.compressor.wkv+attn.compressor.wgate+attn.indexer.compressor.wkv+attn.indexer.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro/attn.compressor.wkv+attn.compressor.wgate+attn.indexer.compressor.wkv+attn.indexer.compressor.wgate" + ] + }, + { + "type": "gemm", + "args": { + "m": 1024, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 7168, + "n": 2560, + "out": "fp32" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.compressor.wkv+attn.compressor.wgate+attn.indexer.compressor.wkv+attn.indexer.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro/attn.compressor.wkv+attn.compressor.wgate+attn.indexer.compressor.wkv+attn.indexer.compressor.wgate" + ] + }, + { + "type": "gemm", + "args": { + "m": 1536, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 7168, + "n": 2560, + "out": "fp32" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.compressor.wkv+attn.compressor.wgate+attn.indexer.compressor.wkv+attn.indexer.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro/attn.compressor.wkv+attn.compressor.wgate+attn.indexer.compressor.wkv+attn.indexer.compressor.wgate" + ] + }, + { + "type": "gemm", + "args": { + "m": 2048, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 7168, + "n": 2560, + "out": "fp32" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.compressor.wkv+attn.compressor.wgate+attn.indexer.compressor.wkv+attn.indexer.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro/attn.compressor.wkv+attn.compressor.wgate+attn.indexer.compressor.wkv+attn.indexer.compressor.wgate" + ] + }, + { + "type": "gemm", + "args": { + "m": 3072, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 7168, + "n": 2560, + "out": "fp32" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.compressor.wkv+attn.compressor.wgate+attn.indexer.compressor.wkv+attn.indexer.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro/attn.compressor.wkv+attn.compressor.wgate+attn.indexer.compressor.wkv+attn.indexer.compressor.wgate" + ] + }, + { + "type": "gemm", + "args": { + "m": 4096, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 7168, + "n": 2560, + "out": "fp32" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.compressor.wkv+attn.compressor.wgate+attn.indexer.compressor.wkv+attn.indexer.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro/attn.compressor.wkv+attn.compressor.wgate+attn.indexer.compressor.wkv+attn.indexer.compressor.wgate" + ] + }, + { + "type": "gemm", + "args": { + "m": 6144, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 7168, + "n": 2560, + "out": "fp32" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.compressor.wkv+attn.compressor.wgate+attn.indexer.compressor.wkv+attn.indexer.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro/attn.compressor.wkv+attn.compressor.wgate+attn.indexer.compressor.wkv+attn.indexer.compressor.wgate" + ] + }, + { + "type": "gemm", + "args": { + "m": 8192, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 7168, + "n": 2560, + "out": "fp32" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.compressor.wkv+attn.compressor.wgate+attn.indexer.compressor.wkv+attn.indexer.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro/attn.compressor.wkv+attn.compressor.wgate+attn.indexer.compressor.wkv+attn.indexer.compressor.wgate" + ] + }, + { + "type": "gemm", + "args": { + "m": 12288, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 7168, + "n": 2560, + "out": "fp32" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.compressor.wkv+attn.compressor.wgate+attn.indexer.compressor.wkv+attn.indexer.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro/attn.compressor.wkv+attn.compressor.wgate+attn.indexer.compressor.wkv+attn.indexer.compressor.wgate" + ] + }, + { + "type": "gemm", + "args": { + "m": 16384, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 7168, + "n": 2560, + "out": "fp32" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.compressor.wkv+attn.compressor.wgate+attn.indexer.compressor.wkv+attn.indexer.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro/attn.compressor.wkv+attn.compressor.wgate+attn.indexer.compressor.wkv+attn.indexer.compressor.wgate" + ] + }, + { + "type": "gemm", + "args": { + "m": 24576, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 7168, + "n": 2560, + "out": "fp32" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.compressor.wkv+attn.compressor.wgate+attn.indexer.compressor.wkv+attn.indexer.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro/attn.compressor.wkv+attn.compressor.wgate+attn.indexer.compressor.wkv+attn.indexer.compressor.wgate" + ] + }, + { + "type": "gemm", + "args": { + "m": 32768, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 7168, + "n": 2560, + "out": "fp32" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.compressor.wkv+attn.compressor.wgate+attn.indexer.compressor.wkv+attn.indexer.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro/attn.compressor.wkv+attn.compressor.wgate+attn.indexer.compressor.wkv+attn.indexer.compressor.wgate" + ] + }, + { + "type": "gemm", + "args": { + "m": 49152, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 7168, + "n": 2560, + "out": "fp32" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.compressor.wkv+attn.compressor.wgate+attn.indexer.compressor.wkv+attn.indexer.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro/attn.compressor.wkv+attn.compressor.wgate+attn.indexer.compressor.wkv+attn.indexer.compressor.wgate" + ] + }, + { + "type": "gemm", + "args": { + "m": 65536, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 7168, + "n": 2560, + "out": "fp32" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.compressor.wkv+attn.compressor.wgate+attn.indexer.compressor.wkv+attn.indexer.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro/attn.compressor.wkv+attn.compressor.wgate+attn.indexer.compressor.wkv+attn.indexer.compressor.wgate" + ] + }, + { + "type": "gemm", + "args": { + "m": 1, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 7168, + "n": 512, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.compressor.wkv+attn.indexer.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.compressor.wkv+attn.indexer.compressor.wgate" + ] + }, + { + "type": "gemm", + "args": { + "m": 2, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 7168, + "n": 512, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.compressor.wkv+attn.indexer.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.compressor.wkv+attn.indexer.compressor.wgate" + ] + }, + { + "type": "gemm", + "args": { + "m": 3, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 7168, + "n": 512, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.compressor.wkv+attn.indexer.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.compressor.wkv+attn.indexer.compressor.wgate" + ] + }, + { + "type": "gemm", + "args": { + "m": 4, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 7168, + "n": 512, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.compressor.wkv+attn.indexer.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.compressor.wkv+attn.indexer.compressor.wgate" + ] + }, + { + "type": "gemm", + "args": { + "m": 6, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 7168, + "n": 512, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.compressor.wkv+attn.indexer.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.compressor.wkv+attn.indexer.compressor.wgate" + ] + }, + { + "type": "gemm", + "args": { + "m": 8, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 7168, + "n": 512, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.compressor.wkv+attn.indexer.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.compressor.wkv+attn.indexer.compressor.wgate" + ] + }, + { + "type": "gemm", + "args": { + "m": 12, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 7168, + "n": 512, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.compressor.wkv+attn.indexer.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.compressor.wkv+attn.indexer.compressor.wgate" + ] + }, + { + "type": "gemm", + "args": { + "m": 16, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 7168, + "n": 512, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.compressor.wkv+attn.indexer.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.compressor.wkv+attn.indexer.compressor.wgate" + ] + }, + { + "type": "gemm", + "args": { + "m": 24, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 7168, + "n": 512, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.compressor.wkv+attn.indexer.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.compressor.wkv+attn.indexer.compressor.wgate" + ] + }, + { + "type": "gemm", + "args": { + "m": 32, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 7168, + "n": 512, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.compressor.wkv+attn.indexer.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.compressor.wkv+attn.indexer.compressor.wgate" + ] + }, + { + "type": "gemm", + "args": { + "m": 48, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 7168, + "n": 512, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.compressor.wkv+attn.indexer.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.compressor.wkv+attn.indexer.compressor.wgate" + ] + }, + { + "type": "gemm", + "args": { + "m": 64, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 7168, + "n": 512, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.compressor.wkv+attn.indexer.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.compressor.wkv+attn.indexer.compressor.wgate" + ] + }, + { + "type": "gemm", + "args": { + "m": 96, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 7168, + "n": 512, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.compressor.wkv+attn.indexer.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.compressor.wkv+attn.indexer.compressor.wgate" + ] + }, + { + "type": "gemm", + "args": { + "m": 128, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 7168, + "n": 512, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.compressor.wkv+attn.indexer.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.compressor.wkv+attn.indexer.compressor.wgate" + ] + }, + { + "type": "gemm", + "args": { + "m": 192, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 7168, + "n": 512, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.compressor.wkv+attn.indexer.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.compressor.wkv+attn.indexer.compressor.wgate" + ] + }, + { + "type": "gemm", + "args": { + "m": 256, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 7168, + "n": 512, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.compressor.wkv+attn.indexer.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.compressor.wkv+attn.indexer.compressor.wgate" + ] + }, + { + "type": "gemm", + "args": { + "m": 384, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 7168, + "n": 512, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.compressor.wkv+attn.indexer.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.compressor.wkv+attn.indexer.compressor.wgate" + ] + }, + { + "type": "gemm", + "args": { + "m": 512, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 7168, + "n": 512, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.compressor.wkv+attn.indexer.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.compressor.wkv+attn.indexer.compressor.wgate" + ] + }, + { + "type": "gemm", + "args": { + "m": 768, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 7168, + "n": 512, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.compressor.wkv+attn.indexer.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.compressor.wkv+attn.indexer.compressor.wgate" + ] + }, + { + "type": "gemm", + "args": { + "m": 1024, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 7168, + "n": 512, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.compressor.wkv+attn.indexer.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.compressor.wkv+attn.indexer.compressor.wgate" + ] + }, + { + "type": "gemm", + "args": { + "m": 1536, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 7168, + "n": 512, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.compressor.wkv+attn.indexer.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.compressor.wkv+attn.indexer.compressor.wgate" + ] + }, + { + "type": "gemm", + "args": { + "m": 2048, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 7168, + "n": 512, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.compressor.wkv+attn.indexer.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.compressor.wkv+attn.indexer.compressor.wgate" + ] + }, + { + "type": "gemm", + "args": { + "m": 3072, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 7168, + "n": 512, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.compressor.wkv+attn.indexer.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.compressor.wkv+attn.indexer.compressor.wgate" + ] + }, + { + "type": "gemm", + "args": { + "m": 4096, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 7168, + "n": 512, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.compressor.wkv+attn.indexer.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.compressor.wkv+attn.indexer.compressor.wgate" + ] + }, + { + "type": "gemm", + "args": { + "m": 6144, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 7168, + "n": 512, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.compressor.wkv+attn.indexer.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.compressor.wkv+attn.indexer.compressor.wgate" + ] + }, + { + "type": "gemm", + "args": { + "m": 8192, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 7168, + "n": 512, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.compressor.wkv+attn.indexer.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.compressor.wkv+attn.indexer.compressor.wgate" + ] + }, + { + "type": "gemm", + "args": { + "m": 12288, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 7168, + "n": 512, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.compressor.wkv+attn.indexer.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.compressor.wkv+attn.indexer.compressor.wgate" + ] + }, + { + "type": "gemm", + "args": { + "m": 16384, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 7168, + "n": 512, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.compressor.wkv+attn.indexer.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.compressor.wkv+attn.indexer.compressor.wgate" + ] + }, + { + "type": "gemm", + "args": { + "m": 24576, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 7168, + "n": 512, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.compressor.wkv+attn.indexer.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.compressor.wkv+attn.indexer.compressor.wgate" + ] + }, + { + "type": "gemm", + "args": { + "m": 32768, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 7168, + "n": 512, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.compressor.wkv+attn.indexer.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.compressor.wkv+attn.indexer.compressor.wgate" + ] + }, + { + "type": "gemm", + "args": { + "m": 49152, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 7168, + "n": 512, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.compressor.wkv+attn.indexer.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.compressor.wkv+attn.indexer.compressor.wgate" + ] + }, + { + "type": "gemm", + "args": { + "m": 65536, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 7168, + "n": 512, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.compressor.wkv+attn.indexer.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.compressor.wkv+attn.indexer.compressor.wgate" + ] + }, + { + "type": "gemm", + "args": { + "m": 1, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 7168, + "n": 512, + "out": "fp32" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.compressor.wkv+attn.indexer.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.compressor.wkv+attn.indexer.compressor.wgate" + ] + }, + { + "type": "gemm", + "args": { + "m": 2, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 7168, + "n": 512, + "out": "fp32" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.compressor.wkv+attn.indexer.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.compressor.wkv+attn.indexer.compressor.wgate" + ] + }, + { + "type": "gemm", + "args": { + "m": 3, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 7168, + "n": 512, + "out": "fp32" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.compressor.wkv+attn.indexer.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.compressor.wkv+attn.indexer.compressor.wgate" + ] + }, + { + "type": "gemm", + "args": { + "m": 4, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 7168, + "n": 512, + "out": "fp32" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.compressor.wkv+attn.indexer.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.compressor.wkv+attn.indexer.compressor.wgate" + ] + }, + { + "type": "gemm", + "args": { + "m": 6, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 7168, + "n": 512, + "out": "fp32" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.compressor.wkv+attn.indexer.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.compressor.wkv+attn.indexer.compressor.wgate" + ] + }, + { + "type": "gemm", + "args": { + "m": 8, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 7168, + "n": 512, + "out": "fp32" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.compressor.wkv+attn.indexer.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.compressor.wkv+attn.indexer.compressor.wgate" + ] + }, + { + "type": "gemm", + "args": { + "m": 12, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 7168, + "n": 512, + "out": "fp32" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.compressor.wkv+attn.indexer.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.compressor.wkv+attn.indexer.compressor.wgate" + ] + }, + { + "type": "gemm", + "args": { + "m": 16, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 7168, + "n": 512, + "out": "fp32" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.compressor.wkv+attn.indexer.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.compressor.wkv+attn.indexer.compressor.wgate" + ] + }, + { + "type": "gemm", + "args": { + "m": 24, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 7168, + "n": 512, + "out": "fp32" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.compressor.wkv+attn.indexer.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.compressor.wkv+attn.indexer.compressor.wgate" + ] + }, + { + "type": "gemm", + "args": { + "m": 32, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 7168, + "n": 512, + "out": "fp32" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.compressor.wkv+attn.indexer.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.compressor.wkv+attn.indexer.compressor.wgate" + ] + }, + { + "type": "gemm", + "args": { + "m": 48, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 7168, + "n": 512, + "out": "fp32" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.compressor.wkv+attn.indexer.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.compressor.wkv+attn.indexer.compressor.wgate" + ] + }, + { + "type": "gemm", + "args": { + "m": 64, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 7168, + "n": 512, + "out": "fp32" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.compressor.wkv+attn.indexer.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.compressor.wkv+attn.indexer.compressor.wgate" + ] + }, + { + "type": "gemm", + "args": { + "m": 96, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 7168, + "n": 512, + "out": "fp32" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.compressor.wkv+attn.indexer.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.compressor.wkv+attn.indexer.compressor.wgate" + ] + }, + { + "type": "gemm", + "args": { + "m": 128, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 7168, + "n": 512, + "out": "fp32" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.compressor.wkv+attn.indexer.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.compressor.wkv+attn.indexer.compressor.wgate" + ] + }, + { + "type": "gemm", + "args": { + "m": 192, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 7168, + "n": 512, + "out": "fp32" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.compressor.wkv+attn.indexer.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.compressor.wkv+attn.indexer.compressor.wgate" + ] + }, + { + "type": "gemm", + "args": { + "m": 256, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 7168, + "n": 512, + "out": "fp32" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.compressor.wkv+attn.indexer.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.compressor.wkv+attn.indexer.compressor.wgate" + ] + }, + { + "type": "gemm", + "args": { + "m": 384, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 7168, + "n": 512, + "out": "fp32" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.compressor.wkv+attn.indexer.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.compressor.wkv+attn.indexer.compressor.wgate" + ] + }, + { + "type": "gemm", + "args": { + "m": 512, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 7168, + "n": 512, + "out": "fp32" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.compressor.wkv+attn.indexer.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.compressor.wkv+attn.indexer.compressor.wgate" + ] + }, + { + "type": "gemm", + "args": { + "m": 768, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 7168, + "n": 512, + "out": "fp32" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.compressor.wkv+attn.indexer.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.compressor.wkv+attn.indexer.compressor.wgate" + ] + }, + { + "type": "gemm", + "args": { + "m": 1024, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 7168, + "n": 512, + "out": "fp32" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.compressor.wkv+attn.indexer.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.compressor.wkv+attn.indexer.compressor.wgate" + ] + }, + { + "type": "gemm", + "args": { + "m": 1536, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 7168, + "n": 512, + "out": "fp32" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.compressor.wkv+attn.indexer.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.compressor.wkv+attn.indexer.compressor.wgate" + ] + }, + { + "type": "gemm", + "args": { + "m": 2048, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 7168, + "n": 512, + "out": "fp32" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.compressor.wkv+attn.indexer.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.compressor.wkv+attn.indexer.compressor.wgate" + ] + }, + { + "type": "gemm", + "args": { + "m": 3072, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 7168, + "n": 512, + "out": "fp32" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.compressor.wkv+attn.indexer.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.compressor.wkv+attn.indexer.compressor.wgate" + ] + }, + { + "type": "gemm", + "args": { + "m": 4096, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 7168, + "n": 512, + "out": "fp32" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.compressor.wkv+attn.indexer.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.compressor.wkv+attn.indexer.compressor.wgate" + ] + }, + { + "type": "gemm", + "args": { + "m": 6144, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 7168, + "n": 512, + "out": "fp32" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.compressor.wkv+attn.indexer.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.compressor.wkv+attn.indexer.compressor.wgate" + ] + }, + { + "type": "gemm", + "args": { + "m": 8192, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 7168, + "n": 512, + "out": "fp32" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.compressor.wkv+attn.indexer.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.compressor.wkv+attn.indexer.compressor.wgate" + ] + }, + { + "type": "gemm", + "args": { + "m": 12288, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 7168, + "n": 512, + "out": "fp32" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.compressor.wkv+attn.indexer.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.compressor.wkv+attn.indexer.compressor.wgate" + ] + }, + { + "type": "gemm", + "args": { + "m": 16384, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 7168, + "n": 512, + "out": "fp32" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.compressor.wkv+attn.indexer.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.compressor.wkv+attn.indexer.compressor.wgate" + ] + }, + { + "type": "gemm", + "args": { + "m": 24576, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 7168, + "n": 512, + "out": "fp32" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.compressor.wkv+attn.indexer.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.compressor.wkv+attn.indexer.compressor.wgate" + ] + }, + { + "type": "gemm", + "args": { + "m": 32768, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 7168, + "n": 512, + "out": "fp32" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.compressor.wkv+attn.indexer.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.compressor.wkv+attn.indexer.compressor.wgate" + ] + }, + { + "type": "gemm", + "args": { + "m": 49152, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 7168, + "n": 512, + "out": "fp32" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.compressor.wkv+attn.indexer.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.compressor.wkv+attn.indexer.compressor.wgate" + ] + }, + { + "type": "gemm", + "args": { + "m": 65536, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 7168, + "n": 512, + "out": "fp32" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.compressor.wkv+attn.indexer.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.compressor.wkv+attn.indexer.compressor.wgate" + ] + }, + { + "type": "gemm", + "args": { + "m": 1, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 7168, + "n": 64, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.weights_proj", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.weights_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 2, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 7168, + "n": 64, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.weights_proj", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.weights_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 3, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 7168, + "n": 64, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.weights_proj", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.weights_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 4, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 7168, + "n": 64, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.weights_proj", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.weights_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 6, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 7168, + "n": 64, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.weights_proj", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.weights_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 8, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 7168, + "n": 64, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.weights_proj", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.weights_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 12, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 7168, + "n": 64, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.weights_proj", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.weights_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 16, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 7168, + "n": 64, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.weights_proj", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.weights_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 24, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 7168, + "n": 64, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.weights_proj", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.weights_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 32, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 7168, + "n": 64, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.weights_proj", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.weights_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 48, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 7168, + "n": 64, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.weights_proj", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.weights_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 64, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 7168, + "n": 64, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.weights_proj", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.weights_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 96, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 7168, + "n": 64, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.weights_proj", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.weights_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 128, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 7168, + "n": 64, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.weights_proj", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.weights_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 192, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 7168, + "n": 64, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.weights_proj", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.weights_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 256, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 7168, + "n": 64, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.weights_proj", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.weights_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 384, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 7168, + "n": 64, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.weights_proj", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.weights_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 512, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 7168, + "n": 64, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.weights_proj", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.weights_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 768, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 7168, + "n": 64, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.weights_proj", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.weights_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 1024, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 7168, + "n": 64, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.weights_proj", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.weights_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 1536, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 7168, + "n": 64, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.weights_proj", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.weights_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 2048, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 7168, + "n": 64, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.weights_proj", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.weights_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 3072, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 7168, + "n": 64, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.weights_proj", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.weights_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 4096, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 7168, + "n": 64, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.weights_proj", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.weights_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 6144, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 7168, + "n": 64, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.weights_proj", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.weights_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 8192, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 7168, + "n": 64, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.weights_proj", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.weights_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 12288, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 7168, + "n": 64, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.weights_proj", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.weights_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 16384, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 7168, + "n": 64, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.weights_proj", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.weights_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 24576, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 7168, + "n": 64, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.weights_proj", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.weights_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 32768, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 7168, + "n": 64, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.weights_proj", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.weights_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 49152, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 7168, + "n": 64, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.weights_proj", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.weights_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 65536, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 7168, + "n": 64, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.weights_proj", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.weights_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 1, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 128 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + }, + "k": 1536, + "n": 8192, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.wq_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 2, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 128 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + }, + "k": 1536, + "n": 8192, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.wq_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 3, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 128 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + }, + "k": 1536, + "n": 8192, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.wq_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 4, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 128 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + }, + "k": 1536, + "n": 8192, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.wq_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 6, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 128 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + }, + "k": 1536, + "n": 8192, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.wq_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 8, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 128 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + }, + "k": 1536, + "n": 8192, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.wq_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 12, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 128 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + }, + "k": 1536, + "n": 8192, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.wq_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 16, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 128 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + }, + "k": 1536, + "n": 8192, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.wq_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 24, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 128 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + }, + "k": 1536, + "n": 8192, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.wq_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 32, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 128 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + }, + "k": 1536, + "n": 8192, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.wq_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 48, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 128 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + }, + "k": 1536, + "n": 8192, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.wq_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 64, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 128 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + }, + "k": 1536, + "n": 8192, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.wq_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 96, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 128 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + }, + "k": 1536, + "n": 8192, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.wq_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 128, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 128 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + }, + "k": 1536, + "n": 8192, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.wq_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 192, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 128 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + }, + "k": 1536, + "n": 8192, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.wq_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 256, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 128 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + }, + "k": 1536, + "n": 8192, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.wq_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 384, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 128 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + }, + "k": 1536, + "n": 8192, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.wq_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 512, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 128 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + }, + "k": 1536, + "n": 8192, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.wq_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 768, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 128 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + }, + "k": 1536, + "n": 8192, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.wq_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 1024, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 128 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + }, + "k": 1536, + "n": 8192, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.wq_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 128 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + }, + "k": 1536, + "n": 8192, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.wq_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 2048, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 128 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + }, + "k": 1536, + "n": 8192, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.wq_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 3072, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 128 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + }, + "k": 1536, + "n": 8192, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.wq_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 4096, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 128 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + }, + "k": 1536, + "n": 8192, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.wq_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 6144, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 128 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + }, + "k": 1536, + "n": 8192, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.wq_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 8192, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 128 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + }, + "k": 1536, + "n": 8192, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.wq_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 12288, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 128 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + }, + "k": 1536, + "n": 8192, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.wq_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 16384, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 128 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + }, + "k": 1536, + "n": 8192, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.wq_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 24576, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 128 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + }, + "k": 1536, + "n": 8192, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.wq_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 32768, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 128 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + }, + "k": 1536, + "n": 8192, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.wq_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 49152, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 128 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + }, + "k": 1536, + "n": 8192, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.wq_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 65536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 128 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + }, + "k": 1536, + "n": 8192, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.wq_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 1, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 128 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + }, + "k": 16384, + "n": 7168, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 2, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 128 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + }, + "k": 16384, + "n": 7168, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 3, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 128 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + }, + "k": 16384, + "n": 7168, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 4, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 128 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + }, + "k": 16384, + "n": 7168, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 6, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 128 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + }, + "k": 16384, + "n": 7168, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 8, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 128 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + }, + "k": 16384, + "n": 7168, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 12, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 128 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + }, + "k": 16384, + "n": 7168, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 16, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 128 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + }, + "k": 16384, + "n": 7168, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 24, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 128 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + }, + "k": 16384, + "n": 7168, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 32, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 128 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + }, + "k": 16384, + "n": 7168, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 48, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 128 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + }, + "k": 16384, + "n": 7168, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 64, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 128 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + }, + "k": 16384, + "n": 7168, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 96, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 128 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + }, + "k": 16384, + "n": 7168, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 128, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 128 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + }, + "k": 16384, + "n": 7168, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 192, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 128 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + }, + "k": 16384, + "n": 7168, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 256, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 128 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + }, + "k": 16384, + "n": 7168, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 384, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 128 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + }, + "k": 16384, + "n": 7168, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 512, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 128 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + }, + "k": 16384, + "n": 7168, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 768, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 128 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + }, + "k": 16384, + "n": 7168, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 1024, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 128 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + }, + "k": 16384, + "n": 7168, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 128 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + }, + "k": 16384, + "n": 7168, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 2048, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 128 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + }, + "k": 16384, + "n": 7168, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 3072, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 128 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + }, + "k": 16384, + "n": 7168, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 4096, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 128 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + }, + "k": 16384, + "n": 7168, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 6144, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 128 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + }, + "k": 16384, + "n": 7168, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 8192, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 128 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + }, + "k": 16384, + "n": 7168, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 12288, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 128 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + }, + "k": 16384, + "n": 7168, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 16384, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 128 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + }, + "k": 16384, + "n": 7168, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 24576, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 128 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + }, + "k": 16384, + "n": 7168, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 32768, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 128 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + }, + "k": 16384, + "n": 7168, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 49152, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 128 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + }, + "k": 16384, + "n": 7168, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 65536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 128 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + }, + "k": 16384, + "n": 7168, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 1, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 128 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + }, + "k": 7168, + "n": 2048, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_a+attn.wkv", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_a+attn.wkv" + ] + }, + { + "type": "gemm", + "args": { + "m": 2, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 128 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + }, + "k": 7168, + "n": 2048, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_a+attn.wkv", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_a+attn.wkv" + ] + }, + { + "type": "gemm", + "args": { + "m": 3, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 128 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + }, + "k": 7168, + "n": 2048, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_a+attn.wkv", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_a+attn.wkv" + ] + }, + { + "type": "gemm", + "args": { + "m": 4, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 128 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + }, + "k": 7168, + "n": 2048, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_a+attn.wkv", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_a+attn.wkv" + ] + }, + { + "type": "gemm", + "args": { + "m": 6, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 128 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + }, + "k": 7168, + "n": 2048, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_a+attn.wkv", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_a+attn.wkv" + ] + }, + { + "type": "gemm", + "args": { + "m": 8, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 128 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + }, + "k": 7168, + "n": 2048, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_a+attn.wkv", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_a+attn.wkv" + ] + }, + { + "type": "gemm", + "args": { + "m": 12, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 128 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + }, + "k": 7168, + "n": 2048, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_a+attn.wkv", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_a+attn.wkv" + ] + }, + { + "type": "gemm", + "args": { + "m": 16, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 128 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + }, + "k": 7168, + "n": 2048, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_a+attn.wkv", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_a+attn.wkv" + ] + }, + { + "type": "gemm", + "args": { + "m": 24, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 128 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + }, + "k": 7168, + "n": 2048, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_a+attn.wkv", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_a+attn.wkv" + ] + }, + { + "type": "gemm", + "args": { + "m": 32, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 128 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + }, + "k": 7168, + "n": 2048, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_a+attn.wkv", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_a+attn.wkv" + ] + }, + { + "type": "gemm", + "args": { + "m": 48, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 128 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + }, + "k": 7168, + "n": 2048, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_a+attn.wkv", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_a+attn.wkv" + ] + }, + { + "type": "gemm", + "args": { + "m": 64, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 128 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + }, + "k": 7168, + "n": 2048, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_a+attn.wkv", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_a+attn.wkv" + ] + }, + { + "type": "gemm", + "args": { + "m": 96, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 128 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + }, + "k": 7168, + "n": 2048, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_a+attn.wkv", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_a+attn.wkv" + ] + }, + { + "type": "gemm", + "args": { + "m": 128, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 128 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + }, + "k": 7168, + "n": 2048, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_a+attn.wkv", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_a+attn.wkv" + ] + }, + { + "type": "gemm", + "args": { + "m": 192, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 128 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + }, + "k": 7168, + "n": 2048, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_a+attn.wkv", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_a+attn.wkv" + ] + }, + { + "type": "gemm", + "args": { + "m": 256, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 128 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + }, + "k": 7168, + "n": 2048, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_a+attn.wkv", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_a+attn.wkv" + ] + }, + { + "type": "gemm", + "args": { + "m": 384, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 128 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + }, + "k": 7168, + "n": 2048, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_a+attn.wkv", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_a+attn.wkv" + ] + }, + { + "type": "gemm", + "args": { + "m": 512, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 128 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + }, + "k": 7168, + "n": 2048, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_a+attn.wkv", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_a+attn.wkv" + ] + }, + { + "type": "gemm", + "args": { + "m": 768, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 128 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + }, + "k": 7168, + "n": 2048, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_a+attn.wkv", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_a+attn.wkv" + ] + }, + { + "type": "gemm", + "args": { + "m": 1024, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 128 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + }, + "k": 7168, + "n": 2048, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_a+attn.wkv", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_a+attn.wkv" + ] + }, + { + "type": "gemm", + "args": { + "m": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 128 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + }, + "k": 7168, + "n": 2048, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_a+attn.wkv", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_a+attn.wkv" + ] + }, + { + "type": "gemm", + "args": { + "m": 2048, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 128 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + }, + "k": 7168, + "n": 2048, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_a+attn.wkv", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_a+attn.wkv" + ] + }, + { + "type": "gemm", + "args": { + "m": 3072, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 128 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + }, + "k": 7168, + "n": 2048, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_a+attn.wkv", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_a+attn.wkv" + ] + }, + { + "type": "gemm", + "args": { + "m": 4096, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 128 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + }, + "k": 7168, + "n": 2048, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_a+attn.wkv", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_a+attn.wkv" + ] + }, + { + "type": "gemm", + "args": { + "m": 6144, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 128 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + }, + "k": 7168, + "n": 2048, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_a+attn.wkv", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_a+attn.wkv" + ] + }, + { + "type": "gemm", + "args": { + "m": 8192, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 128 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + }, + "k": 7168, + "n": 2048, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_a+attn.wkv", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_a+attn.wkv" + ] + }, + { + "type": "gemm", + "args": { + "m": 12288, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 128 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + }, + "k": 7168, + "n": 2048, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_a+attn.wkv", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_a+attn.wkv" + ] + }, + { + "type": "gemm", + "args": { + "m": 16384, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 128 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + }, + "k": 7168, + "n": 2048, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_a+attn.wkv", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_a+attn.wkv" + ] + }, + { + "type": "gemm", + "args": { + "m": 24576, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 128 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + }, + "k": 7168, + "n": 2048, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_a+attn.wkv", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_a+attn.wkv" + ] + }, + { + "type": "gemm", + "args": { + "m": 32768, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 128 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + }, + "k": 7168, + "n": 2048, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_a+attn.wkv", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_a+attn.wkv" + ] + }, + { + "type": "gemm", + "args": { + "m": 49152, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 128 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + }, + "k": 7168, + "n": 2048, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_a+attn.wkv", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_a+attn.wkv" + ] + }, + { + "type": "gemm", + "args": { + "m": 65536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 128 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + }, + "k": 7168, + "n": 2048, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_a+attn.wkv", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_a+attn.wkv" + ] + }, + { + "type": "gemm", + "args": { + "m": 1, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 128 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + }, + "k": 1536, + "n": 65536, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 2, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 128 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + }, + "k": 1536, + "n": 65536, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 3, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 128 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + }, + "k": 1536, + "n": 65536, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 4, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 128 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + }, + "k": 1536, + "n": 65536, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 6, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 128 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + }, + "k": 1536, + "n": 65536, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 8, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 128 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + }, + "k": 1536, + "n": 65536, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 12, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 128 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + }, + "k": 1536, + "n": 65536, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 16, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 128 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + }, + "k": 1536, + "n": 65536, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 24, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 128 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + }, + "k": 1536, + "n": 65536, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 32, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 128 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + }, + "k": 1536, + "n": 65536, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 48, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 128 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + }, + "k": 1536, + "n": 65536, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 64, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 128 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + }, + "k": 1536, + "n": 65536, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 96, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 128 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + }, + "k": 1536, + "n": 65536, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 128, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 128 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + }, + "k": 1536, + "n": 65536, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 192, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 128 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + }, + "k": 1536, + "n": 65536, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 256, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 128 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + }, + "k": 1536, + "n": 65536, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 384, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 128 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + }, + "k": 1536, + "n": 65536, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 512, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 128 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + }, + "k": 1536, + "n": 65536, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 768, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 128 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + }, + "k": 1536, + "n": 65536, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 1024, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 128 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + }, + "k": 1536, + "n": 65536, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 128 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + }, + "k": 1536, + "n": 65536, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 2048, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 128 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + }, + "k": 1536, + "n": 65536, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 3072, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 128 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + }, + "k": 1536, + "n": 65536, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 4096, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 128 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + }, + "k": 1536, + "n": 65536, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 6144, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 128 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + }, + "k": 1536, + "n": 65536, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 8192, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 128 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + }, + "k": 1536, + "n": 65536, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 12288, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 128 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + }, + "k": 1536, + "n": 65536, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 16384, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 128 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + }, + "k": 1536, + "n": 65536, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 24576, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 128 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + }, + "k": 1536, + "n": 65536, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 32768, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 128 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + }, + "k": 1536, + "n": 65536, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 49152, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 128 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + }, + "k": 1536, + "n": 65536, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 65536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 128 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + }, + "k": 1536, + "n": 65536, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 1, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 7168, + "n": 384, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.gate", + "deepseek-ai/DeepSeek-V4-Pro/ffn.gate" + ] + }, + { + "type": "gemm", + "args": { + "m": 2, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 7168, + "n": 384, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.gate", + "deepseek-ai/DeepSeek-V4-Pro/ffn.gate" + ] + }, + { + "type": "gemm", + "args": { + "m": 3, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 7168, + "n": 384, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.gate", + "deepseek-ai/DeepSeek-V4-Pro/ffn.gate" + ] + }, + { + "type": "gemm", + "args": { + "m": 4, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 7168, + "n": 384, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.gate", + "deepseek-ai/DeepSeek-V4-Pro/ffn.gate" + ] + }, + { + "type": "gemm", + "args": { + "m": 6, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 7168, + "n": 384, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.gate", + "deepseek-ai/DeepSeek-V4-Pro/ffn.gate" + ] + }, + { + "type": "gemm", + "args": { + "m": 8, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 7168, + "n": 384, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.gate", + "deepseek-ai/DeepSeek-V4-Pro/ffn.gate" + ] + }, + { + "type": "gemm", + "args": { + "m": 12, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 7168, + "n": 384, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.gate", + "deepseek-ai/DeepSeek-V4-Pro/ffn.gate" + ] + }, + { + "type": "gemm", + "args": { + "m": 16, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 7168, + "n": 384, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.gate", + "deepseek-ai/DeepSeek-V4-Pro/ffn.gate" + ] + }, + { + "type": "gemm", + "args": { + "m": 24, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 7168, + "n": 384, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.gate", + "deepseek-ai/DeepSeek-V4-Pro/ffn.gate" + ] + }, + { + "type": "gemm", + "args": { + "m": 32, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 7168, + "n": 384, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.gate", + "deepseek-ai/DeepSeek-V4-Pro/ffn.gate" + ] + }, + { + "type": "gemm", + "args": { + "m": 48, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 7168, + "n": 384, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.gate", + "deepseek-ai/DeepSeek-V4-Pro/ffn.gate" + ] + }, + { + "type": "gemm", + "args": { + "m": 64, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 7168, + "n": 384, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.gate", + "deepseek-ai/DeepSeek-V4-Pro/ffn.gate" + ] + }, + { + "type": "gemm", + "args": { + "m": 96, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 7168, + "n": 384, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.gate", + "deepseek-ai/DeepSeek-V4-Pro/ffn.gate" + ] + }, + { + "type": "gemm", + "args": { + "m": 128, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 7168, + "n": 384, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.gate", + "deepseek-ai/DeepSeek-V4-Pro/ffn.gate" + ] + }, + { + "type": "gemm", + "args": { + "m": 192, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 7168, + "n": 384, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.gate", + "deepseek-ai/DeepSeek-V4-Pro/ffn.gate" + ] + }, + { + "type": "gemm", + "args": { + "m": 256, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 7168, + "n": 384, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.gate", + "deepseek-ai/DeepSeek-V4-Pro/ffn.gate" + ] + }, + { + "type": "gemm", + "args": { + "m": 384, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 7168, + "n": 384, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.gate", + "deepseek-ai/DeepSeek-V4-Pro/ffn.gate" + ] + }, + { + "type": "gemm", + "args": { + "m": 512, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 7168, + "n": 384, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.gate", + "deepseek-ai/DeepSeek-V4-Pro/ffn.gate" + ] + }, + { + "type": "gemm", + "args": { + "m": 768, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 7168, + "n": 384, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.gate", + "deepseek-ai/DeepSeek-V4-Pro/ffn.gate" + ] + }, + { + "type": "gemm", + "args": { + "m": 1024, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 7168, + "n": 384, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.gate", + "deepseek-ai/DeepSeek-V4-Pro/ffn.gate" + ] + }, + { + "type": "gemm", + "args": { + "m": 1536, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 7168, + "n": 384, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.gate", + "deepseek-ai/DeepSeek-V4-Pro/ffn.gate" + ] + }, + { + "type": "gemm", + "args": { + "m": 2048, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 7168, + "n": 384, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.gate", + "deepseek-ai/DeepSeek-V4-Pro/ffn.gate" + ] + }, + { + "type": "gemm", + "args": { + "m": 3072, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 7168, + "n": 384, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.gate", + "deepseek-ai/DeepSeek-V4-Pro/ffn.gate" + ] + }, + { + "type": "gemm", + "args": { + "m": 4096, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 7168, + "n": 384, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.gate", + "deepseek-ai/DeepSeek-V4-Pro/ffn.gate" + ] + }, + { + "type": "gemm", + "args": { + "m": 6144, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 7168, + "n": 384, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.gate", + "deepseek-ai/DeepSeek-V4-Pro/ffn.gate" + ] + }, + { + "type": "gemm", + "args": { + "m": 8192, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 7168, + "n": 384, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.gate", + "deepseek-ai/DeepSeek-V4-Pro/ffn.gate" + ] + }, + { + "type": "gemm", + "args": { + "m": 12288, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 7168, + "n": 384, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.gate", + "deepseek-ai/DeepSeek-V4-Pro/ffn.gate" + ] + }, + { + "type": "gemm", + "args": { + "m": 16384, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 7168, + "n": 384, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.gate", + "deepseek-ai/DeepSeek-V4-Pro/ffn.gate" + ] + }, + { + "type": "gemm", + "args": { + "m": 24576, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 7168, + "n": 384, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.gate", + "deepseek-ai/DeepSeek-V4-Pro/ffn.gate" + ] + }, + { + "type": "gemm", + "args": { + "m": 32768, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 7168, + "n": 384, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.gate", + "deepseek-ai/DeepSeek-V4-Pro/ffn.gate" + ] + }, + { + "type": "gemm", + "args": { + "m": 49152, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 7168, + "n": 384, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.gate", + "deepseek-ai/DeepSeek-V4-Pro/ffn.gate" + ] + }, + { + "type": "gemm", + "args": { + "m": 65536, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 7168, + "n": 384, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.gate", + "deepseek-ai/DeepSeek-V4-Pro/ffn.gate" + ] + }, + { + "type": "gemm", + "args": { + "m": 1, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 7168, + "n": 384, + "out": "fp32" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.gate", + "deepseek-ai/DeepSeek-V4-Pro/ffn.gate" + ] + }, + { + "type": "gemm", + "args": { + "m": 2, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 7168, + "n": 384, + "out": "fp32" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.gate", + "deepseek-ai/DeepSeek-V4-Pro/ffn.gate" + ] + }, + { + "type": "gemm", + "args": { + "m": 3, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 7168, + "n": 384, + "out": "fp32" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.gate", + "deepseek-ai/DeepSeek-V4-Pro/ffn.gate" + ] + }, + { + "type": "gemm", + "args": { + "m": 4, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 7168, + "n": 384, + "out": "fp32" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.gate", + "deepseek-ai/DeepSeek-V4-Pro/ffn.gate" + ] + }, + { + "type": "gemm", + "args": { + "m": 6, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 7168, + "n": 384, + "out": "fp32" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.gate", + "deepseek-ai/DeepSeek-V4-Pro/ffn.gate" + ] + }, + { + "type": "gemm", + "args": { + "m": 8, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 7168, + "n": 384, + "out": "fp32" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.gate", + "deepseek-ai/DeepSeek-V4-Pro/ffn.gate" + ] + }, + { + "type": "gemm", + "args": { + "m": 12, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 7168, + "n": 384, + "out": "fp32" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.gate", + "deepseek-ai/DeepSeek-V4-Pro/ffn.gate" + ] + }, + { + "type": "gemm", + "args": { + "m": 16, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 7168, + "n": 384, + "out": "fp32" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.gate", + "deepseek-ai/DeepSeek-V4-Pro/ffn.gate" + ] + }, + { + "type": "gemm", + "args": { + "m": 24, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 7168, + "n": 384, + "out": "fp32" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.gate", + "deepseek-ai/DeepSeek-V4-Pro/ffn.gate" + ] + }, + { + "type": "gemm", + "args": { + "m": 32, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 7168, + "n": 384, + "out": "fp32" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.gate", + "deepseek-ai/DeepSeek-V4-Pro/ffn.gate" + ] + }, + { + "type": "gemm", + "args": { + "m": 48, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 7168, + "n": 384, + "out": "fp32" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.gate", + "deepseek-ai/DeepSeek-V4-Pro/ffn.gate" + ] + }, + { + "type": "gemm", + "args": { + "m": 64, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 7168, + "n": 384, + "out": "fp32" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.gate", + "deepseek-ai/DeepSeek-V4-Pro/ffn.gate" + ] + }, + { + "type": "gemm", + "args": { + "m": 96, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 7168, + "n": 384, + "out": "fp32" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.gate", + "deepseek-ai/DeepSeek-V4-Pro/ffn.gate" + ] + }, + { + "type": "gemm", + "args": { + "m": 128, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 7168, + "n": 384, + "out": "fp32" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.gate", + "deepseek-ai/DeepSeek-V4-Pro/ffn.gate" + ] + }, + { + "type": "gemm", + "args": { + "m": 192, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 7168, + "n": 384, + "out": "fp32" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.gate", + "deepseek-ai/DeepSeek-V4-Pro/ffn.gate" + ] + }, + { + "type": "gemm", + "args": { + "m": 256, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 7168, + "n": 384, + "out": "fp32" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.gate", + "deepseek-ai/DeepSeek-V4-Pro/ffn.gate" + ] + }, + { + "type": "gemm", + "args": { + "m": 384, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 7168, + "n": 384, + "out": "fp32" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.gate", + "deepseek-ai/DeepSeek-V4-Pro/ffn.gate" + ] + }, + { + "type": "gemm", + "args": { + "m": 512, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 7168, + "n": 384, + "out": "fp32" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.gate", + "deepseek-ai/DeepSeek-V4-Pro/ffn.gate" + ] + }, + { + "type": "gemm", + "args": { + "m": 768, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 7168, + "n": 384, + "out": "fp32" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.gate", + "deepseek-ai/DeepSeek-V4-Pro/ffn.gate" + ] + }, + { + "type": "gemm", + "args": { + "m": 1024, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 7168, + "n": 384, + "out": "fp32" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.gate", + "deepseek-ai/DeepSeek-V4-Pro/ffn.gate" + ] + }, + { + "type": "gemm", + "args": { + "m": 1536, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 7168, + "n": 384, + "out": "fp32" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.gate", + "deepseek-ai/DeepSeek-V4-Pro/ffn.gate" + ] + }, + { + "type": "gemm", + "args": { + "m": 2048, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 7168, + "n": 384, + "out": "fp32" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.gate", + "deepseek-ai/DeepSeek-V4-Pro/ffn.gate" + ] + }, + { + "type": "gemm", + "args": { + "m": 3072, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 7168, + "n": 384, + "out": "fp32" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.gate", + "deepseek-ai/DeepSeek-V4-Pro/ffn.gate" + ] + }, + { + "type": "gemm", + "args": { + "m": 4096, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 7168, + "n": 384, + "out": "fp32" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.gate", + "deepseek-ai/DeepSeek-V4-Pro/ffn.gate" + ] + }, + { + "type": "gemm", + "args": { + "m": 6144, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 7168, + "n": 384, + "out": "fp32" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.gate", + "deepseek-ai/DeepSeek-V4-Pro/ffn.gate" + ] + }, + { + "type": "gemm", + "args": { + "m": 8192, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 7168, + "n": 384, + "out": "fp32" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.gate", + "deepseek-ai/DeepSeek-V4-Pro/ffn.gate" + ] + }, + { + "type": "gemm", + "args": { + "m": 12288, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 7168, + "n": 384, + "out": "fp32" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.gate", + "deepseek-ai/DeepSeek-V4-Pro/ffn.gate" + ] + }, + { + "type": "gemm", + "args": { + "m": 16384, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 7168, + "n": 384, + "out": "fp32" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.gate", + "deepseek-ai/DeepSeek-V4-Pro/ffn.gate" + ] + }, + { + "type": "gemm", + "args": { + "m": 24576, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 7168, + "n": 384, + "out": "fp32" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.gate", + "deepseek-ai/DeepSeek-V4-Pro/ffn.gate" + ] + }, + { + "type": "gemm", + "args": { + "m": 32768, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 7168, + "n": 384, + "out": "fp32" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.gate", + "deepseek-ai/DeepSeek-V4-Pro/ffn.gate" + ] + }, + { + "type": "gemm", + "args": { + "m": 49152, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 7168, + "n": 384, + "out": "fp32" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.gate", + "deepseek-ai/DeepSeek-V4-Pro/ffn.gate" + ] + }, + { + "type": "gemm", + "args": { + "m": 65536, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 7168, + "n": 384, + "out": "fp32" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.gate", + "deepseek-ai/DeepSeek-V4-Pro/ffn.gate" + ] + }, + { + "type": "gemm", + "args": { + "m": 1, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 128 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + }, + "k": 7168, + "n": 6144, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+ffn.shared_experts.w3" + ] + }, + { + "type": "gemm", + "args": { + "m": 2, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 128 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + }, + "k": 7168, + "n": 6144, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+ffn.shared_experts.w3" + ] + }, + { + "type": "gemm", + "args": { + "m": 3, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 128 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + }, + "k": 7168, + "n": 6144, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+ffn.shared_experts.w3" + ] + }, + { + "type": "gemm", + "args": { + "m": 4, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 128 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + }, + "k": 7168, + "n": 6144, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+ffn.shared_experts.w3" + ] + }, + { + "type": "gemm", + "args": { + "m": 6, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 128 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + }, + "k": 7168, + "n": 6144, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+ffn.shared_experts.w3" + ] + }, + { + "type": "gemm", + "args": { + "m": 8, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 128 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + }, + "k": 7168, + "n": 6144, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+ffn.shared_experts.w3" + ] + }, + { + "type": "gemm", + "args": { + "m": 12, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 128 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + }, + "k": 7168, + "n": 6144, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+ffn.shared_experts.w3" + ] + }, + { + "type": "gemm", + "args": { + "m": 16, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 128 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + }, + "k": 7168, + "n": 6144, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+ffn.shared_experts.w3" + ] + }, + { + "type": "gemm", + "args": { + "m": 24, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 128 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + }, + "k": 7168, + "n": 6144, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+ffn.shared_experts.w3" + ] + }, + { + "type": "gemm", + "args": { + "m": 32, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 128 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + }, + "k": 7168, + "n": 6144, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+ffn.shared_experts.w3" + ] + }, + { + "type": "gemm", + "args": { + "m": 48, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 128 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + }, + "k": 7168, + "n": 6144, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+ffn.shared_experts.w3" + ] + }, + { + "type": "gemm", + "args": { + "m": 64, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 128 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + }, + "k": 7168, + "n": 6144, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+ffn.shared_experts.w3" + ] + }, + { + "type": "gemm", + "args": { + "m": 96, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 128 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + }, + "k": 7168, + "n": 6144, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+ffn.shared_experts.w3" + ] + }, + { + "type": "gemm", + "args": { + "m": 128, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 128 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + }, + "k": 7168, + "n": 6144, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+ffn.shared_experts.w3" + ] + }, + { + "type": "gemm", + "args": { + "m": 192, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 128 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + }, + "k": 7168, + "n": 6144, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+ffn.shared_experts.w3" + ] + }, + { + "type": "gemm", + "args": { + "m": 256, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 128 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + }, + "k": 7168, + "n": 6144, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+ffn.shared_experts.w3" + ] + }, + { + "type": "gemm", + "args": { + "m": 384, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 128 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + }, + "k": 7168, + "n": 6144, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+ffn.shared_experts.w3" + ] + }, + { + "type": "gemm", + "args": { + "m": 512, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 128 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + }, + "k": 7168, + "n": 6144, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+ffn.shared_experts.w3" + ] + }, + { + "type": "gemm", + "args": { + "m": 768, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 128 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + }, + "k": 7168, + "n": 6144, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+ffn.shared_experts.w3" + ] + }, + { + "type": "gemm", + "args": { + "m": 1024, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 128 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + }, + "k": 7168, + "n": 6144, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+ffn.shared_experts.w3" + ] + }, + { + "type": "gemm", + "args": { + "m": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 128 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + }, + "k": 7168, + "n": 6144, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+ffn.shared_experts.w3" + ] + }, + { + "type": "gemm", + "args": { + "m": 2048, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 128 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + }, + "k": 7168, + "n": 6144, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+ffn.shared_experts.w3" + ] + }, + { + "type": "gemm", + "args": { + "m": 3072, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 128 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + }, + "k": 7168, + "n": 6144, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+ffn.shared_experts.w3" + ] + }, + { + "type": "gemm", + "args": { + "m": 4096, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 128 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + }, + "k": 7168, + "n": 6144, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+ffn.shared_experts.w3" + ] + }, + { + "type": "gemm", + "args": { + "m": 6144, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 128 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + }, + "k": 7168, + "n": 6144, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+ffn.shared_experts.w3" + ] + }, + { + "type": "gemm", + "args": { + "m": 8192, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 128 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + }, + "k": 7168, + "n": 6144, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+ffn.shared_experts.w3" + ] + }, + { + "type": "gemm", + "args": { + "m": 12288, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 128 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + }, + "k": 7168, + "n": 6144, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+ffn.shared_experts.w3" + ] + }, + { + "type": "gemm", + "args": { + "m": 16384, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 128 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + }, + "k": 7168, + "n": 6144, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+ffn.shared_experts.w3" + ] + }, + { + "type": "gemm", + "args": { + "m": 24576, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 128 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + }, + "k": 7168, + "n": 6144, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+ffn.shared_experts.w3" + ] + }, + { + "type": "gemm", + "args": { + "m": 32768, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 128 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + }, + "k": 7168, + "n": 6144, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+ffn.shared_experts.w3" + ] + }, + { + "type": "gemm", + "args": { + "m": 49152, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 128 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + }, + "k": 7168, + "n": 6144, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+ffn.shared_experts.w3" + ] + }, + { + "type": "gemm", + "args": { + "m": 65536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 128 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + }, + "k": 7168, + "n": 6144, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+ffn.shared_experts.w3" + ] + }, + { + "type": "gemm", + "args": { + "m": 1, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 128 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + }, + "k": 3072, + "n": 7168, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w2" + ] + }, + { + "type": "gemm", + "args": { + "m": 2, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 128 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + }, + "k": 3072, + "n": 7168, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w2" + ] + }, + { + "type": "gemm", + "args": { + "m": 3, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 128 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + }, + "k": 3072, + "n": 7168, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w2" + ] + }, + { + "type": "gemm", + "args": { + "m": 4, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 128 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + }, + "k": 3072, + "n": 7168, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w2" + ] + }, + { + "type": "gemm", + "args": { + "m": 6, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 128 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + }, + "k": 3072, + "n": 7168, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w2" + ] + }, + { + "type": "gemm", + "args": { + "m": 8, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 128 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + }, + "k": 3072, + "n": 7168, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w2" + ] + }, + { + "type": "gemm", + "args": { + "m": 12, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 128 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + }, + "k": 3072, + "n": 7168, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w2" + ] + }, + { + "type": "gemm", + "args": { + "m": 16, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 128 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + }, + "k": 3072, + "n": 7168, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w2" + ] + }, + { + "type": "gemm", + "args": { + "m": 24, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 128 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + }, + "k": 3072, + "n": 7168, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w2" + ] + }, + { + "type": "gemm", + "args": { + "m": 32, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 128 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + }, + "k": 3072, + "n": 7168, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w2" + ] + }, + { + "type": "gemm", + "args": { + "m": 48, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 128 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + }, + "k": 3072, + "n": 7168, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w2" + ] + }, + { + "type": "gemm", + "args": { + "m": 64, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 128 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + }, + "k": 3072, + "n": 7168, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w2" + ] + }, + { + "type": "gemm", + "args": { + "m": 96, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 128 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + }, + "k": 3072, + "n": 7168, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w2" + ] + }, + { + "type": "gemm", + "args": { + "m": 128, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 128 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + }, + "k": 3072, + "n": 7168, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w2" + ] + }, + { + "type": "gemm", + "args": { + "m": 192, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 128 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + }, + "k": 3072, + "n": 7168, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w2" + ] + }, + { + "type": "gemm", + "args": { + "m": 256, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 128 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + }, + "k": 3072, + "n": 7168, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w2" + ] + }, + { + "type": "gemm", + "args": { + "m": 384, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 128 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + }, + "k": 3072, + "n": 7168, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w2" + ] + }, + { + "type": "gemm", + "args": { + "m": 512, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 128 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + }, + "k": 3072, + "n": 7168, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w2" + ] + }, + { + "type": "gemm", + "args": { + "m": 768, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 128 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + }, + "k": 3072, + "n": 7168, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w2" + ] + }, + { + "type": "gemm", + "args": { + "m": 1024, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 128 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + }, + "k": 3072, + "n": 7168, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w2" + ] + }, + { + "type": "gemm", + "args": { + "m": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 128 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + }, + "k": 3072, + "n": 7168, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w2" + ] + }, + { + "type": "gemm", + "args": { + "m": 2048, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 128 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + }, + "k": 3072, + "n": 7168, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w2" + ] + }, + { + "type": "gemm", + "args": { + "m": 3072, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 128 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + }, + "k": 3072, + "n": 7168, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w2" + ] + }, + { + "type": "gemm", + "args": { + "m": 4096, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 128 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + }, + "k": 3072, + "n": 7168, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w2" + ] + }, + { + "type": "gemm", + "args": { + "m": 6144, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 128 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + }, + "k": 3072, + "n": 7168, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w2" + ] + }, + { + "type": "gemm", + "args": { + "m": 8192, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 128 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + }, + "k": 3072, + "n": 7168, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w2" + ] + }, + { + "type": "gemm", + "args": { + "m": 12288, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 128 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + }, + "k": 3072, + "n": 7168, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w2" + ] + }, + { + "type": "gemm", + "args": { + "m": 16384, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 128 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + }, + "k": 3072, + "n": 7168, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w2" + ] + }, + { + "type": "gemm", + "args": { + "m": 24576, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 128 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + }, + "k": 3072, + "n": 7168, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w2" + ] + }, + { + "type": "gemm", + "args": { + "m": 32768, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 128 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + }, + "k": 3072, + "n": 7168, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w2" + ] + }, + { + "type": "gemm", + "args": { + "m": 49152, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 128 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + }, + "k": 3072, + "n": 7168, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w2" + ] + }, + { + "type": "gemm", + "args": { + "m": 65536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 128 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + }, + "k": 3072, + "n": 7168, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w2" + ] + }, + { + "type": "gemm", + "args": { + "m": 1, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 7168, + "n": 129280, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/head", + "deepseek-ai/DeepSeek-V4-Pro/head" + ] + }, + { + "type": "gemm", + "args": { + "m": 2, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 7168, + "n": 129280, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/head", + "deepseek-ai/DeepSeek-V4-Pro/head" + ] + }, + { + "type": "gemm", + "args": { + "m": 3, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 7168, + "n": 129280, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/head", + "deepseek-ai/DeepSeek-V4-Pro/head" + ] + }, + { + "type": "gemm", + "args": { + "m": 4, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 7168, + "n": 129280, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/head", + "deepseek-ai/DeepSeek-V4-Pro/head" + ] + }, + { + "type": "gemm", + "args": { + "m": 6, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 7168, + "n": 129280, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/head", + "deepseek-ai/DeepSeek-V4-Pro/head" + ] + }, + { + "type": "gemm", + "args": { + "m": 8, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 7168, + "n": 129280, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/head", + "deepseek-ai/DeepSeek-V4-Pro/head" + ] + }, + { + "type": "gemm", + "args": { + "m": 12, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 7168, + "n": 129280, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/head", + "deepseek-ai/DeepSeek-V4-Pro/head" + ] + }, + { + "type": "gemm", + "args": { + "m": 16, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 7168, + "n": 129280, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/head", + "deepseek-ai/DeepSeek-V4-Pro/head" + ] + }, + { + "type": "gemm", + "args": { + "m": 24, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 7168, + "n": 129280, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/head", + "deepseek-ai/DeepSeek-V4-Pro/head" + ] + }, + { + "type": "gemm", + "args": { + "m": 32, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 7168, + "n": 129280, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/head", + "deepseek-ai/DeepSeek-V4-Pro/head" + ] + }, + { + "type": "gemm", + "args": { + "m": 48, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 7168, + "n": 129280, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/head", + "deepseek-ai/DeepSeek-V4-Pro/head" + ] + }, + { + "type": "gemm", + "args": { + "m": 64, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 7168, + "n": 129280, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/head", + "deepseek-ai/DeepSeek-V4-Pro/head" + ] + }, + { + "type": "gemm", + "args": { + "m": 96, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 7168, + "n": 129280, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/head", + "deepseek-ai/DeepSeek-V4-Pro/head" + ] + }, + { + "type": "gemm", + "args": { + "m": 128, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 7168, + "n": 129280, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/head", + "deepseek-ai/DeepSeek-V4-Pro/head" + ] + }, + { + "type": "gemm", + "args": { + "m": 192, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 7168, + "n": 129280, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/head", + "deepseek-ai/DeepSeek-V4-Pro/head" + ] + }, + { + "type": "gemm", + "args": { + "m": 256, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 7168, + "n": 129280, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/head", + "deepseek-ai/DeepSeek-V4-Pro/head" + ] + }, + { + "type": "gemm", + "args": { + "m": 384, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 7168, + "n": 129280, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/head", + "deepseek-ai/DeepSeek-V4-Pro/head" + ] + }, + { + "type": "gemm", + "args": { + "m": 512, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 7168, + "n": 129280, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/head", + "deepseek-ai/DeepSeek-V4-Pro/head" + ] + }, + { + "type": "gemm", + "args": { + "m": 768, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 7168, + "n": 129280, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/head", + "deepseek-ai/DeepSeek-V4-Pro/head" + ] + }, + { + "type": "gemm", + "args": { + "m": 1024, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 7168, + "n": 129280, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/head", + "deepseek-ai/DeepSeek-V4-Pro/head" + ] + }, + { + "type": "gemm", + "args": { + "m": 1536, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 7168, + "n": 129280, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/head", + "deepseek-ai/DeepSeek-V4-Pro/head" + ] + }, + { + "type": "gemm", + "args": { + "m": 2048, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 7168, + "n": 129280, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/head", + "deepseek-ai/DeepSeek-V4-Pro/head" + ] + }, + { + "type": "gemm", + "args": { + "m": 3072, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 7168, + "n": 129280, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/head", + "deepseek-ai/DeepSeek-V4-Pro/head" + ] + }, + { + "type": "gemm", + "args": { + "m": 4096, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 7168, + "n": 129280, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/head", + "deepseek-ai/DeepSeek-V4-Pro/head" + ] + }, + { + "type": "gemm", + "args": { + "m": 6144, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 7168, + "n": 129280, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/head", + "deepseek-ai/DeepSeek-V4-Pro/head" + ] + }, + { + "type": "gemm", + "args": { + "m": 8192, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 7168, + "n": 129280, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/head", + "deepseek-ai/DeepSeek-V4-Pro/head" + ] + }, + { + "type": "gemm", + "args": { + "m": 12288, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 7168, + "n": 129280, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/head", + "deepseek-ai/DeepSeek-V4-Pro/head" + ] + }, + { + "type": "gemm", + "args": { + "m": 16384, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 7168, + "n": 129280, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/head", + "deepseek-ai/DeepSeek-V4-Pro/head" + ] + }, + { + "type": "gemm", + "args": { + "m": 24576, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 7168, + "n": 129280, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/head", + "deepseek-ai/DeepSeek-V4-Pro/head" + ] + }, + { + "type": "gemm", + "args": { + "m": 32768, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 7168, + "n": 129280, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/head", + "deepseek-ai/DeepSeek-V4-Pro/head" + ] + }, + { + "type": "gemm", + "args": { + "m": 49152, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 7168, + "n": 129280, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/head", + "deepseek-ai/DeepSeek-V4-Pro/head" + ] + }, + { + "type": "gemm", + "args": { + "m": 65536, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 7168, + "n": 129280, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/head", + "deepseek-ai/DeepSeek-V4-Pro/head" + ] + }, + { + "type": "gemm", + "args": { + "m": 1, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 5120, + "n": 512, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4.1-Flash/attn.compressor.wkv" + ] + }, + { + "type": "gemm", + "args": { + "m": 2, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 5120, + "n": 512, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4.1-Flash/attn.compressor.wkv" + ] + }, + { + "type": "gemm", + "args": { + "m": 3, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 5120, + "n": 512, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4.1-Flash/attn.compressor.wkv" + ] + }, + { + "type": "gemm", + "args": { + "m": 4, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 5120, + "n": 512, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4.1-Flash/attn.compressor.wkv" + ] + }, + { + "type": "gemm", + "args": { + "m": 6, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 5120, + "n": 512, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4.1-Flash/attn.compressor.wkv" + ] + }, + { + "type": "gemm", + "args": { + "m": 8, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 5120, + "n": 512, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4.1-Flash/attn.compressor.wkv" + ] + }, + { + "type": "gemm", + "args": { + "m": 12, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 5120, + "n": 512, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4.1-Flash/attn.compressor.wkv" + ] + }, + { + "type": "gemm", + "args": { + "m": 16, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 5120, + "n": 512, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4.1-Flash/attn.compressor.wkv" + ] + }, + { + "type": "gemm", + "args": { + "m": 24, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 5120, + "n": 512, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4.1-Flash/attn.compressor.wkv" + ] + }, + { + "type": "gemm", + "args": { + "m": 32, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 5120, + "n": 512, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4.1-Flash/attn.compressor.wkv" + ] + }, + { + "type": "gemm", + "args": { + "m": 48, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 5120, + "n": 512, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4.1-Flash/attn.compressor.wkv" + ] + }, + { + "type": "gemm", + "args": { + "m": 64, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 5120, + "n": 512, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4.1-Flash/attn.compressor.wkv" + ] + }, + { + "type": "gemm", + "args": { + "m": 96, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 5120, + "n": 512, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4.1-Flash/attn.compressor.wkv" + ] + }, + { + "type": "gemm", + "args": { + "m": 128, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 5120, + "n": 512, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4.1-Flash/attn.compressor.wkv" + ] + }, + { + "type": "gemm", + "args": { + "m": 192, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 5120, + "n": 512, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4.1-Flash/attn.compressor.wkv" + ] + }, + { + "type": "gemm", + "args": { + "m": 256, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 5120, + "n": 512, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4.1-Flash/attn.compressor.wkv" + ] + }, + { + "type": "gemm", + "args": { + "m": 384, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 5120, + "n": 512, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4.1-Flash/attn.compressor.wkv" + ] + }, + { + "type": "gemm", + "args": { + "m": 512, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 5120, + "n": 512, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4.1-Flash/attn.compressor.wkv" + ] + }, + { + "type": "gemm", + "args": { + "m": 768, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 5120, + "n": 512, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4.1-Flash/attn.compressor.wkv" + ] + }, + { + "type": "gemm", + "args": { + "m": 1024, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 5120, + "n": 512, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4.1-Flash/attn.compressor.wkv" + ] + }, + { + "type": "gemm", + "args": { + "m": 1536, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 5120, + "n": 512, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4.1-Flash/attn.compressor.wkv" + ] + }, + { + "type": "gemm", + "args": { + "m": 2048, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 5120, + "n": 512, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4.1-Flash/attn.compressor.wkv" + ] + }, + { + "type": "gemm", + "args": { + "m": 3072, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 5120, + "n": 512, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4.1-Flash/attn.compressor.wkv" + ] + }, + { + "type": "gemm", + "args": { + "m": 4096, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 5120, + "n": 512, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4.1-Flash/attn.compressor.wkv" + ] + }, + { + "type": "gemm", + "args": { + "m": 6144, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 5120, + "n": 512, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4.1-Flash/attn.compressor.wkv" + ] + }, + { + "type": "gemm", + "args": { + "m": 8192, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 5120, + "n": 512, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4.1-Flash/attn.compressor.wkv" + ] + }, + { + "type": "gemm", + "args": { + "m": 12288, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 5120, + "n": 512, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4.1-Flash/attn.compressor.wkv" + ] + }, + { + "type": "gemm", + "args": { + "m": 16384, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 5120, + "n": 512, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4.1-Flash/attn.compressor.wkv" + ] + }, + { + "type": "gemm", + "args": { + "m": 24576, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 5120, + "n": 512, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4.1-Flash/attn.compressor.wkv" + ] + }, + { + "type": "gemm", + "args": { + "m": 32768, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 5120, + "n": 512, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4.1-Flash/attn.compressor.wkv" + ] + }, + { + "type": "gemm", + "args": { + "m": 49152, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 5120, + "n": 512, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4.1-Flash/attn.compressor.wkv" + ] + }, + { + "type": "gemm", + "args": { + "m": 65536, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 5120, + "n": 512, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4.1-Flash/attn.compressor.wkv" + ] + }, + { + "type": "gemm", + "args": { + "m": 1, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 5120, + "n": 512, + "out": "fp32" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4.1-Flash/attn.compressor.wkv" + ] + }, + { + "type": "gemm", + "args": { + "m": 2, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 5120, + "n": 512, + "out": "fp32" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4.1-Flash/attn.compressor.wkv" + ] + }, + { + "type": "gemm", + "args": { + "m": 3, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 5120, + "n": 512, + "out": "fp32" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4.1-Flash/attn.compressor.wkv" + ] + }, + { + "type": "gemm", + "args": { + "m": 4, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 5120, + "n": 512, + "out": "fp32" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4.1-Flash/attn.compressor.wkv" + ] + }, + { + "type": "gemm", + "args": { + "m": 6, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 5120, + "n": 512, + "out": "fp32" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4.1-Flash/attn.compressor.wkv" + ] + }, + { + "type": "gemm", + "args": { + "m": 8, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 5120, + "n": 512, + "out": "fp32" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4.1-Flash/attn.compressor.wkv" + ] + }, + { + "type": "gemm", + "args": { + "m": 12, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 5120, + "n": 512, + "out": "fp32" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4.1-Flash/attn.compressor.wkv" + ] + }, + { + "type": "gemm", + "args": { + "m": 16, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 5120, + "n": 512, + "out": "fp32" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4.1-Flash/attn.compressor.wkv" + ] + }, + { + "type": "gemm", + "args": { + "m": 24, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 5120, + "n": 512, + "out": "fp32" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4.1-Flash/attn.compressor.wkv" + ] + }, + { + "type": "gemm", + "args": { + "m": 32, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 5120, + "n": 512, + "out": "fp32" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4.1-Flash/attn.compressor.wkv" + ] + }, + { + "type": "gemm", + "args": { + "m": 48, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 5120, + "n": 512, + "out": "fp32" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4.1-Flash/attn.compressor.wkv" + ] + }, + { + "type": "gemm", + "args": { + "m": 64, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 5120, + "n": 512, + "out": "fp32" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4.1-Flash/attn.compressor.wkv" + ] + }, + { + "type": "gemm", + "args": { + "m": 96, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 5120, + "n": 512, + "out": "fp32" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4.1-Flash/attn.compressor.wkv" + ] + }, + { + "type": "gemm", + "args": { + "m": 128, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 5120, + "n": 512, + "out": "fp32" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4.1-Flash/attn.compressor.wkv" + ] + }, + { + "type": "gemm", + "args": { + "m": 192, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 5120, + "n": 512, + "out": "fp32" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4.1-Flash/attn.compressor.wkv" + ] + }, + { + "type": "gemm", + "args": { + "m": 256, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 5120, + "n": 512, + "out": "fp32" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4.1-Flash/attn.compressor.wkv" + ] + }, + { + "type": "gemm", + "args": { + "m": 384, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 5120, + "n": 512, + "out": "fp32" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4.1-Flash/attn.compressor.wkv" + ] + }, + { + "type": "gemm", + "args": { + "m": 512, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 5120, + "n": 512, + "out": "fp32" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4.1-Flash/attn.compressor.wkv" + ] + }, + { + "type": "gemm", + "args": { + "m": 768, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 5120, + "n": 512, + "out": "fp32" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4.1-Flash/attn.compressor.wkv" + ] + }, + { + "type": "gemm", + "args": { + "m": 1024, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 5120, + "n": 512, + "out": "fp32" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4.1-Flash/attn.compressor.wkv" + ] + }, + { + "type": "gemm", + "args": { + "m": 1536, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 5120, + "n": 512, + "out": "fp32" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4.1-Flash/attn.compressor.wkv" + ] + }, + { + "type": "gemm", + "args": { + "m": 2048, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 5120, + "n": 512, + "out": "fp32" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4.1-Flash/attn.compressor.wkv" + ] + }, + { + "type": "gemm", + "args": { + "m": 3072, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 5120, + "n": 512, + "out": "fp32" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4.1-Flash/attn.compressor.wkv" + ] + }, + { + "type": "gemm", + "args": { + "m": 4096, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 5120, + "n": 512, + "out": "fp32" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4.1-Flash/attn.compressor.wkv" + ] + }, + { + "type": "gemm", + "args": { + "m": 6144, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 5120, + "n": 512, + "out": "fp32" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4.1-Flash/attn.compressor.wkv" + ] + }, + { + "type": "gemm", + "args": { + "m": 8192, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 5120, + "n": 512, + "out": "fp32" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4.1-Flash/attn.compressor.wkv" + ] + }, + { + "type": "gemm", + "args": { + "m": 12288, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 5120, + "n": 512, + "out": "fp32" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4.1-Flash/attn.compressor.wkv" + ] + }, + { + "type": "gemm", + "args": { + "m": 16384, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 5120, + "n": 512, + "out": "fp32" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4.1-Flash/attn.compressor.wkv" + ] + }, + { + "type": "gemm", + "args": { + "m": 24576, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 5120, + "n": 512, + "out": "fp32" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4.1-Flash/attn.compressor.wkv" + ] + }, + { + "type": "gemm", + "args": { + "m": 32768, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 5120, + "n": 512, + "out": "fp32" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4.1-Flash/attn.compressor.wkv" + ] + }, + { + "type": "gemm", + "args": { + "m": 49152, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 5120, + "n": 512, + "out": "fp32" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4.1-Flash/attn.compressor.wkv" + ] + }, + { + "type": "gemm", + "args": { + "m": 65536, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 5120, + "n": 512, + "out": "fp32" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4.1-Flash/attn.compressor.wkv" + ] + }, + { + "type": "gemm", + "args": { + "m": 1, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 5120, + "n": 1024, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4.1-Flash/attn.compressor.wkv+attn.compressor.wgate" + ] + }, + { + "type": "gemm", + "args": { + "m": 2, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 5120, + "n": 1024, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4.1-Flash/attn.compressor.wkv+attn.compressor.wgate" + ] + }, + { + "type": "gemm", + "args": { + "m": 3, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 5120, + "n": 1024, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4.1-Flash/attn.compressor.wkv+attn.compressor.wgate" + ] + }, + { + "type": "gemm", + "args": { + "m": 4, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 5120, + "n": 1024, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4.1-Flash/attn.compressor.wkv+attn.compressor.wgate" + ] + }, + { + "type": "gemm", + "args": { + "m": 6, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 5120, + "n": 1024, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4.1-Flash/attn.compressor.wkv+attn.compressor.wgate" + ] + }, + { + "type": "gemm", + "args": { + "m": 8, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 5120, + "n": 1024, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4.1-Flash/attn.compressor.wkv+attn.compressor.wgate" + ] + }, + { + "type": "gemm", + "args": { + "m": 12, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 5120, + "n": 1024, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4.1-Flash/attn.compressor.wkv+attn.compressor.wgate" + ] + }, + { + "type": "gemm", + "args": { + "m": 16, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 5120, + "n": 1024, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4.1-Flash/attn.compressor.wkv+attn.compressor.wgate" + ] + }, + { + "type": "gemm", + "args": { + "m": 24, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 5120, + "n": 1024, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4.1-Flash/attn.compressor.wkv+attn.compressor.wgate" + ] + }, + { + "type": "gemm", + "args": { + "m": 32, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 5120, + "n": 1024, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4.1-Flash/attn.compressor.wkv+attn.compressor.wgate" + ] + }, + { + "type": "gemm", + "args": { + "m": 48, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 5120, + "n": 1024, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4.1-Flash/attn.compressor.wkv+attn.compressor.wgate" + ] + }, + { + "type": "gemm", + "args": { + "m": 64, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 5120, + "n": 1024, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4.1-Flash/attn.compressor.wkv+attn.compressor.wgate" + ] + }, + { + "type": "gemm", + "args": { + "m": 96, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 5120, + "n": 1024, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4.1-Flash/attn.compressor.wkv+attn.compressor.wgate" + ] + }, + { + "type": "gemm", + "args": { + "m": 128, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 5120, + "n": 1024, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4.1-Flash/attn.compressor.wkv+attn.compressor.wgate" + ] + }, + { + "type": "gemm", + "args": { + "m": 192, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 5120, + "n": 1024, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4.1-Flash/attn.compressor.wkv+attn.compressor.wgate" + ] + }, + { + "type": "gemm", + "args": { + "m": 256, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 5120, + "n": 1024, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4.1-Flash/attn.compressor.wkv+attn.compressor.wgate" + ] + }, + { + "type": "gemm", + "args": { + "m": 384, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 5120, + "n": 1024, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4.1-Flash/attn.compressor.wkv+attn.compressor.wgate" + ] + }, + { + "type": "gemm", + "args": { + "m": 512, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 5120, + "n": 1024, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4.1-Flash/attn.compressor.wkv+attn.compressor.wgate" + ] + }, + { + "type": "gemm", + "args": { + "m": 768, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 5120, + "n": 1024, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4.1-Flash/attn.compressor.wkv+attn.compressor.wgate" + ] + }, + { + "type": "gemm", + "args": { + "m": 1024, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 5120, + "n": 1024, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4.1-Flash/attn.compressor.wkv+attn.compressor.wgate" + ] + }, + { + "type": "gemm", + "args": { + "m": 1536, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 5120, + "n": 1024, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4.1-Flash/attn.compressor.wkv+attn.compressor.wgate" + ] + }, + { + "type": "gemm", + "args": { + "m": 2048, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 5120, + "n": 1024, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4.1-Flash/attn.compressor.wkv+attn.compressor.wgate" + ] + }, + { + "type": "gemm", + "args": { + "m": 3072, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 5120, + "n": 1024, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4.1-Flash/attn.compressor.wkv+attn.compressor.wgate" + ] + }, + { + "type": "gemm", + "args": { + "m": 4096, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 5120, + "n": 1024, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4.1-Flash/attn.compressor.wkv+attn.compressor.wgate" + ] + }, + { + "type": "gemm", + "args": { + "m": 6144, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 5120, + "n": 1024, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4.1-Flash/attn.compressor.wkv+attn.compressor.wgate" + ] + }, + { + "type": "gemm", + "args": { + "m": 8192, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 5120, + "n": 1024, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4.1-Flash/attn.compressor.wkv+attn.compressor.wgate" + ] + }, + { + "type": "gemm", + "args": { + "m": 12288, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 5120, + "n": 1024, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4.1-Flash/attn.compressor.wkv+attn.compressor.wgate" + ] + }, + { + "type": "gemm", + "args": { + "m": 16384, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 5120, + "n": 1024, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4.1-Flash/attn.compressor.wkv+attn.compressor.wgate" + ] + }, + { + "type": "gemm", + "args": { + "m": 24576, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 5120, + "n": 1024, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4.1-Flash/attn.compressor.wkv+attn.compressor.wgate" + ] + }, + { + "type": "gemm", + "args": { + "m": 32768, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 5120, + "n": 1024, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4.1-Flash/attn.compressor.wkv+attn.compressor.wgate" + ] + }, + { + "type": "gemm", + "args": { + "m": 49152, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 5120, + "n": 1024, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4.1-Flash/attn.compressor.wkv+attn.compressor.wgate" + ] + }, + { + "type": "gemm", + "args": { + "m": 65536, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 5120, + "n": 1024, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4.1-Flash/attn.compressor.wkv+attn.compressor.wgate" + ] + }, + { + "type": "gemm", + "args": { + "m": 1, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 5120, + "n": 1024, + "out": "fp32" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4.1-Flash/attn.compressor.wkv+attn.compressor.wgate" + ] + }, + { + "type": "gemm", + "args": { + "m": 2, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 5120, + "n": 1024, + "out": "fp32" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4.1-Flash/attn.compressor.wkv+attn.compressor.wgate" + ] + }, + { + "type": "gemm", + "args": { + "m": 3, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 5120, + "n": 1024, + "out": "fp32" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4.1-Flash/attn.compressor.wkv+attn.compressor.wgate" + ] + }, + { + "type": "gemm", + "args": { + "m": 4, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 5120, + "n": 1024, + "out": "fp32" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4.1-Flash/attn.compressor.wkv+attn.compressor.wgate" + ] + }, + { + "type": "gemm", + "args": { + "m": 6, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 5120, + "n": 1024, + "out": "fp32" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4.1-Flash/attn.compressor.wkv+attn.compressor.wgate" + ] + }, + { + "type": "gemm", + "args": { + "m": 8, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 5120, + "n": 1024, + "out": "fp32" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4.1-Flash/attn.compressor.wkv+attn.compressor.wgate" + ] + }, + { + "type": "gemm", + "args": { + "m": 12, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 5120, + "n": 1024, + "out": "fp32" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4.1-Flash/attn.compressor.wkv+attn.compressor.wgate" + ] + }, + { + "type": "gemm", + "args": { + "m": 16, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 5120, + "n": 1024, + "out": "fp32" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4.1-Flash/attn.compressor.wkv+attn.compressor.wgate" + ] + }, + { + "type": "gemm", + "args": { + "m": 24, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 5120, + "n": 1024, + "out": "fp32" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4.1-Flash/attn.compressor.wkv+attn.compressor.wgate" + ] + }, + { + "type": "gemm", + "args": { + "m": 32, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 5120, + "n": 1024, + "out": "fp32" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4.1-Flash/attn.compressor.wkv+attn.compressor.wgate" + ] + }, + { + "type": "gemm", + "args": { + "m": 48, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 5120, + "n": 1024, + "out": "fp32" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4.1-Flash/attn.compressor.wkv+attn.compressor.wgate" + ] + }, + { + "type": "gemm", + "args": { + "m": 64, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 5120, + "n": 1024, + "out": "fp32" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4.1-Flash/attn.compressor.wkv+attn.compressor.wgate" + ] + }, + { + "type": "gemm", + "args": { + "m": 96, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 5120, + "n": 1024, + "out": "fp32" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4.1-Flash/attn.compressor.wkv+attn.compressor.wgate" + ] + }, + { + "type": "gemm", + "args": { + "m": 128, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 5120, + "n": 1024, + "out": "fp32" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4.1-Flash/attn.compressor.wkv+attn.compressor.wgate" + ] + }, + { + "type": "gemm", + "args": { + "m": 192, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 5120, + "n": 1024, + "out": "fp32" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4.1-Flash/attn.compressor.wkv+attn.compressor.wgate" + ] + }, + { + "type": "gemm", + "args": { + "m": 256, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 5120, + "n": 1024, + "out": "fp32" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4.1-Flash/attn.compressor.wkv+attn.compressor.wgate" + ] + }, + { + "type": "gemm", + "args": { + "m": 384, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 5120, + "n": 1024, + "out": "fp32" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4.1-Flash/attn.compressor.wkv+attn.compressor.wgate" + ] + }, + { + "type": "gemm", + "args": { + "m": 512, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 5120, + "n": 1024, + "out": "fp32" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4.1-Flash/attn.compressor.wkv+attn.compressor.wgate" + ] + }, + { + "type": "gemm", + "args": { + "m": 768, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 5120, + "n": 1024, + "out": "fp32" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4.1-Flash/attn.compressor.wkv+attn.compressor.wgate" + ] + }, + { + "type": "gemm", + "args": { + "m": 1024, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 5120, + "n": 1024, + "out": "fp32" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4.1-Flash/attn.compressor.wkv+attn.compressor.wgate" + ] + }, + { + "type": "gemm", + "args": { + "m": 1536, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 5120, + "n": 1024, + "out": "fp32" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4.1-Flash/attn.compressor.wkv+attn.compressor.wgate" + ] + }, + { + "type": "gemm", + "args": { + "m": 2048, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 5120, + "n": 1024, + "out": "fp32" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4.1-Flash/attn.compressor.wkv+attn.compressor.wgate" + ] + }, + { + "type": "gemm", + "args": { + "m": 3072, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 5120, + "n": 1024, + "out": "fp32" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4.1-Flash/attn.compressor.wkv+attn.compressor.wgate" + ] + }, + { + "type": "gemm", + "args": { + "m": 4096, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 5120, + "n": 1024, + "out": "fp32" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4.1-Flash/attn.compressor.wkv+attn.compressor.wgate" + ] + }, + { + "type": "gemm", + "args": { + "m": 6144, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 5120, + "n": 1024, + "out": "fp32" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4.1-Flash/attn.compressor.wkv+attn.compressor.wgate" + ] + }, + { + "type": "gemm", + "args": { + "m": 8192, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 5120, + "n": 1024, + "out": "fp32" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4.1-Flash/attn.compressor.wkv+attn.compressor.wgate" + ] + }, + { + "type": "gemm", + "args": { + "m": 12288, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 5120, + "n": 1024, + "out": "fp32" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4.1-Flash/attn.compressor.wkv+attn.compressor.wgate" + ] + }, + { + "type": "gemm", + "args": { + "m": 16384, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 5120, + "n": 1024, + "out": "fp32" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4.1-Flash/attn.compressor.wkv+attn.compressor.wgate" + ] + }, + { + "type": "gemm", + "args": { + "m": 24576, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 5120, + "n": 1024, + "out": "fp32" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4.1-Flash/attn.compressor.wkv+attn.compressor.wgate" + ] + }, + { + "type": "gemm", + "args": { + "m": 32768, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 5120, + "n": 1024, + "out": "fp32" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4.1-Flash/attn.compressor.wkv+attn.compressor.wgate" + ] + }, + { + "type": "gemm", + "args": { + "m": 49152, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 5120, + "n": 1024, + "out": "fp32" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4.1-Flash/attn.compressor.wkv+attn.compressor.wgate" + ] + }, + { + "type": "gemm", + "args": { + "m": 65536, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 5120, + "n": 1024, + "out": "fp32" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4.1-Flash/attn.compressor.wkv+attn.compressor.wgate" + ] + }, + { + "type": "gemm", + "args": { + "m": 1, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 5120, + "n": 32, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4.1-Flash/attn.indexer.weights_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 2, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 5120, + "n": 32, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4.1-Flash/attn.indexer.weights_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 3, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 5120, + "n": 32, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4.1-Flash/attn.indexer.weights_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 4, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 5120, + "n": 32, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4.1-Flash/attn.indexer.weights_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 6, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 5120, + "n": 32, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4.1-Flash/attn.indexer.weights_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 8, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 5120, + "n": 32, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4.1-Flash/attn.indexer.weights_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 12, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 5120, + "n": 32, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4.1-Flash/attn.indexer.weights_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 16, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 5120, + "n": 32, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4.1-Flash/attn.indexer.weights_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 24, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 5120, + "n": 32, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4.1-Flash/attn.indexer.weights_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 32, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 5120, + "n": 32, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4.1-Flash/attn.indexer.weights_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 48, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 5120, + "n": 32, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4.1-Flash/attn.indexer.weights_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 64, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 5120, + "n": 32, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4.1-Flash/attn.indexer.weights_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 96, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 5120, + "n": 32, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4.1-Flash/attn.indexer.weights_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 128, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 5120, + "n": 32, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4.1-Flash/attn.indexer.weights_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 192, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 5120, + "n": 32, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4.1-Flash/attn.indexer.weights_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 256, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 5120, + "n": 32, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4.1-Flash/attn.indexer.weights_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 384, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 5120, + "n": 32, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4.1-Flash/attn.indexer.weights_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 512, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 5120, + "n": 32, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4.1-Flash/attn.indexer.weights_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 768, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 5120, + "n": 32, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4.1-Flash/attn.indexer.weights_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 1024, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 5120, + "n": 32, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4.1-Flash/attn.indexer.weights_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 1536, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 5120, + "n": 32, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4.1-Flash/attn.indexer.weights_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 2048, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 5120, + "n": 32, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4.1-Flash/attn.indexer.weights_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 3072, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 5120, + "n": 32, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4.1-Flash/attn.indexer.weights_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 4096, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 5120, + "n": 32, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4.1-Flash/attn.indexer.weights_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 6144, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 5120, + "n": 32, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4.1-Flash/attn.indexer.weights_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 8192, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 5120, + "n": 32, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4.1-Flash/attn.indexer.weights_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 12288, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 5120, + "n": 32, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4.1-Flash/attn.indexer.weights_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 16384, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 5120, + "n": 32, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4.1-Flash/attn.indexer.weights_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 24576, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 5120, + "n": 32, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4.1-Flash/attn.indexer.weights_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 32768, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 5120, + "n": 32, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4.1-Flash/attn.indexer.weights_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 49152, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 5120, + "n": 32, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4.1-Flash/attn.indexer.weights_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 65536, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 5120, + "n": 32, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4.1-Flash/attn.indexer.weights_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 1, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 512, + "n": 128, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4.1-Flash/attn.indexer.wk" + ] + }, + { + "type": "gemm", + "args": { + "m": 2, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 512, + "n": 128, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4.1-Flash/attn.indexer.wk" + ] + }, + { + "type": "gemm", + "args": { + "m": 3, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 512, + "n": 128, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4.1-Flash/attn.indexer.wk" + ] + }, + { + "type": "gemm", + "args": { + "m": 4, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 512, + "n": 128, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4.1-Flash/attn.indexer.wk" + ] + }, + { + "type": "gemm", + "args": { + "m": 6, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 512, + "n": 128, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4.1-Flash/attn.indexer.wk" + ] + }, + { + "type": "gemm", + "args": { + "m": 8, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 512, + "n": 128, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4.1-Flash/attn.indexer.wk" + ] + }, + { + "type": "gemm", + "args": { + "m": 12, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 512, + "n": 128, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4.1-Flash/attn.indexer.wk" + ] + }, + { + "type": "gemm", + "args": { + "m": 16, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 512, + "n": 128, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4.1-Flash/attn.indexer.wk" + ] + }, + { + "type": "gemm", + "args": { + "m": 24, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 512, + "n": 128, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4.1-Flash/attn.indexer.wk" + ] + }, + { + "type": "gemm", + "args": { + "m": 32, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 512, + "n": 128, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4.1-Flash/attn.indexer.wk" + ] + }, + { + "type": "gemm", + "args": { + "m": 48, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 512, + "n": 128, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4.1-Flash/attn.indexer.wk" + ] + }, + { + "type": "gemm", + "args": { + "m": 64, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 512, + "n": 128, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4.1-Flash/attn.indexer.wk" + ] + }, + { + "type": "gemm", + "args": { + "m": 96, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 512, + "n": 128, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4.1-Flash/attn.indexer.wk" + ] + }, + { + "type": "gemm", + "args": { + "m": 128, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 512, + "n": 128, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4.1-Flash/attn.indexer.wk" + ] + }, + { + "type": "gemm", + "args": { + "m": 192, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 512, + "n": 128, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4.1-Flash/attn.indexer.wk" + ] + }, + { + "type": "gemm", + "args": { + "m": 256, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 512, + "n": 128, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4.1-Flash/attn.indexer.wk" + ] + }, + { + "type": "gemm", + "args": { + "m": 384, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 512, + "n": 128, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4.1-Flash/attn.indexer.wk" + ] + }, + { + "type": "gemm", + "args": { + "m": 512, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 512, + "n": 128, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4.1-Flash/attn.indexer.wk" + ] + }, + { + "type": "gemm", + "args": { + "m": 768, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 512, + "n": 128, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4.1-Flash/attn.indexer.wk" + ] + }, + { + "type": "gemm", + "args": { + "m": 1024, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 512, + "n": 128, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4.1-Flash/attn.indexer.wk" + ] + }, + { + "type": "gemm", + "args": { + "m": 1536, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 512, + "n": 128, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4.1-Flash/attn.indexer.wk" + ] + }, + { + "type": "gemm", + "args": { + "m": 2048, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 512, + "n": 128, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4.1-Flash/attn.indexer.wk" + ] + }, + { + "type": "gemm", + "args": { + "m": 3072, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 512, + "n": 128, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4.1-Flash/attn.indexer.wk" + ] + }, + { + "type": "gemm", + "args": { + "m": 4096, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 512, + "n": 128, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4.1-Flash/attn.indexer.wk" + ] + }, + { + "type": "gemm", + "args": { + "m": 6144, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 512, + "n": 128, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4.1-Flash/attn.indexer.wk" + ] + }, + { + "type": "gemm", + "args": { + "m": 8192, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 512, + "n": 128, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4.1-Flash/attn.indexer.wk" + ] + }, + { + "type": "gemm", + "args": { + "m": 12288, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 512, + "n": 128, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4.1-Flash/attn.indexer.wk" + ] + }, + { + "type": "gemm", + "args": { + "m": 16384, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 512, + "n": 128, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4.1-Flash/attn.indexer.wk" + ] + }, + { + "type": "gemm", + "args": { + "m": 24576, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 512, + "n": 128, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4.1-Flash/attn.indexer.wk" + ] + }, + { + "type": "gemm", + "args": { + "m": 32768, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 512, + "n": 128, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4.1-Flash/attn.indexer.wk" + ] + }, + { + "type": "gemm", + "args": { + "m": 49152, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 512, + "n": 128, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4.1-Flash/attn.indexer.wk" + ] + }, + { + "type": "gemm", + "args": { + "m": 65536, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 512, + "n": 128, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4.1-Flash/attn.indexer.wk" + ] + }, + { + "type": "gemm", + "args": { + "m": 1, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + }, + "k": 1280, + "n": 4096, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4.1-Flash/attn.indexer.wq_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 2, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + }, + "k": 1280, + "n": 4096, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4.1-Flash/attn.indexer.wq_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 3, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + }, + "k": 1280, + "n": 4096, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4.1-Flash/attn.indexer.wq_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 4, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + }, + "k": 1280, + "n": 4096, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4.1-Flash/attn.indexer.wq_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 6, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + }, + "k": 1280, + "n": 4096, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4.1-Flash/attn.indexer.wq_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 8, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + }, + "k": 1280, + "n": 4096, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4.1-Flash/attn.indexer.wq_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 12, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + }, + "k": 1280, + "n": 4096, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4.1-Flash/attn.indexer.wq_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 16, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + }, + "k": 1280, + "n": 4096, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4.1-Flash/attn.indexer.wq_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 24, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + }, + "k": 1280, + "n": 4096, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4.1-Flash/attn.indexer.wq_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 32, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + }, + "k": 1280, + "n": 4096, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4.1-Flash/attn.indexer.wq_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 48, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + }, + "k": 1280, + "n": 4096, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4.1-Flash/attn.indexer.wq_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 64, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + }, + "k": 1280, + "n": 4096, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4.1-Flash/attn.indexer.wq_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 96, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + }, + "k": 1280, + "n": 4096, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4.1-Flash/attn.indexer.wq_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 128, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + }, + "k": 1280, + "n": 4096, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4.1-Flash/attn.indexer.wq_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 192, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + }, + "k": 1280, + "n": 4096, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4.1-Flash/attn.indexer.wq_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 256, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + }, + "k": 1280, + "n": 4096, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4.1-Flash/attn.indexer.wq_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 384, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + }, + "k": 1280, + "n": 4096, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4.1-Flash/attn.indexer.wq_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 512, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + }, + "k": 1280, + "n": 4096, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4.1-Flash/attn.indexer.wq_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 768, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + }, + "k": 1280, + "n": 4096, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4.1-Flash/attn.indexer.wq_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 1024, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + }, + "k": 1280, + "n": 4096, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4.1-Flash/attn.indexer.wq_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + }, + "k": 1280, + "n": 4096, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4.1-Flash/attn.indexer.wq_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 2048, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + }, + "k": 1280, + "n": 4096, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4.1-Flash/attn.indexer.wq_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 3072, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + }, + "k": 1280, + "n": 4096, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4.1-Flash/attn.indexer.wq_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 4096, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + }, + "k": 1280, + "n": 4096, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4.1-Flash/attn.indexer.wq_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 6144, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + }, + "k": 1280, + "n": 4096, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4.1-Flash/attn.indexer.wq_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 8192, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + }, + "k": 1280, + "n": 4096, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4.1-Flash/attn.indexer.wq_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 12288, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + }, + "k": 1280, + "n": 4096, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4.1-Flash/attn.indexer.wq_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 16384, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + }, + "k": 1280, + "n": 4096, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4.1-Flash/attn.indexer.wq_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 24576, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + }, + "k": 1280, + "n": 4096, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4.1-Flash/attn.indexer.wq_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 32768, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + }, + "k": 1280, + "n": 4096, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4.1-Flash/attn.indexer.wq_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 49152, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + }, + "k": 1280, + "n": 4096, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4.1-Flash/attn.indexer.wq_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 65536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + }, + "k": 1280, + "n": 4096, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4.1-Flash/attn.indexer.wq_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 1, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + }, + "k": 8192, + "n": 5120, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4.1-Flash/attn.wo_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 2, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + }, + "k": 8192, + "n": 5120, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4.1-Flash/attn.wo_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 3, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + }, + "k": 8192, + "n": 5120, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4.1-Flash/attn.wo_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 4, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + }, + "k": 8192, + "n": 5120, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4.1-Flash/attn.wo_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 6, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + }, + "k": 8192, + "n": 5120, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4.1-Flash/attn.wo_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 8, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + }, + "k": 8192, + "n": 5120, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4.1-Flash/attn.wo_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 12, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + }, + "k": 8192, + "n": 5120, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4.1-Flash/attn.wo_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 16, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + }, + "k": 8192, + "n": 5120, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4.1-Flash/attn.wo_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 24, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + }, + "k": 8192, + "n": 5120, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4.1-Flash/attn.wo_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 32, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + }, + "k": 8192, + "n": 5120, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4.1-Flash/attn.wo_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 48, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + }, + "k": 8192, + "n": 5120, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4.1-Flash/attn.wo_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 64, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + }, + "k": 8192, + "n": 5120, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4.1-Flash/attn.wo_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 96, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + }, + "k": 8192, + "n": 5120, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4.1-Flash/attn.wo_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 128, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + }, + "k": 8192, + "n": 5120, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4.1-Flash/attn.wo_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 192, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + }, + "k": 8192, + "n": 5120, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4.1-Flash/attn.wo_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 256, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + }, + "k": 8192, + "n": 5120, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4.1-Flash/attn.wo_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 384, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + }, + "k": 8192, + "n": 5120, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4.1-Flash/attn.wo_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 512, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + }, + "k": 8192, + "n": 5120, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4.1-Flash/attn.wo_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 768, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + }, + "k": 8192, + "n": 5120, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4.1-Flash/attn.wo_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 1024, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + }, + "k": 8192, + "n": 5120, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4.1-Flash/attn.wo_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + }, + "k": 8192, + "n": 5120, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4.1-Flash/attn.wo_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 2048, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + }, + "k": 8192, + "n": 5120, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4.1-Flash/attn.wo_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 3072, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + }, + "k": 8192, + "n": 5120, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4.1-Flash/attn.wo_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 4096, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + }, + "k": 8192, + "n": 5120, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4.1-Flash/attn.wo_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 6144, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + }, + "k": 8192, + "n": 5120, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4.1-Flash/attn.wo_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 8192, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + }, + "k": 8192, + "n": 5120, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4.1-Flash/attn.wo_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 12288, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + }, + "k": 8192, + "n": 5120, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4.1-Flash/attn.wo_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 16384, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + }, + "k": 8192, + "n": 5120, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4.1-Flash/attn.wo_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 24576, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + }, + "k": 8192, + "n": 5120, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4.1-Flash/attn.wo_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 32768, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + }, + "k": 8192, + "n": 5120, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4.1-Flash/attn.wo_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 49152, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + }, + "k": 8192, + "n": 5120, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4.1-Flash/attn.wo_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 65536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + }, + "k": 8192, + "n": 5120, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4.1-Flash/attn.wo_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 1, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + }, + "k": 5120, + "n": 1792, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4.1-Flash/attn.wq_a+attn.wkv" + ] + }, + { + "type": "gemm", + "args": { + "m": 2, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + }, + "k": 5120, + "n": 1792, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4.1-Flash/attn.wq_a+attn.wkv" + ] + }, + { + "type": "gemm", + "args": { + "m": 3, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + }, + "k": 5120, + "n": 1792, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4.1-Flash/attn.wq_a+attn.wkv" + ] + }, + { + "type": "gemm", + "args": { + "m": 4, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + }, + "k": 5120, + "n": 1792, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4.1-Flash/attn.wq_a+attn.wkv" + ] + }, + { + "type": "gemm", + "args": { + "m": 6, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + }, + "k": 5120, + "n": 1792, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4.1-Flash/attn.wq_a+attn.wkv" + ] + }, + { + "type": "gemm", + "args": { + "m": 8, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + }, + "k": 5120, + "n": 1792, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4.1-Flash/attn.wq_a+attn.wkv" + ] + }, + { + "type": "gemm", + "args": { + "m": 12, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + }, + "k": 5120, + "n": 1792, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4.1-Flash/attn.wq_a+attn.wkv" + ] + }, + { + "type": "gemm", + "args": { + "m": 16, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + }, + "k": 5120, + "n": 1792, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4.1-Flash/attn.wq_a+attn.wkv" + ] + }, + { + "type": "gemm", + "args": { + "m": 24, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + }, + "k": 5120, + "n": 1792, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4.1-Flash/attn.wq_a+attn.wkv" + ] + }, + { + "type": "gemm", + "args": { + "m": 32, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + }, + "k": 5120, + "n": 1792, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4.1-Flash/attn.wq_a+attn.wkv" + ] + }, + { + "type": "gemm", + "args": { + "m": 48, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + }, + "k": 5120, + "n": 1792, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4.1-Flash/attn.wq_a+attn.wkv" + ] + }, + { + "type": "gemm", + "args": { + "m": 64, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + }, + "k": 5120, + "n": 1792, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4.1-Flash/attn.wq_a+attn.wkv" + ] + }, + { + "type": "gemm", + "args": { + "m": 96, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + }, + "k": 5120, + "n": 1792, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4.1-Flash/attn.wq_a+attn.wkv" + ] + }, + { + "type": "gemm", + "args": { + "m": 128, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + }, + "k": 5120, + "n": 1792, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4.1-Flash/attn.wq_a+attn.wkv" + ] + }, + { + "type": "gemm", + "args": { + "m": 192, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + }, + "k": 5120, + "n": 1792, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4.1-Flash/attn.wq_a+attn.wkv" + ] + }, + { + "type": "gemm", + "args": { + "m": 256, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + }, + "k": 5120, + "n": 1792, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4.1-Flash/attn.wq_a+attn.wkv" + ] + }, + { + "type": "gemm", + "args": { + "m": 384, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + }, + "k": 5120, + "n": 1792, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4.1-Flash/attn.wq_a+attn.wkv" + ] + }, + { + "type": "gemm", + "args": { + "m": 512, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + }, + "k": 5120, + "n": 1792, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4.1-Flash/attn.wq_a+attn.wkv" + ] + }, + { + "type": "gemm", + "args": { + "m": 768, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + }, + "k": 5120, + "n": 1792, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4.1-Flash/attn.wq_a+attn.wkv" + ] + }, + { + "type": "gemm", + "args": { + "m": 1024, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + }, + "k": 5120, + "n": 1792, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4.1-Flash/attn.wq_a+attn.wkv" + ] + }, + { + "type": "gemm", + "args": { + "m": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + }, + "k": 5120, + "n": 1792, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4.1-Flash/attn.wq_a+attn.wkv" + ] + }, + { + "type": "gemm", + "args": { + "m": 2048, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + }, + "k": 5120, + "n": 1792, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4.1-Flash/attn.wq_a+attn.wkv" + ] + }, + { + "type": "gemm", + "args": { + "m": 3072, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + }, + "k": 5120, + "n": 1792, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4.1-Flash/attn.wq_a+attn.wkv" + ] + }, + { + "type": "gemm", + "args": { + "m": 4096, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + }, + "k": 5120, + "n": 1792, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4.1-Flash/attn.wq_a+attn.wkv" + ] + }, + { + "type": "gemm", + "args": { + "m": 6144, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + }, + "k": 5120, + "n": 1792, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4.1-Flash/attn.wq_a+attn.wkv" + ] + }, + { + "type": "gemm", + "args": { + "m": 8192, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + }, + "k": 5120, + "n": 1792, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4.1-Flash/attn.wq_a+attn.wkv" + ] + }, + { + "type": "gemm", + "args": { + "m": 12288, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + }, + "k": 5120, + "n": 1792, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4.1-Flash/attn.wq_a+attn.wkv" + ] + }, + { + "type": "gemm", + "args": { + "m": 16384, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + }, + "k": 5120, + "n": 1792, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4.1-Flash/attn.wq_a+attn.wkv" + ] + }, + { + "type": "gemm", + "args": { + "m": 24576, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + }, + "k": 5120, + "n": 1792, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4.1-Flash/attn.wq_a+attn.wkv" + ] + }, + { + "type": "gemm", + "args": { + "m": 32768, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + }, + "k": 5120, + "n": 1792, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4.1-Flash/attn.wq_a+attn.wkv" + ] + }, + { + "type": "gemm", + "args": { + "m": 49152, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + }, + "k": 5120, + "n": 1792, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4.1-Flash/attn.wq_a+attn.wkv" + ] + }, + { + "type": "gemm", + "args": { + "m": 65536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + }, + "k": 5120, + "n": 1792, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4.1-Flash/attn.wq_a+attn.wkv" + ] + }, + { + "type": "gemm", + "args": { + "m": 1, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + }, + "k": 1280, + "n": 32768, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4.1-Flash/attn.wq_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 2, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + }, + "k": 1280, + "n": 32768, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4.1-Flash/attn.wq_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 3, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + }, + "k": 1280, + "n": 32768, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4.1-Flash/attn.wq_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 4, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + }, + "k": 1280, + "n": 32768, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4.1-Flash/attn.wq_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 6, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + }, + "k": 1280, + "n": 32768, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4.1-Flash/attn.wq_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 8, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + }, + "k": 1280, + "n": 32768, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4.1-Flash/attn.wq_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 12, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + }, + "k": 1280, + "n": 32768, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4.1-Flash/attn.wq_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 16, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + }, + "k": 1280, + "n": 32768, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4.1-Flash/attn.wq_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 24, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + }, + "k": 1280, + "n": 32768, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4.1-Flash/attn.wq_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 32, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + }, + "k": 1280, + "n": 32768, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4.1-Flash/attn.wq_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 48, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + }, + "k": 1280, + "n": 32768, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4.1-Flash/attn.wq_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 64, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + }, + "k": 1280, + "n": 32768, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4.1-Flash/attn.wq_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 96, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + }, + "k": 1280, + "n": 32768, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4.1-Flash/attn.wq_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 128, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + }, + "k": 1280, + "n": 32768, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4.1-Flash/attn.wq_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 192, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + }, + "k": 1280, + "n": 32768, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4.1-Flash/attn.wq_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 256, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + }, + "k": 1280, + "n": 32768, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4.1-Flash/attn.wq_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 384, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + }, + "k": 1280, + "n": 32768, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4.1-Flash/attn.wq_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 512, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + }, + "k": 1280, + "n": 32768, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4.1-Flash/attn.wq_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 768, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + }, + "k": 1280, + "n": 32768, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4.1-Flash/attn.wq_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 1024, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + }, + "k": 1280, + "n": 32768, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4.1-Flash/attn.wq_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + }, + "k": 1280, + "n": 32768, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4.1-Flash/attn.wq_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 2048, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + }, + "k": 1280, + "n": 32768, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4.1-Flash/attn.wq_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 3072, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + }, + "k": 1280, + "n": 32768, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4.1-Flash/attn.wq_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 4096, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + }, + "k": 1280, + "n": 32768, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4.1-Flash/attn.wq_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 6144, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + }, + "k": 1280, + "n": 32768, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4.1-Flash/attn.wq_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 8192, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + }, + "k": 1280, + "n": 32768, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4.1-Flash/attn.wq_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 12288, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + }, + "k": 1280, + "n": 32768, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4.1-Flash/attn.wq_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 16384, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + }, + "k": 1280, + "n": 32768, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4.1-Flash/attn.wq_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 24576, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + }, + "k": 1280, + "n": 32768, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4.1-Flash/attn.wq_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 32768, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + }, + "k": 1280, + "n": 32768, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4.1-Flash/attn.wq_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 49152, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + }, + "k": 1280, + "n": 32768, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4.1-Flash/attn.wq_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 65536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + }, + "k": 1280, + "n": 32768, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4.1-Flash/attn.wq_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 1, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + }, + "k": 6144, + "n": 25600, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4.1-Flash/engram.wkv" + ] + }, + { + "type": "gemm", + "args": { + "m": 2, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + }, + "k": 6144, + "n": 25600, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4.1-Flash/engram.wkv" + ] + }, + { + "type": "gemm", + "args": { + "m": 3, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + }, + "k": 6144, + "n": 25600, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4.1-Flash/engram.wkv" + ] + }, + { + "type": "gemm", + "args": { + "m": 4, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + }, + "k": 6144, + "n": 25600, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4.1-Flash/engram.wkv" + ] + }, + { + "type": "gemm", + "args": { + "m": 6, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + }, + "k": 6144, + "n": 25600, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4.1-Flash/engram.wkv" + ] + }, + { + "type": "gemm", + "args": { + "m": 8, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + }, + "k": 6144, + "n": 25600, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4.1-Flash/engram.wkv" + ] + }, + { + "type": "gemm", + "args": { + "m": 12, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + }, + "k": 6144, + "n": 25600, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4.1-Flash/engram.wkv" + ] + }, + { + "type": "gemm", + "args": { + "m": 16, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + }, + "k": 6144, + "n": 25600, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4.1-Flash/engram.wkv" + ] + }, + { + "type": "gemm", + "args": { + "m": 24, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + }, + "k": 6144, + "n": 25600, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4.1-Flash/engram.wkv" + ] + }, + { + "type": "gemm", + "args": { + "m": 32, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + }, + "k": 6144, + "n": 25600, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4.1-Flash/engram.wkv" + ] + }, + { + "type": "gemm", + "args": { + "m": 48, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + }, + "k": 6144, + "n": 25600, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4.1-Flash/engram.wkv" + ] + }, + { + "type": "gemm", + "args": { + "m": 64, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + }, + "k": 6144, + "n": 25600, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4.1-Flash/engram.wkv" + ] + }, + { + "type": "gemm", + "args": { + "m": 96, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + }, + "k": 6144, + "n": 25600, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4.1-Flash/engram.wkv" + ] + }, + { + "type": "gemm", + "args": { + "m": 128, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + }, + "k": 6144, + "n": 25600, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4.1-Flash/engram.wkv" + ] + }, + { + "type": "gemm", + "args": { + "m": 192, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + }, + "k": 6144, + "n": 25600, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4.1-Flash/engram.wkv" + ] + }, + { + "type": "gemm", + "args": { + "m": 256, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + }, + "k": 6144, + "n": 25600, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4.1-Flash/engram.wkv" + ] + }, + { + "type": "gemm", + "args": { + "m": 384, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + }, + "k": 6144, + "n": 25600, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4.1-Flash/engram.wkv" + ] + }, + { + "type": "gemm", + "args": { + "m": 512, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + }, + "k": 6144, + "n": 25600, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4.1-Flash/engram.wkv" + ] + }, + { + "type": "gemm", + "args": { + "m": 768, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + }, + "k": 6144, + "n": 25600, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4.1-Flash/engram.wkv" + ] + }, + { + "type": "gemm", + "args": { + "m": 1024, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + }, + "k": 6144, + "n": 25600, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4.1-Flash/engram.wkv" + ] + }, + { + "type": "gemm", + "args": { + "m": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + }, + "k": 6144, + "n": 25600, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4.1-Flash/engram.wkv" + ] + }, + { + "type": "gemm", + "args": { + "m": 2048, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + }, + "k": 6144, + "n": 25600, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4.1-Flash/engram.wkv" + ] + }, + { + "type": "gemm", + "args": { + "m": 3072, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + }, + "k": 6144, + "n": 25600, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4.1-Flash/engram.wkv" + ] + }, + { + "type": "gemm", + "args": { + "m": 4096, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + }, + "k": 6144, + "n": 25600, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4.1-Flash/engram.wkv" + ] + }, + { + "type": "gemm", + "args": { + "m": 6144, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + }, + "k": 6144, + "n": 25600, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4.1-Flash/engram.wkv" + ] + }, + { + "type": "gemm", + "args": { + "m": 8192, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + }, + "k": 6144, + "n": 25600, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4.1-Flash/engram.wkv" + ] + }, + { + "type": "gemm", + "args": { + "m": 12288, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + }, + "k": 6144, + "n": 25600, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4.1-Flash/engram.wkv" + ] + }, + { + "type": "gemm", + "args": { + "m": 16384, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + }, + "k": 6144, + "n": 25600, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4.1-Flash/engram.wkv" + ] + }, + { + "type": "gemm", + "args": { + "m": 24576, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + }, + "k": 6144, + "n": 25600, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4.1-Flash/engram.wkv" + ] + }, + { + "type": "gemm", + "args": { + "m": 32768, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + }, + "k": 6144, + "n": 25600, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4.1-Flash/engram.wkv" + ] + }, + { + "type": "gemm", + "args": { + "m": 49152, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + }, + "k": 6144, + "n": 25600, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4.1-Flash/engram.wkv" + ] + }, + { + "type": "gemm", + "args": { + "m": 65536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + }, + "k": 6144, + "n": 25600, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4.1-Flash/engram.wkv" + ] + }, + { + "type": "gemm", + "args": { + "m": 1, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 5120, + "n": 384, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4.1-Flash/ffn.gate" + ] + }, + { + "type": "gemm", + "args": { + "m": 2, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 5120, + "n": 384, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4.1-Flash/ffn.gate" + ] + }, + { + "type": "gemm", + "args": { + "m": 3, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 5120, + "n": 384, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4.1-Flash/ffn.gate" + ] + }, + { + "type": "gemm", + "args": { + "m": 4, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 5120, + "n": 384, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4.1-Flash/ffn.gate" + ] + }, + { + "type": "gemm", + "args": { + "m": 6, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 5120, + "n": 384, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4.1-Flash/ffn.gate" + ] + }, + { + "type": "gemm", + "args": { + "m": 8, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 5120, + "n": 384, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4.1-Flash/ffn.gate" + ] + }, + { + "type": "gemm", + "args": { + "m": 12, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 5120, + "n": 384, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4.1-Flash/ffn.gate" + ] + }, + { + "type": "gemm", + "args": { + "m": 16, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 5120, + "n": 384, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4.1-Flash/ffn.gate" + ] + }, + { + "type": "gemm", + "args": { + "m": 24, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 5120, + "n": 384, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4.1-Flash/ffn.gate" + ] + }, + { + "type": "gemm", + "args": { + "m": 32, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 5120, + "n": 384, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4.1-Flash/ffn.gate" + ] + }, + { + "type": "gemm", + "args": { + "m": 48, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 5120, + "n": 384, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4.1-Flash/ffn.gate" + ] + }, + { + "type": "gemm", + "args": { + "m": 64, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 5120, + "n": 384, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4.1-Flash/ffn.gate" + ] + }, + { + "type": "gemm", + "args": { + "m": 96, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 5120, + "n": 384, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4.1-Flash/ffn.gate" + ] + }, + { + "type": "gemm", + "args": { + "m": 128, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 5120, + "n": 384, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4.1-Flash/ffn.gate" + ] + }, + { + "type": "gemm", + "args": { + "m": 192, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 5120, + "n": 384, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4.1-Flash/ffn.gate" + ] + }, + { + "type": "gemm", + "args": { + "m": 256, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 5120, + "n": 384, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4.1-Flash/ffn.gate" + ] + }, + { + "type": "gemm", + "args": { + "m": 384, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 5120, + "n": 384, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4.1-Flash/ffn.gate" + ] + }, + { + "type": "gemm", + "args": { + "m": 512, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 5120, + "n": 384, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4.1-Flash/ffn.gate" + ] + }, + { + "type": "gemm", + "args": { + "m": 768, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 5120, + "n": 384, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4.1-Flash/ffn.gate" + ] + }, + { + "type": "gemm", + "args": { + "m": 1024, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 5120, + "n": 384, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4.1-Flash/ffn.gate" + ] + }, + { + "type": "gemm", + "args": { + "m": 1536, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 5120, + "n": 384, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4.1-Flash/ffn.gate" + ] + }, + { + "type": "gemm", + "args": { + "m": 2048, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 5120, + "n": 384, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4.1-Flash/ffn.gate" + ] + }, + { + "type": "gemm", + "args": { + "m": 3072, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 5120, + "n": 384, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4.1-Flash/ffn.gate" + ] + }, + { + "type": "gemm", + "args": { + "m": 4096, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 5120, + "n": 384, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4.1-Flash/ffn.gate" + ] + }, + { + "type": "gemm", + "args": { + "m": 6144, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 5120, + "n": 384, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4.1-Flash/ffn.gate" + ] + }, + { + "type": "gemm", + "args": { + "m": 8192, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 5120, + "n": 384, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4.1-Flash/ffn.gate" + ] + }, + { + "type": "gemm", + "args": { + "m": 12288, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 5120, + "n": 384, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4.1-Flash/ffn.gate" + ] + }, + { + "type": "gemm", + "args": { + "m": 16384, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 5120, + "n": 384, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4.1-Flash/ffn.gate" + ] + }, + { + "type": "gemm", + "args": { + "m": 24576, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 5120, + "n": 384, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4.1-Flash/ffn.gate" + ] + }, + { + "type": "gemm", + "args": { + "m": 32768, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 5120, + "n": 384, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4.1-Flash/ffn.gate" + ] + }, + { + "type": "gemm", + "args": { + "m": 49152, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 5120, + "n": 384, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4.1-Flash/ffn.gate" + ] + }, + { + "type": "gemm", + "args": { + "m": 65536, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 5120, + "n": 384, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4.1-Flash/ffn.gate" + ] + }, + { + "type": "gemm", + "args": { + "m": 1, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 5120, + "n": 384, + "out": "fp32" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4.1-Flash/ffn.gate" + ] + }, + { + "type": "gemm", + "args": { + "m": 2, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 5120, + "n": 384, + "out": "fp32" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4.1-Flash/ffn.gate" + ] + }, + { + "type": "gemm", + "args": { + "m": 3, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 5120, + "n": 384, + "out": "fp32" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4.1-Flash/ffn.gate" + ] + }, + { + "type": "gemm", + "args": { + "m": 4, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 5120, + "n": 384, + "out": "fp32" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4.1-Flash/ffn.gate" + ] + }, + { + "type": "gemm", + "args": { + "m": 6, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 5120, + "n": 384, + "out": "fp32" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4.1-Flash/ffn.gate" + ] + }, + { + "type": "gemm", + "args": { + "m": 8, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 5120, + "n": 384, + "out": "fp32" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4.1-Flash/ffn.gate" + ] + }, + { + "type": "gemm", + "args": { + "m": 12, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 5120, + "n": 384, + "out": "fp32" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4.1-Flash/ffn.gate" + ] + }, + { + "type": "gemm", + "args": { + "m": 16, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 5120, + "n": 384, + "out": "fp32" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4.1-Flash/ffn.gate" + ] + }, + { + "type": "gemm", + "args": { + "m": 24, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 5120, + "n": 384, + "out": "fp32" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4.1-Flash/ffn.gate" + ] + }, + { + "type": "gemm", + "args": { + "m": 32, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 5120, + "n": 384, + "out": "fp32" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4.1-Flash/ffn.gate" + ] + }, + { + "type": "gemm", + "args": { + "m": 48, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 5120, + "n": 384, + "out": "fp32" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4.1-Flash/ffn.gate" + ] + }, + { + "type": "gemm", + "args": { + "m": 64, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 5120, + "n": 384, + "out": "fp32" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4.1-Flash/ffn.gate" + ] + }, + { + "type": "gemm", + "args": { + "m": 96, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 5120, + "n": 384, + "out": "fp32" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4.1-Flash/ffn.gate" + ] + }, + { + "type": "gemm", + "args": { + "m": 128, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 5120, + "n": 384, + "out": "fp32" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4.1-Flash/ffn.gate" + ] + }, + { + "type": "gemm", + "args": { + "m": 192, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 5120, + "n": 384, + "out": "fp32" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4.1-Flash/ffn.gate" + ] + }, + { + "type": "gemm", + "args": { + "m": 256, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 5120, + "n": 384, + "out": "fp32" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4.1-Flash/ffn.gate" + ] + }, + { + "type": "gemm", + "args": { + "m": 384, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 5120, + "n": 384, + "out": "fp32" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4.1-Flash/ffn.gate" + ] + }, + { + "type": "gemm", + "args": { + "m": 512, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 5120, + "n": 384, + "out": "fp32" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4.1-Flash/ffn.gate" + ] + }, + { + "type": "gemm", + "args": { + "m": 768, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 5120, + "n": 384, + "out": "fp32" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4.1-Flash/ffn.gate" + ] + }, + { + "type": "gemm", + "args": { + "m": 1024, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 5120, + "n": 384, + "out": "fp32" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4.1-Flash/ffn.gate" + ] + }, + { + "type": "gemm", + "args": { + "m": 1536, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 5120, + "n": 384, + "out": "fp32" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4.1-Flash/ffn.gate" + ] + }, + { + "type": "gemm", + "args": { + "m": 2048, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 5120, + "n": 384, + "out": "fp32" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4.1-Flash/ffn.gate" + ] + }, + { + "type": "gemm", + "args": { + "m": 3072, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 5120, + "n": 384, + "out": "fp32" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4.1-Flash/ffn.gate" + ] + }, + { + "type": "gemm", + "args": { + "m": 4096, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 5120, + "n": 384, + "out": "fp32" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4.1-Flash/ffn.gate" + ] + }, + { + "type": "gemm", + "args": { + "m": 6144, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 5120, + "n": 384, + "out": "fp32" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4.1-Flash/ffn.gate" + ] + }, + { + "type": "gemm", + "args": { + "m": 8192, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 5120, + "n": 384, + "out": "fp32" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4.1-Flash/ffn.gate" + ] + }, + { + "type": "gemm", + "args": { + "m": 12288, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 5120, + "n": 384, + "out": "fp32" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4.1-Flash/ffn.gate" + ] + }, + { + "type": "gemm", + "args": { + "m": 16384, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 5120, + "n": 384, + "out": "fp32" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4.1-Flash/ffn.gate" + ] + }, + { + "type": "gemm", + "args": { + "m": 24576, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 5120, + "n": 384, + "out": "fp32" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4.1-Flash/ffn.gate" + ] + }, + { + "type": "gemm", + "args": { + "m": 32768, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 5120, + "n": 384, + "out": "fp32" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4.1-Flash/ffn.gate" + ] + }, + { + "type": "gemm", + "args": { + "m": 49152, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 5120, + "n": 384, + "out": "fp32" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4.1-Flash/ffn.gate" + ] + }, + { + "type": "gemm", + "args": { + "m": 65536, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 5120, + "n": 384, + "out": "fp32" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4.1-Flash/ffn.gate" + ] + }, + { + "type": "gemm", + "args": { + "m": 1, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + }, + "k": 5120, + "n": 4608, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4.1-Flash/ffn.shared_experts.w1+ffn.shared_experts.w3" + ] + }, + { + "type": "gemm", + "args": { + "m": 2, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + }, + "k": 5120, + "n": 4608, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4.1-Flash/ffn.shared_experts.w1+ffn.shared_experts.w3" + ] + }, + { + "type": "gemm", + "args": { + "m": 3, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + }, + "k": 5120, + "n": 4608, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4.1-Flash/ffn.shared_experts.w1+ffn.shared_experts.w3" + ] + }, + { + "type": "gemm", + "args": { + "m": 4, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + }, + "k": 5120, + "n": 4608, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4.1-Flash/ffn.shared_experts.w1+ffn.shared_experts.w3" + ] + }, + { + "type": "gemm", + "args": { + "m": 6, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + }, + "k": 5120, + "n": 4608, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4.1-Flash/ffn.shared_experts.w1+ffn.shared_experts.w3" + ] + }, + { + "type": "gemm", + "args": { + "m": 8, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + }, + "k": 5120, + "n": 4608, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4.1-Flash/ffn.shared_experts.w1+ffn.shared_experts.w3" + ] + }, + { + "type": "gemm", + "args": { + "m": 12, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + }, + "k": 5120, + "n": 4608, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4.1-Flash/ffn.shared_experts.w1+ffn.shared_experts.w3" + ] + }, + { + "type": "gemm", + "args": { + "m": 16, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + }, + "k": 5120, + "n": 4608, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4.1-Flash/ffn.shared_experts.w1+ffn.shared_experts.w3" + ] + }, + { + "type": "gemm", + "args": { + "m": 24, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + }, + "k": 5120, + "n": 4608, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4.1-Flash/ffn.shared_experts.w1+ffn.shared_experts.w3" + ] + }, + { + "type": "gemm", + "args": { + "m": 32, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + }, + "k": 5120, + "n": 4608, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4.1-Flash/ffn.shared_experts.w1+ffn.shared_experts.w3" + ] + }, + { + "type": "gemm", + "args": { + "m": 48, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + }, + "k": 5120, + "n": 4608, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4.1-Flash/ffn.shared_experts.w1+ffn.shared_experts.w3" + ] + }, + { + "type": "gemm", + "args": { + "m": 64, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + }, + "k": 5120, + "n": 4608, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4.1-Flash/ffn.shared_experts.w1+ffn.shared_experts.w3" + ] + }, + { + "type": "gemm", + "args": { + "m": 96, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + }, + "k": 5120, + "n": 4608, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4.1-Flash/ffn.shared_experts.w1+ffn.shared_experts.w3" + ] + }, + { + "type": "gemm", + "args": { + "m": 128, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + }, + "k": 5120, + "n": 4608, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4.1-Flash/ffn.shared_experts.w1+ffn.shared_experts.w3" + ] + }, + { + "type": "gemm", + "args": { + "m": 192, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + }, + "k": 5120, + "n": 4608, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4.1-Flash/ffn.shared_experts.w1+ffn.shared_experts.w3" + ] + }, + { + "type": "gemm", + "args": { + "m": 256, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + }, + "k": 5120, + "n": 4608, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4.1-Flash/ffn.shared_experts.w1+ffn.shared_experts.w3" + ] + }, + { + "type": "gemm", + "args": { + "m": 384, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + }, + "k": 5120, + "n": 4608, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4.1-Flash/ffn.shared_experts.w1+ffn.shared_experts.w3" + ] + }, + { + "type": "gemm", + "args": { + "m": 512, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + }, + "k": 5120, + "n": 4608, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4.1-Flash/ffn.shared_experts.w1+ffn.shared_experts.w3" + ] + }, + { + "type": "gemm", + "args": { + "m": 768, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + }, + "k": 5120, + "n": 4608, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4.1-Flash/ffn.shared_experts.w1+ffn.shared_experts.w3" + ] + }, + { + "type": "gemm", + "args": { + "m": 1024, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + }, + "k": 5120, + "n": 4608, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4.1-Flash/ffn.shared_experts.w1+ffn.shared_experts.w3" + ] + }, + { + "type": "gemm", + "args": { + "m": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + }, + "k": 5120, + "n": 4608, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4.1-Flash/ffn.shared_experts.w1+ffn.shared_experts.w3" + ] + }, + { + "type": "gemm", + "args": { + "m": 2048, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + }, + "k": 5120, + "n": 4608, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4.1-Flash/ffn.shared_experts.w1+ffn.shared_experts.w3" + ] + }, + { + "type": "gemm", + "args": { + "m": 3072, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + }, + "k": 5120, + "n": 4608, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4.1-Flash/ffn.shared_experts.w1+ffn.shared_experts.w3" + ] + }, + { + "type": "gemm", + "args": { + "m": 4096, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + }, + "k": 5120, + "n": 4608, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4.1-Flash/ffn.shared_experts.w1+ffn.shared_experts.w3" + ] + }, + { + "type": "gemm", + "args": { + "m": 6144, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + }, + "k": 5120, + "n": 4608, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4.1-Flash/ffn.shared_experts.w1+ffn.shared_experts.w3" + ] + }, + { + "type": "gemm", + "args": { + "m": 8192, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + }, + "k": 5120, + "n": 4608, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4.1-Flash/ffn.shared_experts.w1+ffn.shared_experts.w3" + ] + }, + { + "type": "gemm", + "args": { + "m": 12288, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + }, + "k": 5120, + "n": 4608, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4.1-Flash/ffn.shared_experts.w1+ffn.shared_experts.w3" + ] + }, + { + "type": "gemm", + "args": { + "m": 16384, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + }, + "k": 5120, + "n": 4608, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4.1-Flash/ffn.shared_experts.w1+ffn.shared_experts.w3" + ] + }, + { + "type": "gemm", + "args": { + "m": 24576, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + }, + "k": 5120, + "n": 4608, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4.1-Flash/ffn.shared_experts.w1+ffn.shared_experts.w3" + ] + }, + { + "type": "gemm", + "args": { + "m": 32768, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + }, + "k": 5120, + "n": 4608, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4.1-Flash/ffn.shared_experts.w1+ffn.shared_experts.w3" + ] + }, + { + "type": "gemm", + "args": { + "m": 49152, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + }, + "k": 5120, + "n": 4608, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4.1-Flash/ffn.shared_experts.w1+ffn.shared_experts.w3" + ] + }, + { + "type": "gemm", + "args": { + "m": 65536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + }, + "k": 5120, + "n": 4608, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4.1-Flash/ffn.shared_experts.w1+ffn.shared_experts.w3" + ] + }, + { + "type": "gemm", + "args": { + "m": 1, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + }, + "k": 2304, + "n": 5120, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4.1-Flash/ffn.shared_experts.w2" + ] + }, + { + "type": "gemm", + "args": { + "m": 2, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + }, + "k": 2304, + "n": 5120, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4.1-Flash/ffn.shared_experts.w2" + ] + }, + { + "type": "gemm", + "args": { + "m": 3, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + }, + "k": 2304, + "n": 5120, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4.1-Flash/ffn.shared_experts.w2" + ] + }, + { + "type": "gemm", + "args": { + "m": 4, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + }, + "k": 2304, + "n": 5120, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4.1-Flash/ffn.shared_experts.w2" + ] + }, + { + "type": "gemm", + "args": { + "m": 6, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + }, + "k": 2304, + "n": 5120, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4.1-Flash/ffn.shared_experts.w2" + ] + }, + { + "type": "gemm", + "args": { + "m": 8, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + }, + "k": 2304, + "n": 5120, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4.1-Flash/ffn.shared_experts.w2" + ] + }, + { + "type": "gemm", + "args": { + "m": 12, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + }, + "k": 2304, + "n": 5120, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4.1-Flash/ffn.shared_experts.w2" + ] + }, + { + "type": "gemm", + "args": { + "m": 16, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + }, + "k": 2304, + "n": 5120, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4.1-Flash/ffn.shared_experts.w2" + ] + }, + { + "type": "gemm", + "args": { + "m": 24, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + }, + "k": 2304, + "n": 5120, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4.1-Flash/ffn.shared_experts.w2" + ] + }, + { + "type": "gemm", + "args": { + "m": 32, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + }, + "k": 2304, + "n": 5120, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4.1-Flash/ffn.shared_experts.w2" + ] + }, + { + "type": "gemm", + "args": { + "m": 48, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + }, + "k": 2304, + "n": 5120, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4.1-Flash/ffn.shared_experts.w2" + ] + }, + { + "type": "gemm", + "args": { + "m": 64, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + }, + "k": 2304, + "n": 5120, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4.1-Flash/ffn.shared_experts.w2" + ] + }, + { + "type": "gemm", + "args": { + "m": 96, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + }, + "k": 2304, + "n": 5120, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4.1-Flash/ffn.shared_experts.w2" + ] + }, + { + "type": "gemm", + "args": { + "m": 128, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + }, + "k": 2304, + "n": 5120, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4.1-Flash/ffn.shared_experts.w2" + ] + }, + { + "type": "gemm", + "args": { + "m": 192, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + }, + "k": 2304, + "n": 5120, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4.1-Flash/ffn.shared_experts.w2" + ] + }, + { + "type": "gemm", + "args": { + "m": 256, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + }, + "k": 2304, + "n": 5120, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4.1-Flash/ffn.shared_experts.w2" + ] + }, + { + "type": "gemm", + "args": { + "m": 384, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + }, + "k": 2304, + "n": 5120, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4.1-Flash/ffn.shared_experts.w2" + ] + }, + { + "type": "gemm", + "args": { + "m": 512, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + }, + "k": 2304, + "n": 5120, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4.1-Flash/ffn.shared_experts.w2" + ] + }, + { + "type": "gemm", + "args": { + "m": 768, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + }, + "k": 2304, + "n": 5120, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4.1-Flash/ffn.shared_experts.w2" + ] + }, + { + "type": "gemm", + "args": { + "m": 1024, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + }, + "k": 2304, + "n": 5120, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4.1-Flash/ffn.shared_experts.w2" + ] + }, + { + "type": "gemm", + "args": { + "m": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + }, + "k": 2304, + "n": 5120, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4.1-Flash/ffn.shared_experts.w2" + ] + }, + { + "type": "gemm", + "args": { + "m": 2048, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + }, + "k": 2304, + "n": 5120, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4.1-Flash/ffn.shared_experts.w2" + ] + }, + { + "type": "gemm", + "args": { + "m": 3072, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + }, + "k": 2304, + "n": 5120, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4.1-Flash/ffn.shared_experts.w2" + ] + }, + { + "type": "gemm", + "args": { + "m": 4096, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + }, + "k": 2304, + "n": 5120, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4.1-Flash/ffn.shared_experts.w2" + ] + }, + { + "type": "gemm", + "args": { + "m": 6144, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + }, + "k": 2304, + "n": 5120, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4.1-Flash/ffn.shared_experts.w2" + ] + }, + { + "type": "gemm", + "args": { + "m": 8192, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + }, + "k": 2304, + "n": 5120, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4.1-Flash/ffn.shared_experts.w2" + ] + }, + { + "type": "gemm", + "args": { + "m": 12288, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + }, + "k": 2304, + "n": 5120, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4.1-Flash/ffn.shared_experts.w2" + ] + }, + { + "type": "gemm", + "args": { + "m": 16384, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + }, + "k": 2304, + "n": 5120, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4.1-Flash/ffn.shared_experts.w2" + ] + }, + { + "type": "gemm", + "args": { + "m": 24576, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + }, + "k": 2304, + "n": 5120, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4.1-Flash/ffn.shared_experts.w2" + ] + }, + { + "type": "gemm", + "args": { + "m": 32768, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + }, + "k": 2304, + "n": 5120, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4.1-Flash/ffn.shared_experts.w2" + ] + }, + { + "type": "gemm", + "args": { + "m": 49152, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + }, + "k": 2304, + "n": 5120, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4.1-Flash/ffn.shared_experts.w2" + ] + }, + { + "type": "gemm", + "args": { + "m": 65536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + }, + "k": 2304, + "n": 5120, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4.1-Flash/ffn.shared_experts.w2" + ] + }, + { + "type": "gemm", + "args": { + "m": 1, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 5120, + "n": 129280, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4.1-Flash/head" + ] + }, + { + "type": "gemm", + "args": { + "m": 2, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 5120, + "n": 129280, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4.1-Flash/head" + ] + }, + { + "type": "gemm", + "args": { + "m": 3, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 5120, + "n": 129280, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4.1-Flash/head" + ] + }, + { + "type": "gemm", + "args": { + "m": 4, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 5120, + "n": 129280, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4.1-Flash/head" + ] + }, + { + "type": "gemm", + "args": { + "m": 6, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 5120, + "n": 129280, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4.1-Flash/head" + ] + }, + { + "type": "gemm", + "args": { + "m": 8, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 5120, + "n": 129280, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4.1-Flash/head" + ] + }, + { + "type": "gemm", + "args": { + "m": 12, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 5120, + "n": 129280, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4.1-Flash/head" + ] + }, + { + "type": "gemm", + "args": { + "m": 16, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 5120, + "n": 129280, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4.1-Flash/head" + ] + }, + { + "type": "gemm", + "args": { + "m": 24, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 5120, + "n": 129280, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4.1-Flash/head" + ] + }, + { + "type": "gemm", + "args": { + "m": 32, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 5120, + "n": 129280, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4.1-Flash/head" + ] + }, + { + "type": "gemm", + "args": { + "m": 48, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 5120, + "n": 129280, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4.1-Flash/head" + ] + }, + { + "type": "gemm", + "args": { + "m": 64, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 5120, + "n": 129280, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4.1-Flash/head" + ] + }, + { + "type": "gemm", + "args": { + "m": 96, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 5120, + "n": 129280, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4.1-Flash/head" + ] + }, + { + "type": "gemm", + "args": { + "m": 128, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 5120, + "n": 129280, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4.1-Flash/head" + ] + }, + { + "type": "gemm", + "args": { + "m": 192, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 5120, + "n": 129280, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4.1-Flash/head" + ] + }, + { + "type": "gemm", + "args": { + "m": 256, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 5120, + "n": 129280, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4.1-Flash/head" + ] + }, + { + "type": "gemm", + "args": { + "m": 384, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 5120, + "n": 129280, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4.1-Flash/head" + ] + }, + { + "type": "gemm", + "args": { + "m": 512, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 5120, + "n": 129280, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4.1-Flash/head" + ] + }, + { + "type": "gemm", + "args": { + "m": 768, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 5120, + "n": 129280, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4.1-Flash/head" + ] + }, + { + "type": "gemm", + "args": { + "m": 1024, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 5120, + "n": 129280, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4.1-Flash/head" + ] + }, + { + "type": "gemm", + "args": { + "m": 1536, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 5120, + "n": 129280, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4.1-Flash/head" + ] + }, + { + "type": "gemm", + "args": { + "m": 2048, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 5120, + "n": 129280, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4.1-Flash/head" + ] + }, + { + "type": "gemm", + "args": { + "m": 3072, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 5120, + "n": 129280, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4.1-Flash/head" + ] + }, + { + "type": "gemm", + "args": { + "m": 4096, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 5120, + "n": 129280, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4.1-Flash/head" + ] + }, + { + "type": "gemm", + "args": { + "m": 6144, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 5120, + "n": 129280, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4.1-Flash/head" + ] + }, + { + "type": "gemm", + "args": { + "m": 8192, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 5120, + "n": 129280, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4.1-Flash/head" + ] + }, + { + "type": "gemm", + "args": { + "m": 12288, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 5120, + "n": 129280, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4.1-Flash/head" + ] + }, + { + "type": "gemm", + "args": { + "m": 16384, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 5120, + "n": 129280, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4.1-Flash/head" + ] + }, + { + "type": "gemm", + "args": { + "m": 24576, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 5120, + "n": 129280, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4.1-Flash/head" + ] + }, + { + "type": "gemm", + "args": { + "m": 32768, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 5120, + "n": 129280, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4.1-Flash/head" + ] + }, + { + "type": "gemm", + "args": { + "m": 49152, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 5120, + "n": 129280, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4.1-Flash/head" + ] + }, + { + "type": "gemm", + "args": { + "m": 65536, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 5120, + "n": 129280, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4.1-Flash/head" + ] + } +] diff --git a/operatorx/testlists/gemm_kimi_k3.json b/operatorx/testlists/gemm_kimi_k3.json new file mode 100644 index 0000000000..9e64ed1131 --- /dev/null +++ b/operatorx/testlists/gemm_kimi_k3.json @@ -0,0 +1,13282 @@ +[ + { + "type": "gemm", + "args": { + "m": 1, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 7168, + "n": 896, + "out": "bf16" + }, + "sources": [ + "moonshotai/Kimi-K3/block_sparse_moe.gate" + ] + }, + { + "type": "gemm", + "args": { + "m": 2, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 7168, + "n": 896, + "out": "bf16" + }, + "sources": [ + "moonshotai/Kimi-K3/block_sparse_moe.gate" + ] + }, + { + "type": "gemm", + "args": { + "m": 3, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 7168, + "n": 896, + "out": "bf16" + }, + "sources": [ + "moonshotai/Kimi-K3/block_sparse_moe.gate" + ] + }, + { + "type": "gemm", + "args": { + "m": 4, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 7168, + "n": 896, + "out": "bf16" + }, + "sources": [ + "moonshotai/Kimi-K3/block_sparse_moe.gate" + ] + }, + { + "type": "gemm", + "args": { + "m": 6, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 7168, + "n": 896, + "out": "bf16" + }, + "sources": [ + "moonshotai/Kimi-K3/block_sparse_moe.gate" + ] + }, + { + "type": "gemm", + "args": { + "m": 8, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 7168, + "n": 896, + "out": "bf16" + }, + "sources": [ + "moonshotai/Kimi-K3/block_sparse_moe.gate" + ] + }, + { + "type": "gemm", + "args": { + "m": 12, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 7168, + "n": 896, + "out": "bf16" + }, + "sources": [ + "moonshotai/Kimi-K3/block_sparse_moe.gate" + ] + }, + { + "type": "gemm", + "args": { + "m": 16, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 7168, + "n": 896, + "out": "bf16" + }, + "sources": [ + "moonshotai/Kimi-K3/block_sparse_moe.gate" + ] + }, + { + "type": "gemm", + "args": { + "m": 24, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 7168, + "n": 896, + "out": "bf16" + }, + "sources": [ + "moonshotai/Kimi-K3/block_sparse_moe.gate" + ] + }, + { + "type": "gemm", + "args": { + "m": 32, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 7168, + "n": 896, + "out": "bf16" + }, + "sources": [ + "moonshotai/Kimi-K3/block_sparse_moe.gate" + ] + }, + { + "type": "gemm", + "args": { + "m": 48, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 7168, + "n": 896, + "out": "bf16" + }, + "sources": [ + "moonshotai/Kimi-K3/block_sparse_moe.gate" + ] + }, + { + "type": "gemm", + "args": { + "m": 64, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 7168, + "n": 896, + "out": "bf16" + }, + "sources": [ + "moonshotai/Kimi-K3/block_sparse_moe.gate" + ] + }, + { + "type": "gemm", + "args": { + "m": 96, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 7168, + "n": 896, + "out": "bf16" + }, + "sources": [ + "moonshotai/Kimi-K3/block_sparse_moe.gate" + ] + }, + { + "type": "gemm", + "args": { + "m": 128, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 7168, + "n": 896, + "out": "bf16" + }, + "sources": [ + "moonshotai/Kimi-K3/block_sparse_moe.gate" + ] + }, + { + "type": "gemm", + "args": { + "m": 192, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 7168, + "n": 896, + "out": "bf16" + }, + "sources": [ + "moonshotai/Kimi-K3/block_sparse_moe.gate" + ] + }, + { + "type": "gemm", + "args": { + "m": 256, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 7168, + "n": 896, + "out": "bf16" + }, + "sources": [ + "moonshotai/Kimi-K3/block_sparse_moe.gate" + ] + }, + { + "type": "gemm", + "args": { + "m": 384, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 7168, + "n": 896, + "out": "bf16" + }, + "sources": [ + "moonshotai/Kimi-K3/block_sparse_moe.gate" + ] + }, + { + "type": "gemm", + "args": { + "m": 512, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 7168, + "n": 896, + "out": "bf16" + }, + "sources": [ + "moonshotai/Kimi-K3/block_sparse_moe.gate" + ] + }, + { + "type": "gemm", + "args": { + "m": 768, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 7168, + "n": 896, + "out": "bf16" + }, + "sources": [ + "moonshotai/Kimi-K3/block_sparse_moe.gate" + ] + }, + { + "type": "gemm", + "args": { + "m": 1024, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 7168, + "n": 896, + "out": "bf16" + }, + "sources": [ + "moonshotai/Kimi-K3/block_sparse_moe.gate" + ] + }, + { + "type": "gemm", + "args": { + "m": 1536, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 7168, + "n": 896, + "out": "bf16" + }, + "sources": [ + "moonshotai/Kimi-K3/block_sparse_moe.gate" + ] + }, + { + "type": "gemm", + "args": { + "m": 2048, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 7168, + "n": 896, + "out": "bf16" + }, + "sources": [ + "moonshotai/Kimi-K3/block_sparse_moe.gate" + ] + }, + { + "type": "gemm", + "args": { + "m": 3072, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 7168, + "n": 896, + "out": "bf16" + }, + "sources": [ + "moonshotai/Kimi-K3/block_sparse_moe.gate" + ] + }, + { + "type": "gemm", + "args": { + "m": 4096, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 7168, + "n": 896, + "out": "bf16" + }, + "sources": [ + "moonshotai/Kimi-K3/block_sparse_moe.gate" + ] + }, + { + "type": "gemm", + "args": { + "m": 6144, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 7168, + "n": 896, + "out": "bf16" + }, + "sources": [ + "moonshotai/Kimi-K3/block_sparse_moe.gate" + ] + }, + { + "type": "gemm", + "args": { + "m": 8192, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 7168, + "n": 896, + "out": "bf16" + }, + "sources": [ + "moonshotai/Kimi-K3/block_sparse_moe.gate" + ] + }, + { + "type": "gemm", + "args": { + "m": 12288, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 7168, + "n": 896, + "out": "bf16" + }, + "sources": [ + "moonshotai/Kimi-K3/block_sparse_moe.gate" + ] + }, + { + "type": "gemm", + "args": { + "m": 16384, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 7168, + "n": 896, + "out": "bf16" + }, + "sources": [ + "moonshotai/Kimi-K3/block_sparse_moe.gate" + ] + }, + { + "type": "gemm", + "args": { + "m": 24576, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 7168, + "n": 896, + "out": "bf16" + }, + "sources": [ + "moonshotai/Kimi-K3/block_sparse_moe.gate" + ] + }, + { + "type": "gemm", + "args": { + "m": 32768, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 7168, + "n": 896, + "out": "bf16" + }, + "sources": [ + "moonshotai/Kimi-K3/block_sparse_moe.gate" + ] + }, + { + "type": "gemm", + "args": { + "m": 49152, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 7168, + "n": 896, + "out": "bf16" + }, + "sources": [ + "moonshotai/Kimi-K3/block_sparse_moe.gate" + ] + }, + { + "type": "gemm", + "args": { + "m": 65536, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 7168, + "n": 896, + "out": "bf16" + }, + "sources": [ + "moonshotai/Kimi-K3/block_sparse_moe.gate" + ] + }, + { + "type": "gemm", + "args": { + "m": 1, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 7168, + "n": 896, + "out": "fp32" + }, + "sources": [ + "moonshotai/Kimi-K3/block_sparse_moe.gate" + ] + }, + { + "type": "gemm", + "args": { + "m": 2, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 7168, + "n": 896, + "out": "fp32" + }, + "sources": [ + "moonshotai/Kimi-K3/block_sparse_moe.gate" + ] + }, + { + "type": "gemm", + "args": { + "m": 3, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 7168, + "n": 896, + "out": "fp32" + }, + "sources": [ + "moonshotai/Kimi-K3/block_sparse_moe.gate" + ] + }, + { + "type": "gemm", + "args": { + "m": 4, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 7168, + "n": 896, + "out": "fp32" + }, + "sources": [ + "moonshotai/Kimi-K3/block_sparse_moe.gate" + ] + }, + { + "type": "gemm", + "args": { + "m": 6, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 7168, + "n": 896, + "out": "fp32" + }, + "sources": [ + "moonshotai/Kimi-K3/block_sparse_moe.gate" + ] + }, + { + "type": "gemm", + "args": { + "m": 8, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 7168, + "n": 896, + "out": "fp32" + }, + "sources": [ + "moonshotai/Kimi-K3/block_sparse_moe.gate" + ] + }, + { + "type": "gemm", + "args": { + "m": 12, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 7168, + "n": 896, + "out": "fp32" + }, + "sources": [ + "moonshotai/Kimi-K3/block_sparse_moe.gate" + ] + }, + { + "type": "gemm", + "args": { + "m": 16, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 7168, + "n": 896, + "out": "fp32" + }, + "sources": [ + "moonshotai/Kimi-K3/block_sparse_moe.gate" + ] + }, + { + "type": "gemm", + "args": { + "m": 24, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 7168, + "n": 896, + "out": "fp32" + }, + "sources": [ + "moonshotai/Kimi-K3/block_sparse_moe.gate" + ] + }, + { + "type": "gemm", + "args": { + "m": 32, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 7168, + "n": 896, + "out": "fp32" + }, + "sources": [ + "moonshotai/Kimi-K3/block_sparse_moe.gate" + ] + }, + { + "type": "gemm", + "args": { + "m": 48, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 7168, + "n": 896, + "out": "fp32" + }, + "sources": [ + "moonshotai/Kimi-K3/block_sparse_moe.gate" + ] + }, + { + "type": "gemm", + "args": { + "m": 64, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 7168, + "n": 896, + "out": "fp32" + }, + "sources": [ + "moonshotai/Kimi-K3/block_sparse_moe.gate" + ] + }, + { + "type": "gemm", + "args": { + "m": 96, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 7168, + "n": 896, + "out": "fp32" + }, + "sources": [ + "moonshotai/Kimi-K3/block_sparse_moe.gate" + ] + }, + { + "type": "gemm", + "args": { + "m": 128, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 7168, + "n": 896, + "out": "fp32" + }, + "sources": [ + "moonshotai/Kimi-K3/block_sparse_moe.gate" + ] + }, + { + "type": "gemm", + "args": { + "m": 192, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 7168, + "n": 896, + "out": "fp32" + }, + "sources": [ + "moonshotai/Kimi-K3/block_sparse_moe.gate" + ] + }, + { + "type": "gemm", + "args": { + "m": 256, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 7168, + "n": 896, + "out": "fp32" + }, + "sources": [ + "moonshotai/Kimi-K3/block_sparse_moe.gate" + ] + }, + { + "type": "gemm", + "args": { + "m": 384, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 7168, + "n": 896, + "out": "fp32" + }, + "sources": [ + "moonshotai/Kimi-K3/block_sparse_moe.gate" + ] + }, + { + "type": "gemm", + "args": { + "m": 512, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 7168, + "n": 896, + "out": "fp32" + }, + "sources": [ + "moonshotai/Kimi-K3/block_sparse_moe.gate" + ] + }, + { + "type": "gemm", + "args": { + "m": 768, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 7168, + "n": 896, + "out": "fp32" + }, + "sources": [ + "moonshotai/Kimi-K3/block_sparse_moe.gate" + ] + }, + { + "type": "gemm", + "args": { + "m": 1024, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 7168, + "n": 896, + "out": "fp32" + }, + "sources": [ + "moonshotai/Kimi-K3/block_sparse_moe.gate" + ] + }, + { + "type": "gemm", + "args": { + "m": 1536, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 7168, + "n": 896, + "out": "fp32" + }, + "sources": [ + "moonshotai/Kimi-K3/block_sparse_moe.gate" + ] + }, + { + "type": "gemm", + "args": { + "m": 2048, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 7168, + "n": 896, + "out": "fp32" + }, + "sources": [ + "moonshotai/Kimi-K3/block_sparse_moe.gate" + ] + }, + { + "type": "gemm", + "args": { + "m": 3072, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 7168, + "n": 896, + "out": "fp32" + }, + "sources": [ + "moonshotai/Kimi-K3/block_sparse_moe.gate" + ] + }, + { + "type": "gemm", + "args": { + "m": 4096, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 7168, + "n": 896, + "out": "fp32" + }, + "sources": [ + "moonshotai/Kimi-K3/block_sparse_moe.gate" + ] + }, + { + "type": "gemm", + "args": { + "m": 6144, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 7168, + "n": 896, + "out": "fp32" + }, + "sources": [ + "moonshotai/Kimi-K3/block_sparse_moe.gate" + ] + }, + { + "type": "gemm", + "args": { + "m": 8192, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 7168, + "n": 896, + "out": "fp32" + }, + "sources": [ + "moonshotai/Kimi-K3/block_sparse_moe.gate" + ] + }, + { + "type": "gemm", + "args": { + "m": 12288, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 7168, + "n": 896, + "out": "fp32" + }, + "sources": [ + "moonshotai/Kimi-K3/block_sparse_moe.gate" + ] + }, + { + "type": "gemm", + "args": { + "m": 16384, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 7168, + "n": 896, + "out": "fp32" + }, + "sources": [ + "moonshotai/Kimi-K3/block_sparse_moe.gate" + ] + }, + { + "type": "gemm", + "args": { + "m": 24576, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 7168, + "n": 896, + "out": "fp32" + }, + "sources": [ + "moonshotai/Kimi-K3/block_sparse_moe.gate" + ] + }, + { + "type": "gemm", + "args": { + "m": 32768, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 7168, + "n": 896, + "out": "fp32" + }, + "sources": [ + "moonshotai/Kimi-K3/block_sparse_moe.gate" + ] + }, + { + "type": "gemm", + "args": { + "m": 49152, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 7168, + "n": 896, + "out": "fp32" + }, + "sources": [ + "moonshotai/Kimi-K3/block_sparse_moe.gate" + ] + }, + { + "type": "gemm", + "args": { + "m": 65536, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 7168, + "n": 896, + "out": "fp32" + }, + "sources": [ + "moonshotai/Kimi-K3/block_sparse_moe.gate" + ] + }, + { + "type": "gemm", + "args": { + "m": 1, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 7168, + "n": 4480, + "out": "bf16" + }, + "sources": [ + "moonshotai/Kimi-K3/block_sparse_moe.gate+block_sparse_moe.routed_expert_down_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 2, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 7168, + "n": 4480, + "out": "bf16" + }, + "sources": [ + "moonshotai/Kimi-K3/block_sparse_moe.gate+block_sparse_moe.routed_expert_down_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 3, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 7168, + "n": 4480, + "out": "bf16" + }, + "sources": [ + "moonshotai/Kimi-K3/block_sparse_moe.gate+block_sparse_moe.routed_expert_down_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 4, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 7168, + "n": 4480, + "out": "bf16" + }, + "sources": [ + "moonshotai/Kimi-K3/block_sparse_moe.gate+block_sparse_moe.routed_expert_down_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 6, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 7168, + "n": 4480, + "out": "bf16" + }, + "sources": [ + "moonshotai/Kimi-K3/block_sparse_moe.gate+block_sparse_moe.routed_expert_down_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 8, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 7168, + "n": 4480, + "out": "bf16" + }, + "sources": [ + "moonshotai/Kimi-K3/block_sparse_moe.gate+block_sparse_moe.routed_expert_down_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 12, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 7168, + "n": 4480, + "out": "bf16" + }, + "sources": [ + "moonshotai/Kimi-K3/block_sparse_moe.gate+block_sparse_moe.routed_expert_down_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 16, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 7168, + "n": 4480, + "out": "bf16" + }, + "sources": [ + "moonshotai/Kimi-K3/block_sparse_moe.gate+block_sparse_moe.routed_expert_down_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 24, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 7168, + "n": 4480, + "out": "bf16" + }, + "sources": [ + "moonshotai/Kimi-K3/block_sparse_moe.gate+block_sparse_moe.routed_expert_down_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 32, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 7168, + "n": 4480, + "out": "bf16" + }, + "sources": [ + "moonshotai/Kimi-K3/block_sparse_moe.gate+block_sparse_moe.routed_expert_down_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 48, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 7168, + "n": 4480, + "out": "bf16" + }, + "sources": [ + "moonshotai/Kimi-K3/block_sparse_moe.gate+block_sparse_moe.routed_expert_down_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 64, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 7168, + "n": 4480, + "out": "bf16" + }, + "sources": [ + "moonshotai/Kimi-K3/block_sparse_moe.gate+block_sparse_moe.routed_expert_down_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 96, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 7168, + "n": 4480, + "out": "bf16" + }, + "sources": [ + "moonshotai/Kimi-K3/block_sparse_moe.gate+block_sparse_moe.routed_expert_down_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 128, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 7168, + "n": 4480, + "out": "bf16" + }, + "sources": [ + "moonshotai/Kimi-K3/block_sparse_moe.gate+block_sparse_moe.routed_expert_down_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 192, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 7168, + "n": 4480, + "out": "bf16" + }, + "sources": [ + "moonshotai/Kimi-K3/block_sparse_moe.gate+block_sparse_moe.routed_expert_down_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 256, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 7168, + "n": 4480, + "out": "bf16" + }, + "sources": [ + "moonshotai/Kimi-K3/block_sparse_moe.gate+block_sparse_moe.routed_expert_down_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 384, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 7168, + "n": 4480, + "out": "bf16" + }, + "sources": [ + "moonshotai/Kimi-K3/block_sparse_moe.gate+block_sparse_moe.routed_expert_down_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 512, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 7168, + "n": 4480, + "out": "bf16" + }, + "sources": [ + "moonshotai/Kimi-K3/block_sparse_moe.gate+block_sparse_moe.routed_expert_down_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 768, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 7168, + "n": 4480, + "out": "bf16" + }, + "sources": [ + "moonshotai/Kimi-K3/block_sparse_moe.gate+block_sparse_moe.routed_expert_down_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 1024, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 7168, + "n": 4480, + "out": "bf16" + }, + "sources": [ + "moonshotai/Kimi-K3/block_sparse_moe.gate+block_sparse_moe.routed_expert_down_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 1536, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 7168, + "n": 4480, + "out": "bf16" + }, + "sources": [ + "moonshotai/Kimi-K3/block_sparse_moe.gate+block_sparse_moe.routed_expert_down_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 2048, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 7168, + "n": 4480, + "out": "bf16" + }, + "sources": [ + "moonshotai/Kimi-K3/block_sparse_moe.gate+block_sparse_moe.routed_expert_down_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 3072, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 7168, + "n": 4480, + "out": "bf16" + }, + "sources": [ + "moonshotai/Kimi-K3/block_sparse_moe.gate+block_sparse_moe.routed_expert_down_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 4096, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 7168, + "n": 4480, + "out": "bf16" + }, + "sources": [ + "moonshotai/Kimi-K3/block_sparse_moe.gate+block_sparse_moe.routed_expert_down_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 6144, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 7168, + "n": 4480, + "out": "bf16" + }, + "sources": [ + "moonshotai/Kimi-K3/block_sparse_moe.gate+block_sparse_moe.routed_expert_down_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 8192, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 7168, + "n": 4480, + "out": "bf16" + }, + "sources": [ + "moonshotai/Kimi-K3/block_sparse_moe.gate+block_sparse_moe.routed_expert_down_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 12288, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 7168, + "n": 4480, + "out": "bf16" + }, + "sources": [ + "moonshotai/Kimi-K3/block_sparse_moe.gate+block_sparse_moe.routed_expert_down_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 16384, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 7168, + "n": 4480, + "out": "bf16" + }, + "sources": [ + "moonshotai/Kimi-K3/block_sparse_moe.gate+block_sparse_moe.routed_expert_down_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 24576, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 7168, + "n": 4480, + "out": "bf16" + }, + "sources": [ + "moonshotai/Kimi-K3/block_sparse_moe.gate+block_sparse_moe.routed_expert_down_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 32768, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 7168, + "n": 4480, + "out": "bf16" + }, + "sources": [ + "moonshotai/Kimi-K3/block_sparse_moe.gate+block_sparse_moe.routed_expert_down_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 49152, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 7168, + "n": 4480, + "out": "bf16" + }, + "sources": [ + "moonshotai/Kimi-K3/block_sparse_moe.gate+block_sparse_moe.routed_expert_down_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 65536, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 7168, + "n": 4480, + "out": "bf16" + }, + "sources": [ + "moonshotai/Kimi-K3/block_sparse_moe.gate+block_sparse_moe.routed_expert_down_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 1, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 7168, + "n": 4480, + "out": "fp32" + }, + "sources": [ + "moonshotai/Kimi-K3/block_sparse_moe.gate+block_sparse_moe.routed_expert_down_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 2, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 7168, + "n": 4480, + "out": "fp32" + }, + "sources": [ + "moonshotai/Kimi-K3/block_sparse_moe.gate+block_sparse_moe.routed_expert_down_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 3, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 7168, + "n": 4480, + "out": "fp32" + }, + "sources": [ + "moonshotai/Kimi-K3/block_sparse_moe.gate+block_sparse_moe.routed_expert_down_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 4, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 7168, + "n": 4480, + "out": "fp32" + }, + "sources": [ + "moonshotai/Kimi-K3/block_sparse_moe.gate+block_sparse_moe.routed_expert_down_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 6, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 7168, + "n": 4480, + "out": "fp32" + }, + "sources": [ + "moonshotai/Kimi-K3/block_sparse_moe.gate+block_sparse_moe.routed_expert_down_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 8, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 7168, + "n": 4480, + "out": "fp32" + }, + "sources": [ + "moonshotai/Kimi-K3/block_sparse_moe.gate+block_sparse_moe.routed_expert_down_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 12, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 7168, + "n": 4480, + "out": "fp32" + }, + "sources": [ + "moonshotai/Kimi-K3/block_sparse_moe.gate+block_sparse_moe.routed_expert_down_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 16, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 7168, + "n": 4480, + "out": "fp32" + }, + "sources": [ + "moonshotai/Kimi-K3/block_sparse_moe.gate+block_sparse_moe.routed_expert_down_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 24, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 7168, + "n": 4480, + "out": "fp32" + }, + "sources": [ + "moonshotai/Kimi-K3/block_sparse_moe.gate+block_sparse_moe.routed_expert_down_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 32, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 7168, + "n": 4480, + "out": "fp32" + }, + "sources": [ + "moonshotai/Kimi-K3/block_sparse_moe.gate+block_sparse_moe.routed_expert_down_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 48, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 7168, + "n": 4480, + "out": "fp32" + }, + "sources": [ + "moonshotai/Kimi-K3/block_sparse_moe.gate+block_sparse_moe.routed_expert_down_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 64, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 7168, + "n": 4480, + "out": "fp32" + }, + "sources": [ + "moonshotai/Kimi-K3/block_sparse_moe.gate+block_sparse_moe.routed_expert_down_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 96, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 7168, + "n": 4480, + "out": "fp32" + }, + "sources": [ + "moonshotai/Kimi-K3/block_sparse_moe.gate+block_sparse_moe.routed_expert_down_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 128, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 7168, + "n": 4480, + "out": "fp32" + }, + "sources": [ + "moonshotai/Kimi-K3/block_sparse_moe.gate+block_sparse_moe.routed_expert_down_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 192, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 7168, + "n": 4480, + "out": "fp32" + }, + "sources": [ + "moonshotai/Kimi-K3/block_sparse_moe.gate+block_sparse_moe.routed_expert_down_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 256, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 7168, + "n": 4480, + "out": "fp32" + }, + "sources": [ + "moonshotai/Kimi-K3/block_sparse_moe.gate+block_sparse_moe.routed_expert_down_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 384, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 7168, + "n": 4480, + "out": "fp32" + }, + "sources": [ + "moonshotai/Kimi-K3/block_sparse_moe.gate+block_sparse_moe.routed_expert_down_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 512, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 7168, + "n": 4480, + "out": "fp32" + }, + "sources": [ + "moonshotai/Kimi-K3/block_sparse_moe.gate+block_sparse_moe.routed_expert_down_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 768, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 7168, + "n": 4480, + "out": "fp32" + }, + "sources": [ + "moonshotai/Kimi-K3/block_sparse_moe.gate+block_sparse_moe.routed_expert_down_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 1024, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 7168, + "n": 4480, + "out": "fp32" + }, + "sources": [ + "moonshotai/Kimi-K3/block_sparse_moe.gate+block_sparse_moe.routed_expert_down_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 1536, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 7168, + "n": 4480, + "out": "fp32" + }, + "sources": [ + "moonshotai/Kimi-K3/block_sparse_moe.gate+block_sparse_moe.routed_expert_down_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 2048, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 7168, + "n": 4480, + "out": "fp32" + }, + "sources": [ + "moonshotai/Kimi-K3/block_sparse_moe.gate+block_sparse_moe.routed_expert_down_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 3072, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 7168, + "n": 4480, + "out": "fp32" + }, + "sources": [ + "moonshotai/Kimi-K3/block_sparse_moe.gate+block_sparse_moe.routed_expert_down_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 4096, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 7168, + "n": 4480, + "out": "fp32" + }, + "sources": [ + "moonshotai/Kimi-K3/block_sparse_moe.gate+block_sparse_moe.routed_expert_down_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 6144, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 7168, + "n": 4480, + "out": "fp32" + }, + "sources": [ + "moonshotai/Kimi-K3/block_sparse_moe.gate+block_sparse_moe.routed_expert_down_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 8192, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 7168, + "n": 4480, + "out": "fp32" + }, + "sources": [ + "moonshotai/Kimi-K3/block_sparse_moe.gate+block_sparse_moe.routed_expert_down_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 12288, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 7168, + "n": 4480, + "out": "fp32" + }, + "sources": [ + "moonshotai/Kimi-K3/block_sparse_moe.gate+block_sparse_moe.routed_expert_down_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 16384, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 7168, + "n": 4480, + "out": "fp32" + }, + "sources": [ + "moonshotai/Kimi-K3/block_sparse_moe.gate+block_sparse_moe.routed_expert_down_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 24576, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 7168, + "n": 4480, + "out": "fp32" + }, + "sources": [ + "moonshotai/Kimi-K3/block_sparse_moe.gate+block_sparse_moe.routed_expert_down_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 32768, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 7168, + "n": 4480, + "out": "fp32" + }, + "sources": [ + "moonshotai/Kimi-K3/block_sparse_moe.gate+block_sparse_moe.routed_expert_down_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 49152, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 7168, + "n": 4480, + "out": "fp32" + }, + "sources": [ + "moonshotai/Kimi-K3/block_sparse_moe.gate+block_sparse_moe.routed_expert_down_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 65536, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 7168, + "n": 4480, + "out": "fp32" + }, + "sources": [ + "moonshotai/Kimi-K3/block_sparse_moe.gate+block_sparse_moe.routed_expert_down_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 1, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 7168, + "n": 3584, + "out": "bf16" + }, + "sources": [ + "moonshotai/Kimi-K3/block_sparse_moe.routed_expert_down_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 2, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 7168, + "n": 3584, + "out": "bf16" + }, + "sources": [ + "moonshotai/Kimi-K3/block_sparse_moe.routed_expert_down_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 3, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 7168, + "n": 3584, + "out": "bf16" + }, + "sources": [ + "moonshotai/Kimi-K3/block_sparse_moe.routed_expert_down_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 4, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 7168, + "n": 3584, + "out": "bf16" + }, + "sources": [ + "moonshotai/Kimi-K3/block_sparse_moe.routed_expert_down_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 6, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 7168, + "n": 3584, + "out": "bf16" + }, + "sources": [ + "moonshotai/Kimi-K3/block_sparse_moe.routed_expert_down_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 8, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 7168, + "n": 3584, + "out": "bf16" + }, + "sources": [ + "moonshotai/Kimi-K3/block_sparse_moe.routed_expert_down_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 12, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 7168, + "n": 3584, + "out": "bf16" + }, + "sources": [ + "moonshotai/Kimi-K3/block_sparse_moe.routed_expert_down_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 16, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 7168, + "n": 3584, + "out": "bf16" + }, + "sources": [ + "moonshotai/Kimi-K3/block_sparse_moe.routed_expert_down_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 24, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 7168, + "n": 3584, + "out": "bf16" + }, + "sources": [ + "moonshotai/Kimi-K3/block_sparse_moe.routed_expert_down_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 32, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 7168, + "n": 3584, + "out": "bf16" + }, + "sources": [ + "moonshotai/Kimi-K3/block_sparse_moe.routed_expert_down_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 48, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 7168, + "n": 3584, + "out": "bf16" + }, + "sources": [ + "moonshotai/Kimi-K3/block_sparse_moe.routed_expert_down_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 64, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 7168, + "n": 3584, + "out": "bf16" + }, + "sources": [ + "moonshotai/Kimi-K3/block_sparse_moe.routed_expert_down_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 96, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 7168, + "n": 3584, + "out": "bf16" + }, + "sources": [ + "moonshotai/Kimi-K3/block_sparse_moe.routed_expert_down_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 128, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 7168, + "n": 3584, + "out": "bf16" + }, + "sources": [ + "moonshotai/Kimi-K3/block_sparse_moe.routed_expert_down_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 192, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 7168, + "n": 3584, + "out": "bf16" + }, + "sources": [ + "moonshotai/Kimi-K3/block_sparse_moe.routed_expert_down_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 256, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 7168, + "n": 3584, + "out": "bf16" + }, + "sources": [ + "moonshotai/Kimi-K3/block_sparse_moe.routed_expert_down_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 384, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 7168, + "n": 3584, + "out": "bf16" + }, + "sources": [ + "moonshotai/Kimi-K3/block_sparse_moe.routed_expert_down_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 512, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 7168, + "n": 3584, + "out": "bf16" + }, + "sources": [ + "moonshotai/Kimi-K3/block_sparse_moe.routed_expert_down_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 768, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 7168, + "n": 3584, + "out": "bf16" + }, + "sources": [ + "moonshotai/Kimi-K3/block_sparse_moe.routed_expert_down_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 1024, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 7168, + "n": 3584, + "out": "bf16" + }, + "sources": [ + "moonshotai/Kimi-K3/block_sparse_moe.routed_expert_down_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 1536, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 7168, + "n": 3584, + "out": "bf16" + }, + "sources": [ + "moonshotai/Kimi-K3/block_sparse_moe.routed_expert_down_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 2048, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 7168, + "n": 3584, + "out": "bf16" + }, + "sources": [ + "moonshotai/Kimi-K3/block_sparse_moe.routed_expert_down_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 3072, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 7168, + "n": 3584, + "out": "bf16" + }, + "sources": [ + "moonshotai/Kimi-K3/block_sparse_moe.routed_expert_down_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 4096, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 7168, + "n": 3584, + "out": "bf16" + }, + "sources": [ + "moonshotai/Kimi-K3/block_sparse_moe.routed_expert_down_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 6144, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 7168, + "n": 3584, + "out": "bf16" + }, + "sources": [ + "moonshotai/Kimi-K3/block_sparse_moe.routed_expert_down_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 8192, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 7168, + "n": 3584, + "out": "bf16" + }, + "sources": [ + "moonshotai/Kimi-K3/block_sparse_moe.routed_expert_down_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 12288, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 7168, + "n": 3584, + "out": "bf16" + }, + "sources": [ + "moonshotai/Kimi-K3/block_sparse_moe.routed_expert_down_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 16384, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 7168, + "n": 3584, + "out": "bf16" + }, + "sources": [ + "moonshotai/Kimi-K3/block_sparse_moe.routed_expert_down_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 24576, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 7168, + "n": 3584, + "out": "bf16" + }, + "sources": [ + "moonshotai/Kimi-K3/block_sparse_moe.routed_expert_down_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 32768, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 7168, + "n": 3584, + "out": "bf16" + }, + "sources": [ + "moonshotai/Kimi-K3/block_sparse_moe.routed_expert_down_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 49152, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 7168, + "n": 3584, + "out": "bf16" + }, + "sources": [ + "moonshotai/Kimi-K3/block_sparse_moe.routed_expert_down_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 65536, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 7168, + "n": 3584, + "out": "bf16" + }, + "sources": [ + "moonshotai/Kimi-K3/block_sparse_moe.routed_expert_down_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 1, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 3584, + "n": 7168, + "out": "bf16" + }, + "sources": [ + "moonshotai/Kimi-K3/block_sparse_moe.routed_expert_up_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 2, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 3584, + "n": 7168, + "out": "bf16" + }, + "sources": [ + "moonshotai/Kimi-K3/block_sparse_moe.routed_expert_up_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 3, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 3584, + "n": 7168, + "out": "bf16" + }, + "sources": [ + "moonshotai/Kimi-K3/block_sparse_moe.routed_expert_up_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 4, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 3584, + "n": 7168, + "out": "bf16" + }, + "sources": [ + "moonshotai/Kimi-K3/block_sparse_moe.routed_expert_up_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 6, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 3584, + "n": 7168, + "out": "bf16" + }, + "sources": [ + "moonshotai/Kimi-K3/block_sparse_moe.routed_expert_up_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 8, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 3584, + "n": 7168, + "out": "bf16" + }, + "sources": [ + "moonshotai/Kimi-K3/block_sparse_moe.routed_expert_up_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 12, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 3584, + "n": 7168, + "out": "bf16" + }, + "sources": [ + "moonshotai/Kimi-K3/block_sparse_moe.routed_expert_up_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 16, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 3584, + "n": 7168, + "out": "bf16" + }, + "sources": [ + "moonshotai/Kimi-K3/block_sparse_moe.routed_expert_up_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 24, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 3584, + "n": 7168, + "out": "bf16" + }, + "sources": [ + "moonshotai/Kimi-K3/block_sparse_moe.routed_expert_up_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 32, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 3584, + "n": 7168, + "out": "bf16" + }, + "sources": [ + "moonshotai/Kimi-K3/block_sparse_moe.routed_expert_up_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 48, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 3584, + "n": 7168, + "out": "bf16" + }, + "sources": [ + "moonshotai/Kimi-K3/block_sparse_moe.routed_expert_up_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 64, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 3584, + "n": 7168, + "out": "bf16" + }, + "sources": [ + "moonshotai/Kimi-K3/block_sparse_moe.routed_expert_up_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 96, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 3584, + "n": 7168, + "out": "bf16" + }, + "sources": [ + "moonshotai/Kimi-K3/block_sparse_moe.routed_expert_up_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 128, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 3584, + "n": 7168, + "out": "bf16" + }, + "sources": [ + "moonshotai/Kimi-K3/block_sparse_moe.routed_expert_up_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 192, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 3584, + "n": 7168, + "out": "bf16" + }, + "sources": [ + "moonshotai/Kimi-K3/block_sparse_moe.routed_expert_up_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 256, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 3584, + "n": 7168, + "out": "bf16" + }, + "sources": [ + "moonshotai/Kimi-K3/block_sparse_moe.routed_expert_up_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 384, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 3584, + "n": 7168, + "out": "bf16" + }, + "sources": [ + "moonshotai/Kimi-K3/block_sparse_moe.routed_expert_up_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 512, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 3584, + "n": 7168, + "out": "bf16" + }, + "sources": [ + "moonshotai/Kimi-K3/block_sparse_moe.routed_expert_up_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 768, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 3584, + "n": 7168, + "out": "bf16" + }, + "sources": [ + "moonshotai/Kimi-K3/block_sparse_moe.routed_expert_up_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 1024, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 3584, + "n": 7168, + "out": "bf16" + }, + "sources": [ + "moonshotai/Kimi-K3/block_sparse_moe.routed_expert_up_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 1536, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 3584, + "n": 7168, + "out": "bf16" + }, + "sources": [ + "moonshotai/Kimi-K3/block_sparse_moe.routed_expert_up_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 2048, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 3584, + "n": 7168, + "out": "bf16" + }, + "sources": [ + "moonshotai/Kimi-K3/block_sparse_moe.routed_expert_up_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 3072, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 3584, + "n": 7168, + "out": "bf16" + }, + "sources": [ + "moonshotai/Kimi-K3/block_sparse_moe.routed_expert_up_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 4096, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 3584, + "n": 7168, + "out": "bf16" + }, + "sources": [ + "moonshotai/Kimi-K3/block_sparse_moe.routed_expert_up_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 6144, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 3584, + "n": 7168, + "out": "bf16" + }, + "sources": [ + "moonshotai/Kimi-K3/block_sparse_moe.routed_expert_up_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 8192, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 3584, + "n": 7168, + "out": "bf16" + }, + "sources": [ + "moonshotai/Kimi-K3/block_sparse_moe.routed_expert_up_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 12288, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 3584, + "n": 7168, + "out": "bf16" + }, + "sources": [ + "moonshotai/Kimi-K3/block_sparse_moe.routed_expert_up_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 16384, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 3584, + "n": 7168, + "out": "bf16" + }, + "sources": [ + "moonshotai/Kimi-K3/block_sparse_moe.routed_expert_up_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 24576, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 3584, + "n": 7168, + "out": "bf16" + }, + "sources": [ + "moonshotai/Kimi-K3/block_sparse_moe.routed_expert_up_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 32768, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 3584, + "n": 7168, + "out": "bf16" + }, + "sources": [ + "moonshotai/Kimi-K3/block_sparse_moe.routed_expert_up_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 49152, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 3584, + "n": 7168, + "out": "bf16" + }, + "sources": [ + "moonshotai/Kimi-K3/block_sparse_moe.routed_expert_up_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 65536, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 3584, + "n": 7168, + "out": "bf16" + }, + "sources": [ + "moonshotai/Kimi-K3/block_sparse_moe.routed_expert_up_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 1, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 6144, + "n": 7168, + "out": "bf16" + }, + "sources": [ + "moonshotai/Kimi-K3/block_sparse_moe.shared_experts.down_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 2, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 6144, + "n": 7168, + "out": "bf16" + }, + "sources": [ + "moonshotai/Kimi-K3/block_sparse_moe.shared_experts.down_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 3, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 6144, + "n": 7168, + "out": "bf16" + }, + "sources": [ + "moonshotai/Kimi-K3/block_sparse_moe.shared_experts.down_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 4, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 6144, + "n": 7168, + "out": "bf16" + }, + "sources": [ + "moonshotai/Kimi-K3/block_sparse_moe.shared_experts.down_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 6, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 6144, + "n": 7168, + "out": "bf16" + }, + "sources": [ + "moonshotai/Kimi-K3/block_sparse_moe.shared_experts.down_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 8, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 6144, + "n": 7168, + "out": "bf16" + }, + "sources": [ + "moonshotai/Kimi-K3/block_sparse_moe.shared_experts.down_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 12, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 6144, + "n": 7168, + "out": "bf16" + }, + "sources": [ + "moonshotai/Kimi-K3/block_sparse_moe.shared_experts.down_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 16, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 6144, + "n": 7168, + "out": "bf16" + }, + "sources": [ + "moonshotai/Kimi-K3/block_sparse_moe.shared_experts.down_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 24, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 6144, + "n": 7168, + "out": "bf16" + }, + "sources": [ + "moonshotai/Kimi-K3/block_sparse_moe.shared_experts.down_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 32, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 6144, + "n": 7168, + "out": "bf16" + }, + "sources": [ + "moonshotai/Kimi-K3/block_sparse_moe.shared_experts.down_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 48, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 6144, + "n": 7168, + "out": "bf16" + }, + "sources": [ + "moonshotai/Kimi-K3/block_sparse_moe.shared_experts.down_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 64, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 6144, + "n": 7168, + "out": "bf16" + }, + "sources": [ + "moonshotai/Kimi-K3/block_sparse_moe.shared_experts.down_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 96, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 6144, + "n": 7168, + "out": "bf16" + }, + "sources": [ + "moonshotai/Kimi-K3/block_sparse_moe.shared_experts.down_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 128, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 6144, + "n": 7168, + "out": "bf16" + }, + "sources": [ + "moonshotai/Kimi-K3/block_sparse_moe.shared_experts.down_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 192, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 6144, + "n": 7168, + "out": "bf16" + }, + "sources": [ + "moonshotai/Kimi-K3/block_sparse_moe.shared_experts.down_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 256, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 6144, + "n": 7168, + "out": "bf16" + }, + "sources": [ + "moonshotai/Kimi-K3/block_sparse_moe.shared_experts.down_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 384, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 6144, + "n": 7168, + "out": "bf16" + }, + "sources": [ + "moonshotai/Kimi-K3/block_sparse_moe.shared_experts.down_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 512, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 6144, + "n": 7168, + "out": "bf16" + }, + "sources": [ + "moonshotai/Kimi-K3/block_sparse_moe.shared_experts.down_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 768, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 6144, + "n": 7168, + "out": "bf16" + }, + "sources": [ + "moonshotai/Kimi-K3/block_sparse_moe.shared_experts.down_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 1024, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 6144, + "n": 7168, + "out": "bf16" + }, + "sources": [ + "moonshotai/Kimi-K3/block_sparse_moe.shared_experts.down_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 1536, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 6144, + "n": 7168, + "out": "bf16" + }, + "sources": [ + "moonshotai/Kimi-K3/block_sparse_moe.shared_experts.down_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 2048, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 6144, + "n": 7168, + "out": "bf16" + }, + "sources": [ + "moonshotai/Kimi-K3/block_sparse_moe.shared_experts.down_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 3072, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 6144, + "n": 7168, + "out": "bf16" + }, + "sources": [ + "moonshotai/Kimi-K3/block_sparse_moe.shared_experts.down_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 4096, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 6144, + "n": 7168, + "out": "bf16" + }, + "sources": [ + "moonshotai/Kimi-K3/block_sparse_moe.shared_experts.down_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 6144, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 6144, + "n": 7168, + "out": "bf16" + }, + "sources": [ + "moonshotai/Kimi-K3/block_sparse_moe.shared_experts.down_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 8192, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 6144, + "n": 7168, + "out": "bf16" + }, + "sources": [ + "moonshotai/Kimi-K3/block_sparse_moe.shared_experts.down_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 12288, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 6144, + "n": 7168, + "out": "bf16" + }, + "sources": [ + "moonshotai/Kimi-K3/block_sparse_moe.shared_experts.down_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 16384, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 6144, + "n": 7168, + "out": "bf16" + }, + "sources": [ + "moonshotai/Kimi-K3/block_sparse_moe.shared_experts.down_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 24576, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 6144, + "n": 7168, + "out": "bf16" + }, + "sources": [ + "moonshotai/Kimi-K3/block_sparse_moe.shared_experts.down_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 32768, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 6144, + "n": 7168, + "out": "bf16" + }, + "sources": [ + "moonshotai/Kimi-K3/block_sparse_moe.shared_experts.down_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 49152, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 6144, + "n": 7168, + "out": "bf16" + }, + "sources": [ + "moonshotai/Kimi-K3/block_sparse_moe.shared_experts.down_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 65536, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 6144, + "n": 7168, + "out": "bf16" + }, + "sources": [ + "moonshotai/Kimi-K3/block_sparse_moe.shared_experts.down_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 1, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 7168, + "n": 12288, + "out": "bf16" + }, + "sources": [ + "moonshotai/Kimi-K3/block_sparse_moe.shared_experts.gate_proj+block_sparse_moe.shared_experts.up_proj", + "moonshotai/Kimi-K3/self_attn.g_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 2, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 7168, + "n": 12288, + "out": "bf16" + }, + "sources": [ + "moonshotai/Kimi-K3/block_sparse_moe.shared_experts.gate_proj+block_sparse_moe.shared_experts.up_proj", + "moonshotai/Kimi-K3/self_attn.g_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 3, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 7168, + "n": 12288, + "out": "bf16" + }, + "sources": [ + "moonshotai/Kimi-K3/block_sparse_moe.shared_experts.gate_proj+block_sparse_moe.shared_experts.up_proj", + "moonshotai/Kimi-K3/self_attn.g_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 4, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 7168, + "n": 12288, + "out": "bf16" + }, + "sources": [ + "moonshotai/Kimi-K3/block_sparse_moe.shared_experts.gate_proj+block_sparse_moe.shared_experts.up_proj", + "moonshotai/Kimi-K3/self_attn.g_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 6, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 7168, + "n": 12288, + "out": "bf16" + }, + "sources": [ + "moonshotai/Kimi-K3/block_sparse_moe.shared_experts.gate_proj+block_sparse_moe.shared_experts.up_proj", + "moonshotai/Kimi-K3/self_attn.g_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 8, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 7168, + "n": 12288, + "out": "bf16" + }, + "sources": [ + "moonshotai/Kimi-K3/block_sparse_moe.shared_experts.gate_proj+block_sparse_moe.shared_experts.up_proj", + "moonshotai/Kimi-K3/self_attn.g_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 12, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 7168, + "n": 12288, + "out": "bf16" + }, + "sources": [ + "moonshotai/Kimi-K3/block_sparse_moe.shared_experts.gate_proj+block_sparse_moe.shared_experts.up_proj", + "moonshotai/Kimi-K3/self_attn.g_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 16, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 7168, + "n": 12288, + "out": "bf16" + }, + "sources": [ + "moonshotai/Kimi-K3/block_sparse_moe.shared_experts.gate_proj+block_sparse_moe.shared_experts.up_proj", + "moonshotai/Kimi-K3/self_attn.g_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 24, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 7168, + "n": 12288, + "out": "bf16" + }, + "sources": [ + "moonshotai/Kimi-K3/block_sparse_moe.shared_experts.gate_proj+block_sparse_moe.shared_experts.up_proj", + "moonshotai/Kimi-K3/self_attn.g_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 32, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 7168, + "n": 12288, + "out": "bf16" + }, + "sources": [ + "moonshotai/Kimi-K3/block_sparse_moe.shared_experts.gate_proj+block_sparse_moe.shared_experts.up_proj", + "moonshotai/Kimi-K3/self_attn.g_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 48, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 7168, + "n": 12288, + "out": "bf16" + }, + "sources": [ + "moonshotai/Kimi-K3/block_sparse_moe.shared_experts.gate_proj+block_sparse_moe.shared_experts.up_proj", + "moonshotai/Kimi-K3/self_attn.g_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 64, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 7168, + "n": 12288, + "out": "bf16" + }, + "sources": [ + "moonshotai/Kimi-K3/block_sparse_moe.shared_experts.gate_proj+block_sparse_moe.shared_experts.up_proj", + "moonshotai/Kimi-K3/self_attn.g_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 96, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 7168, + "n": 12288, + "out": "bf16" + }, + "sources": [ + "moonshotai/Kimi-K3/block_sparse_moe.shared_experts.gate_proj+block_sparse_moe.shared_experts.up_proj", + "moonshotai/Kimi-K3/self_attn.g_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 128, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 7168, + "n": 12288, + "out": "bf16" + }, + "sources": [ + "moonshotai/Kimi-K3/block_sparse_moe.shared_experts.gate_proj+block_sparse_moe.shared_experts.up_proj", + "moonshotai/Kimi-K3/self_attn.g_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 192, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 7168, + "n": 12288, + "out": "bf16" + }, + "sources": [ + "moonshotai/Kimi-K3/block_sparse_moe.shared_experts.gate_proj+block_sparse_moe.shared_experts.up_proj", + "moonshotai/Kimi-K3/self_attn.g_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 256, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 7168, + "n": 12288, + "out": "bf16" + }, + "sources": [ + "moonshotai/Kimi-K3/block_sparse_moe.shared_experts.gate_proj+block_sparse_moe.shared_experts.up_proj", + "moonshotai/Kimi-K3/self_attn.g_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 384, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 7168, + "n": 12288, + "out": "bf16" + }, + "sources": [ + "moonshotai/Kimi-K3/block_sparse_moe.shared_experts.gate_proj+block_sparse_moe.shared_experts.up_proj", + "moonshotai/Kimi-K3/self_attn.g_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 512, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 7168, + "n": 12288, + "out": "bf16" + }, + "sources": [ + "moonshotai/Kimi-K3/block_sparse_moe.shared_experts.gate_proj+block_sparse_moe.shared_experts.up_proj", + "moonshotai/Kimi-K3/self_attn.g_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 768, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 7168, + "n": 12288, + "out": "bf16" + }, + "sources": [ + "moonshotai/Kimi-K3/block_sparse_moe.shared_experts.gate_proj+block_sparse_moe.shared_experts.up_proj", + "moonshotai/Kimi-K3/self_attn.g_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 1024, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 7168, + "n": 12288, + "out": "bf16" + }, + "sources": [ + "moonshotai/Kimi-K3/block_sparse_moe.shared_experts.gate_proj+block_sparse_moe.shared_experts.up_proj", + "moonshotai/Kimi-K3/self_attn.g_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 1536, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 7168, + "n": 12288, + "out": "bf16" + }, + "sources": [ + "moonshotai/Kimi-K3/block_sparse_moe.shared_experts.gate_proj+block_sparse_moe.shared_experts.up_proj", + "moonshotai/Kimi-K3/self_attn.g_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 2048, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 7168, + "n": 12288, + "out": "bf16" + }, + "sources": [ + "moonshotai/Kimi-K3/block_sparse_moe.shared_experts.gate_proj+block_sparse_moe.shared_experts.up_proj", + "moonshotai/Kimi-K3/self_attn.g_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 3072, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 7168, + "n": 12288, + "out": "bf16" + }, + "sources": [ + "moonshotai/Kimi-K3/block_sparse_moe.shared_experts.gate_proj+block_sparse_moe.shared_experts.up_proj", + "moonshotai/Kimi-K3/self_attn.g_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 4096, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 7168, + "n": 12288, + "out": "bf16" + }, + "sources": [ + "moonshotai/Kimi-K3/block_sparse_moe.shared_experts.gate_proj+block_sparse_moe.shared_experts.up_proj", + "moonshotai/Kimi-K3/self_attn.g_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 6144, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 7168, + "n": 12288, + "out": "bf16" + }, + "sources": [ + "moonshotai/Kimi-K3/block_sparse_moe.shared_experts.gate_proj+block_sparse_moe.shared_experts.up_proj", + "moonshotai/Kimi-K3/self_attn.g_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 8192, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 7168, + "n": 12288, + "out": "bf16" + }, + "sources": [ + "moonshotai/Kimi-K3/block_sparse_moe.shared_experts.gate_proj+block_sparse_moe.shared_experts.up_proj", + "moonshotai/Kimi-K3/self_attn.g_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 12288, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 7168, + "n": 12288, + "out": "bf16" + }, + "sources": [ + "moonshotai/Kimi-K3/block_sparse_moe.shared_experts.gate_proj+block_sparse_moe.shared_experts.up_proj", + "moonshotai/Kimi-K3/self_attn.g_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 16384, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 7168, + "n": 12288, + "out": "bf16" + }, + "sources": [ + "moonshotai/Kimi-K3/block_sparse_moe.shared_experts.gate_proj+block_sparse_moe.shared_experts.up_proj", + "moonshotai/Kimi-K3/self_attn.g_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 24576, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 7168, + "n": 12288, + "out": "bf16" + }, + "sources": [ + "moonshotai/Kimi-K3/block_sparse_moe.shared_experts.gate_proj+block_sparse_moe.shared_experts.up_proj", + "moonshotai/Kimi-K3/self_attn.g_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 32768, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 7168, + "n": 12288, + "out": "bf16" + }, + "sources": [ + "moonshotai/Kimi-K3/block_sparse_moe.shared_experts.gate_proj+block_sparse_moe.shared_experts.up_proj", + "moonshotai/Kimi-K3/self_attn.g_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 49152, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 7168, + "n": 12288, + "out": "bf16" + }, + "sources": [ + "moonshotai/Kimi-K3/block_sparse_moe.shared_experts.gate_proj+block_sparse_moe.shared_experts.up_proj", + "moonshotai/Kimi-K3/self_attn.g_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 65536, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 7168, + "n": 12288, + "out": "bf16" + }, + "sources": [ + "moonshotai/Kimi-K3/block_sparse_moe.shared_experts.gate_proj+block_sparse_moe.shared_experts.up_proj", + "moonshotai/Kimi-K3/self_attn.g_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 1, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 7168, + "n": 16768, + "out": "bf16" + }, + "sources": [ + "moonshotai/Kimi-K3/block_sparse_moe.shared_experts.gate_proj+block_sparse_moe.shared_experts.up_proj+block_sparse_moe.gate+block_sparse_moe.routed_expert_down_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 2, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 7168, + "n": 16768, + "out": "bf16" + }, + "sources": [ + "moonshotai/Kimi-K3/block_sparse_moe.shared_experts.gate_proj+block_sparse_moe.shared_experts.up_proj+block_sparse_moe.gate+block_sparse_moe.routed_expert_down_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 3, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 7168, + "n": 16768, + "out": "bf16" + }, + "sources": [ + "moonshotai/Kimi-K3/block_sparse_moe.shared_experts.gate_proj+block_sparse_moe.shared_experts.up_proj+block_sparse_moe.gate+block_sparse_moe.routed_expert_down_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 4, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 7168, + "n": 16768, + "out": "bf16" + }, + "sources": [ + "moonshotai/Kimi-K3/block_sparse_moe.shared_experts.gate_proj+block_sparse_moe.shared_experts.up_proj+block_sparse_moe.gate+block_sparse_moe.routed_expert_down_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 6, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 7168, + "n": 16768, + "out": "bf16" + }, + "sources": [ + "moonshotai/Kimi-K3/block_sparse_moe.shared_experts.gate_proj+block_sparse_moe.shared_experts.up_proj+block_sparse_moe.gate+block_sparse_moe.routed_expert_down_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 8, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 7168, + "n": 16768, + "out": "bf16" + }, + "sources": [ + "moonshotai/Kimi-K3/block_sparse_moe.shared_experts.gate_proj+block_sparse_moe.shared_experts.up_proj+block_sparse_moe.gate+block_sparse_moe.routed_expert_down_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 12, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 7168, + "n": 16768, + "out": "bf16" + }, + "sources": [ + "moonshotai/Kimi-K3/block_sparse_moe.shared_experts.gate_proj+block_sparse_moe.shared_experts.up_proj+block_sparse_moe.gate+block_sparse_moe.routed_expert_down_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 16, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 7168, + "n": 16768, + "out": "bf16" + }, + "sources": [ + "moonshotai/Kimi-K3/block_sparse_moe.shared_experts.gate_proj+block_sparse_moe.shared_experts.up_proj+block_sparse_moe.gate+block_sparse_moe.routed_expert_down_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 24, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 7168, + "n": 16768, + "out": "bf16" + }, + "sources": [ + "moonshotai/Kimi-K3/block_sparse_moe.shared_experts.gate_proj+block_sparse_moe.shared_experts.up_proj+block_sparse_moe.gate+block_sparse_moe.routed_expert_down_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 32, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 7168, + "n": 16768, + "out": "bf16" + }, + "sources": [ + "moonshotai/Kimi-K3/block_sparse_moe.shared_experts.gate_proj+block_sparse_moe.shared_experts.up_proj+block_sparse_moe.gate+block_sparse_moe.routed_expert_down_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 48, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 7168, + "n": 16768, + "out": "bf16" + }, + "sources": [ + "moonshotai/Kimi-K3/block_sparse_moe.shared_experts.gate_proj+block_sparse_moe.shared_experts.up_proj+block_sparse_moe.gate+block_sparse_moe.routed_expert_down_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 64, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 7168, + "n": 16768, + "out": "bf16" + }, + "sources": [ + "moonshotai/Kimi-K3/block_sparse_moe.shared_experts.gate_proj+block_sparse_moe.shared_experts.up_proj+block_sparse_moe.gate+block_sparse_moe.routed_expert_down_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 96, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 7168, + "n": 16768, + "out": "bf16" + }, + "sources": [ + "moonshotai/Kimi-K3/block_sparse_moe.shared_experts.gate_proj+block_sparse_moe.shared_experts.up_proj+block_sparse_moe.gate+block_sparse_moe.routed_expert_down_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 128, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 7168, + "n": 16768, + "out": "bf16" + }, + "sources": [ + "moonshotai/Kimi-K3/block_sparse_moe.shared_experts.gate_proj+block_sparse_moe.shared_experts.up_proj+block_sparse_moe.gate+block_sparse_moe.routed_expert_down_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 192, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 7168, + "n": 16768, + "out": "bf16" + }, + "sources": [ + "moonshotai/Kimi-K3/block_sparse_moe.shared_experts.gate_proj+block_sparse_moe.shared_experts.up_proj+block_sparse_moe.gate+block_sparse_moe.routed_expert_down_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 256, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 7168, + "n": 16768, + "out": "bf16" + }, + "sources": [ + "moonshotai/Kimi-K3/block_sparse_moe.shared_experts.gate_proj+block_sparse_moe.shared_experts.up_proj+block_sparse_moe.gate+block_sparse_moe.routed_expert_down_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 384, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 7168, + "n": 16768, + "out": "bf16" + }, + "sources": [ + "moonshotai/Kimi-K3/block_sparse_moe.shared_experts.gate_proj+block_sparse_moe.shared_experts.up_proj+block_sparse_moe.gate+block_sparse_moe.routed_expert_down_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 512, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 7168, + "n": 16768, + "out": "bf16" + }, + "sources": [ + "moonshotai/Kimi-K3/block_sparse_moe.shared_experts.gate_proj+block_sparse_moe.shared_experts.up_proj+block_sparse_moe.gate+block_sparse_moe.routed_expert_down_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 768, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 7168, + "n": 16768, + "out": "bf16" + }, + "sources": [ + "moonshotai/Kimi-K3/block_sparse_moe.shared_experts.gate_proj+block_sparse_moe.shared_experts.up_proj+block_sparse_moe.gate+block_sparse_moe.routed_expert_down_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 1024, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 7168, + "n": 16768, + "out": "bf16" + }, + "sources": [ + "moonshotai/Kimi-K3/block_sparse_moe.shared_experts.gate_proj+block_sparse_moe.shared_experts.up_proj+block_sparse_moe.gate+block_sparse_moe.routed_expert_down_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 1536, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 7168, + "n": 16768, + "out": "bf16" + }, + "sources": [ + "moonshotai/Kimi-K3/block_sparse_moe.shared_experts.gate_proj+block_sparse_moe.shared_experts.up_proj+block_sparse_moe.gate+block_sparse_moe.routed_expert_down_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 2048, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 7168, + "n": 16768, + "out": "bf16" + }, + "sources": [ + "moonshotai/Kimi-K3/block_sparse_moe.shared_experts.gate_proj+block_sparse_moe.shared_experts.up_proj+block_sparse_moe.gate+block_sparse_moe.routed_expert_down_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 3072, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 7168, + "n": 16768, + "out": "bf16" + }, + "sources": [ + "moonshotai/Kimi-K3/block_sparse_moe.shared_experts.gate_proj+block_sparse_moe.shared_experts.up_proj+block_sparse_moe.gate+block_sparse_moe.routed_expert_down_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 4096, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 7168, + "n": 16768, + "out": "bf16" + }, + "sources": [ + "moonshotai/Kimi-K3/block_sparse_moe.shared_experts.gate_proj+block_sparse_moe.shared_experts.up_proj+block_sparse_moe.gate+block_sparse_moe.routed_expert_down_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 6144, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 7168, + "n": 16768, + "out": "bf16" + }, + "sources": [ + "moonshotai/Kimi-K3/block_sparse_moe.shared_experts.gate_proj+block_sparse_moe.shared_experts.up_proj+block_sparse_moe.gate+block_sparse_moe.routed_expert_down_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 8192, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 7168, + "n": 16768, + "out": "bf16" + }, + "sources": [ + "moonshotai/Kimi-K3/block_sparse_moe.shared_experts.gate_proj+block_sparse_moe.shared_experts.up_proj+block_sparse_moe.gate+block_sparse_moe.routed_expert_down_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 12288, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 7168, + "n": 16768, + "out": "bf16" + }, + "sources": [ + "moonshotai/Kimi-K3/block_sparse_moe.shared_experts.gate_proj+block_sparse_moe.shared_experts.up_proj+block_sparse_moe.gate+block_sparse_moe.routed_expert_down_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 16384, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 7168, + "n": 16768, + "out": "bf16" + }, + "sources": [ + "moonshotai/Kimi-K3/block_sparse_moe.shared_experts.gate_proj+block_sparse_moe.shared_experts.up_proj+block_sparse_moe.gate+block_sparse_moe.routed_expert_down_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 24576, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 7168, + "n": 16768, + "out": "bf16" + }, + "sources": [ + "moonshotai/Kimi-K3/block_sparse_moe.shared_experts.gate_proj+block_sparse_moe.shared_experts.up_proj+block_sparse_moe.gate+block_sparse_moe.routed_expert_down_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 32768, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 7168, + "n": 16768, + "out": "bf16" + }, + "sources": [ + "moonshotai/Kimi-K3/block_sparse_moe.shared_experts.gate_proj+block_sparse_moe.shared_experts.up_proj+block_sparse_moe.gate+block_sparse_moe.routed_expert_down_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 49152, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 7168, + "n": 16768, + "out": "bf16" + }, + "sources": [ + "moonshotai/Kimi-K3/block_sparse_moe.shared_experts.gate_proj+block_sparse_moe.shared_experts.up_proj+block_sparse_moe.gate+block_sparse_moe.routed_expert_down_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 65536, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 7168, + "n": 16768, + "out": "bf16" + }, + "sources": [ + "moonshotai/Kimi-K3/block_sparse_moe.shared_experts.gate_proj+block_sparse_moe.shared_experts.up_proj+block_sparse_moe.gate+block_sparse_moe.routed_expert_down_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 1, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 7168, + "n": 16768, + "out": "fp32" + }, + "sources": [ + "moonshotai/Kimi-K3/block_sparse_moe.shared_experts.gate_proj+block_sparse_moe.shared_experts.up_proj+block_sparse_moe.gate+block_sparse_moe.routed_expert_down_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 2, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 7168, + "n": 16768, + "out": "fp32" + }, + "sources": [ + "moonshotai/Kimi-K3/block_sparse_moe.shared_experts.gate_proj+block_sparse_moe.shared_experts.up_proj+block_sparse_moe.gate+block_sparse_moe.routed_expert_down_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 3, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 7168, + "n": 16768, + "out": "fp32" + }, + "sources": [ + "moonshotai/Kimi-K3/block_sparse_moe.shared_experts.gate_proj+block_sparse_moe.shared_experts.up_proj+block_sparse_moe.gate+block_sparse_moe.routed_expert_down_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 4, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 7168, + "n": 16768, + "out": "fp32" + }, + "sources": [ + "moonshotai/Kimi-K3/block_sparse_moe.shared_experts.gate_proj+block_sparse_moe.shared_experts.up_proj+block_sparse_moe.gate+block_sparse_moe.routed_expert_down_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 6, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 7168, + "n": 16768, + "out": "fp32" + }, + "sources": [ + "moonshotai/Kimi-K3/block_sparse_moe.shared_experts.gate_proj+block_sparse_moe.shared_experts.up_proj+block_sparse_moe.gate+block_sparse_moe.routed_expert_down_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 8, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 7168, + "n": 16768, + "out": "fp32" + }, + "sources": [ + "moonshotai/Kimi-K3/block_sparse_moe.shared_experts.gate_proj+block_sparse_moe.shared_experts.up_proj+block_sparse_moe.gate+block_sparse_moe.routed_expert_down_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 12, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 7168, + "n": 16768, + "out": "fp32" + }, + "sources": [ + "moonshotai/Kimi-K3/block_sparse_moe.shared_experts.gate_proj+block_sparse_moe.shared_experts.up_proj+block_sparse_moe.gate+block_sparse_moe.routed_expert_down_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 16, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 7168, + "n": 16768, + "out": "fp32" + }, + "sources": [ + "moonshotai/Kimi-K3/block_sparse_moe.shared_experts.gate_proj+block_sparse_moe.shared_experts.up_proj+block_sparse_moe.gate+block_sparse_moe.routed_expert_down_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 24, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 7168, + "n": 16768, + "out": "fp32" + }, + "sources": [ + "moonshotai/Kimi-K3/block_sparse_moe.shared_experts.gate_proj+block_sparse_moe.shared_experts.up_proj+block_sparse_moe.gate+block_sparse_moe.routed_expert_down_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 32, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 7168, + "n": 16768, + "out": "fp32" + }, + "sources": [ + "moonshotai/Kimi-K3/block_sparse_moe.shared_experts.gate_proj+block_sparse_moe.shared_experts.up_proj+block_sparse_moe.gate+block_sparse_moe.routed_expert_down_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 48, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 7168, + "n": 16768, + "out": "fp32" + }, + "sources": [ + "moonshotai/Kimi-K3/block_sparse_moe.shared_experts.gate_proj+block_sparse_moe.shared_experts.up_proj+block_sparse_moe.gate+block_sparse_moe.routed_expert_down_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 64, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 7168, + "n": 16768, + "out": "fp32" + }, + "sources": [ + "moonshotai/Kimi-K3/block_sparse_moe.shared_experts.gate_proj+block_sparse_moe.shared_experts.up_proj+block_sparse_moe.gate+block_sparse_moe.routed_expert_down_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 96, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 7168, + "n": 16768, + "out": "fp32" + }, + "sources": [ + "moonshotai/Kimi-K3/block_sparse_moe.shared_experts.gate_proj+block_sparse_moe.shared_experts.up_proj+block_sparse_moe.gate+block_sparse_moe.routed_expert_down_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 128, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 7168, + "n": 16768, + "out": "fp32" + }, + "sources": [ + "moonshotai/Kimi-K3/block_sparse_moe.shared_experts.gate_proj+block_sparse_moe.shared_experts.up_proj+block_sparse_moe.gate+block_sparse_moe.routed_expert_down_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 192, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 7168, + "n": 16768, + "out": "fp32" + }, + "sources": [ + "moonshotai/Kimi-K3/block_sparse_moe.shared_experts.gate_proj+block_sparse_moe.shared_experts.up_proj+block_sparse_moe.gate+block_sparse_moe.routed_expert_down_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 256, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 7168, + "n": 16768, + "out": "fp32" + }, + "sources": [ + "moonshotai/Kimi-K3/block_sparse_moe.shared_experts.gate_proj+block_sparse_moe.shared_experts.up_proj+block_sparse_moe.gate+block_sparse_moe.routed_expert_down_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 384, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 7168, + "n": 16768, + "out": "fp32" + }, + "sources": [ + "moonshotai/Kimi-K3/block_sparse_moe.shared_experts.gate_proj+block_sparse_moe.shared_experts.up_proj+block_sparse_moe.gate+block_sparse_moe.routed_expert_down_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 512, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 7168, + "n": 16768, + "out": "fp32" + }, + "sources": [ + "moonshotai/Kimi-K3/block_sparse_moe.shared_experts.gate_proj+block_sparse_moe.shared_experts.up_proj+block_sparse_moe.gate+block_sparse_moe.routed_expert_down_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 768, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 7168, + "n": 16768, + "out": "fp32" + }, + "sources": [ + "moonshotai/Kimi-K3/block_sparse_moe.shared_experts.gate_proj+block_sparse_moe.shared_experts.up_proj+block_sparse_moe.gate+block_sparse_moe.routed_expert_down_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 1024, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 7168, + "n": 16768, + "out": "fp32" + }, + "sources": [ + "moonshotai/Kimi-K3/block_sparse_moe.shared_experts.gate_proj+block_sparse_moe.shared_experts.up_proj+block_sparse_moe.gate+block_sparse_moe.routed_expert_down_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 1536, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 7168, + "n": 16768, + "out": "fp32" + }, + "sources": [ + "moonshotai/Kimi-K3/block_sparse_moe.shared_experts.gate_proj+block_sparse_moe.shared_experts.up_proj+block_sparse_moe.gate+block_sparse_moe.routed_expert_down_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 2048, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 7168, + "n": 16768, + "out": "fp32" + }, + "sources": [ + "moonshotai/Kimi-K3/block_sparse_moe.shared_experts.gate_proj+block_sparse_moe.shared_experts.up_proj+block_sparse_moe.gate+block_sparse_moe.routed_expert_down_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 3072, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 7168, + "n": 16768, + "out": "fp32" + }, + "sources": [ + "moonshotai/Kimi-K3/block_sparse_moe.shared_experts.gate_proj+block_sparse_moe.shared_experts.up_proj+block_sparse_moe.gate+block_sparse_moe.routed_expert_down_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 4096, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 7168, + "n": 16768, + "out": "fp32" + }, + "sources": [ + "moonshotai/Kimi-K3/block_sparse_moe.shared_experts.gate_proj+block_sparse_moe.shared_experts.up_proj+block_sparse_moe.gate+block_sparse_moe.routed_expert_down_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 6144, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 7168, + "n": 16768, + "out": "fp32" + }, + "sources": [ + "moonshotai/Kimi-K3/block_sparse_moe.shared_experts.gate_proj+block_sparse_moe.shared_experts.up_proj+block_sparse_moe.gate+block_sparse_moe.routed_expert_down_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 8192, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 7168, + "n": 16768, + "out": "fp32" + }, + "sources": [ + "moonshotai/Kimi-K3/block_sparse_moe.shared_experts.gate_proj+block_sparse_moe.shared_experts.up_proj+block_sparse_moe.gate+block_sparse_moe.routed_expert_down_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 12288, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 7168, + "n": 16768, + "out": "fp32" + }, + "sources": [ + "moonshotai/Kimi-K3/block_sparse_moe.shared_experts.gate_proj+block_sparse_moe.shared_experts.up_proj+block_sparse_moe.gate+block_sparse_moe.routed_expert_down_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 16384, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 7168, + "n": 16768, + "out": "fp32" + }, + "sources": [ + "moonshotai/Kimi-K3/block_sparse_moe.shared_experts.gate_proj+block_sparse_moe.shared_experts.up_proj+block_sparse_moe.gate+block_sparse_moe.routed_expert_down_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 24576, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 7168, + "n": 16768, + "out": "fp32" + }, + "sources": [ + "moonshotai/Kimi-K3/block_sparse_moe.shared_experts.gate_proj+block_sparse_moe.shared_experts.up_proj+block_sparse_moe.gate+block_sparse_moe.routed_expert_down_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 32768, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 7168, + "n": 16768, + "out": "fp32" + }, + "sources": [ + "moonshotai/Kimi-K3/block_sparse_moe.shared_experts.gate_proj+block_sparse_moe.shared_experts.up_proj+block_sparse_moe.gate+block_sparse_moe.routed_expert_down_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 49152, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 7168, + "n": 16768, + "out": "fp32" + }, + "sources": [ + "moonshotai/Kimi-K3/block_sparse_moe.shared_experts.gate_proj+block_sparse_moe.shared_experts.up_proj+block_sparse_moe.gate+block_sparse_moe.routed_expert_down_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 65536, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 7168, + "n": 16768, + "out": "fp32" + }, + "sources": [ + "moonshotai/Kimi-K3/block_sparse_moe.shared_experts.gate_proj+block_sparse_moe.shared_experts.up_proj+block_sparse_moe.gate+block_sparse_moe.routed_expert_down_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 1, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 7168, + "n": 163840, + "out": "bf16" + }, + "sources": [ + "moonshotai/Kimi-K3/lm_head" + ] + }, + { + "type": "gemm", + "args": { + "m": 2, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 7168, + "n": 163840, + "out": "bf16" + }, + "sources": [ + "moonshotai/Kimi-K3/lm_head" + ] + }, + { + "type": "gemm", + "args": { + "m": 3, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 7168, + "n": 163840, + "out": "bf16" + }, + "sources": [ + "moonshotai/Kimi-K3/lm_head" + ] + }, + { + "type": "gemm", + "args": { + "m": 4, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 7168, + "n": 163840, + "out": "bf16" + }, + "sources": [ + "moonshotai/Kimi-K3/lm_head" + ] + }, + { + "type": "gemm", + "args": { + "m": 6, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 7168, + "n": 163840, + "out": "bf16" + }, + "sources": [ + "moonshotai/Kimi-K3/lm_head" + ] + }, + { + "type": "gemm", + "args": { + "m": 8, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 7168, + "n": 163840, + "out": "bf16" + }, + "sources": [ + "moonshotai/Kimi-K3/lm_head" + ] + }, + { + "type": "gemm", + "args": { + "m": 12, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 7168, + "n": 163840, + "out": "bf16" + }, + "sources": [ + "moonshotai/Kimi-K3/lm_head" + ] + }, + { + "type": "gemm", + "args": { + "m": 16, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 7168, + "n": 163840, + "out": "bf16" + }, + "sources": [ + "moonshotai/Kimi-K3/lm_head" + ] + }, + { + "type": "gemm", + "args": { + "m": 24, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 7168, + "n": 163840, + "out": "bf16" + }, + "sources": [ + "moonshotai/Kimi-K3/lm_head" + ] + }, + { + "type": "gemm", + "args": { + "m": 32, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 7168, + "n": 163840, + "out": "bf16" + }, + "sources": [ + "moonshotai/Kimi-K3/lm_head" + ] + }, + { + "type": "gemm", + "args": { + "m": 48, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 7168, + "n": 163840, + "out": "bf16" + }, + "sources": [ + "moonshotai/Kimi-K3/lm_head" + ] + }, + { + "type": "gemm", + "args": { + "m": 64, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 7168, + "n": 163840, + "out": "bf16" + }, + "sources": [ + "moonshotai/Kimi-K3/lm_head" + ] + }, + { + "type": "gemm", + "args": { + "m": 96, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 7168, + "n": 163840, + "out": "bf16" + }, + "sources": [ + "moonshotai/Kimi-K3/lm_head" + ] + }, + { + "type": "gemm", + "args": { + "m": 128, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 7168, + "n": 163840, + "out": "bf16" + }, + "sources": [ + "moonshotai/Kimi-K3/lm_head" + ] + }, + { + "type": "gemm", + "args": { + "m": 192, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 7168, + "n": 163840, + "out": "bf16" + }, + "sources": [ + "moonshotai/Kimi-K3/lm_head" + ] + }, + { + "type": "gemm", + "args": { + "m": 256, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 7168, + "n": 163840, + "out": "bf16" + }, + "sources": [ + "moonshotai/Kimi-K3/lm_head" + ] + }, + { + "type": "gemm", + "args": { + "m": 384, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 7168, + "n": 163840, + "out": "bf16" + }, + "sources": [ + "moonshotai/Kimi-K3/lm_head" + ] + }, + { + "type": "gemm", + "args": { + "m": 512, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 7168, + "n": 163840, + "out": "bf16" + }, + "sources": [ + "moonshotai/Kimi-K3/lm_head" + ] + }, + { + "type": "gemm", + "args": { + "m": 768, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 7168, + "n": 163840, + "out": "bf16" + }, + "sources": [ + "moonshotai/Kimi-K3/lm_head" + ] + }, + { + "type": "gemm", + "args": { + "m": 1024, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 7168, + "n": 163840, + "out": "bf16" + }, + "sources": [ + "moonshotai/Kimi-K3/lm_head" + ] + }, + { + "type": "gemm", + "args": { + "m": 1536, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 7168, + "n": 163840, + "out": "bf16" + }, + "sources": [ + "moonshotai/Kimi-K3/lm_head" + ] + }, + { + "type": "gemm", + "args": { + "m": 2048, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 7168, + "n": 163840, + "out": "bf16" + }, + "sources": [ + "moonshotai/Kimi-K3/lm_head" + ] + }, + { + "type": "gemm", + "args": { + "m": 3072, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 7168, + "n": 163840, + "out": "bf16" + }, + "sources": [ + "moonshotai/Kimi-K3/lm_head" + ] + }, + { + "type": "gemm", + "args": { + "m": 4096, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 7168, + "n": 163840, + "out": "bf16" + }, + "sources": [ + "moonshotai/Kimi-K3/lm_head" + ] + }, + { + "type": "gemm", + "args": { + "m": 6144, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 7168, + "n": 163840, + "out": "bf16" + }, + "sources": [ + "moonshotai/Kimi-K3/lm_head" + ] + }, + { + "type": "gemm", + "args": { + "m": 8192, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 7168, + "n": 163840, + "out": "bf16" + }, + "sources": [ + "moonshotai/Kimi-K3/lm_head" + ] + }, + { + "type": "gemm", + "args": { + "m": 12288, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 7168, + "n": 163840, + "out": "bf16" + }, + "sources": [ + "moonshotai/Kimi-K3/lm_head" + ] + }, + { + "type": "gemm", + "args": { + "m": 16384, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 7168, + "n": 163840, + "out": "bf16" + }, + "sources": [ + "moonshotai/Kimi-K3/lm_head" + ] + }, + { + "type": "gemm", + "args": { + "m": 24576, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 7168, + "n": 163840, + "out": "bf16" + }, + "sources": [ + "moonshotai/Kimi-K3/lm_head" + ] + }, + { + "type": "gemm", + "args": { + "m": 32768, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 7168, + "n": 163840, + "out": "bf16" + }, + "sources": [ + "moonshotai/Kimi-K3/lm_head" + ] + }, + { + "type": "gemm", + "args": { + "m": 49152, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 7168, + "n": 163840, + "out": "bf16" + }, + "sources": [ + "moonshotai/Kimi-K3/lm_head" + ] + }, + { + "type": "gemm", + "args": { + "m": 65536, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 7168, + "n": 163840, + "out": "bf16" + }, + "sources": [ + "moonshotai/Kimi-K3/lm_head" + ] + }, + { + "type": "gemm", + "args": { + "m": 1, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 33792, + "n": 7168, + "out": "bf16" + }, + "sources": [ + "moonshotai/Kimi-K3/mlp.down_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 2, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 33792, + "n": 7168, + "out": "bf16" + }, + "sources": [ + "moonshotai/Kimi-K3/mlp.down_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 3, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 33792, + "n": 7168, + "out": "bf16" + }, + "sources": [ + "moonshotai/Kimi-K3/mlp.down_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 4, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 33792, + "n": 7168, + "out": "bf16" + }, + "sources": [ + "moonshotai/Kimi-K3/mlp.down_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 6, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 33792, + "n": 7168, + "out": "bf16" + }, + "sources": [ + "moonshotai/Kimi-K3/mlp.down_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 8, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 33792, + "n": 7168, + "out": "bf16" + }, + "sources": [ + "moonshotai/Kimi-K3/mlp.down_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 12, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 33792, + "n": 7168, + "out": "bf16" + }, + "sources": [ + "moonshotai/Kimi-K3/mlp.down_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 16, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 33792, + "n": 7168, + "out": "bf16" + }, + "sources": [ + "moonshotai/Kimi-K3/mlp.down_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 24, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 33792, + "n": 7168, + "out": "bf16" + }, + "sources": [ + "moonshotai/Kimi-K3/mlp.down_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 32, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 33792, + "n": 7168, + "out": "bf16" + }, + "sources": [ + "moonshotai/Kimi-K3/mlp.down_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 48, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 33792, + "n": 7168, + "out": "bf16" + }, + "sources": [ + "moonshotai/Kimi-K3/mlp.down_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 64, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 33792, + "n": 7168, + "out": "bf16" + }, + "sources": [ + "moonshotai/Kimi-K3/mlp.down_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 96, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 33792, + "n": 7168, + "out": "bf16" + }, + "sources": [ + "moonshotai/Kimi-K3/mlp.down_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 128, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 33792, + "n": 7168, + "out": "bf16" + }, + "sources": [ + "moonshotai/Kimi-K3/mlp.down_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 192, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 33792, + "n": 7168, + "out": "bf16" + }, + "sources": [ + "moonshotai/Kimi-K3/mlp.down_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 256, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 33792, + "n": 7168, + "out": "bf16" + }, + "sources": [ + "moonshotai/Kimi-K3/mlp.down_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 384, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 33792, + "n": 7168, + "out": "bf16" + }, + "sources": [ + "moonshotai/Kimi-K3/mlp.down_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 512, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 33792, + "n": 7168, + "out": "bf16" + }, + "sources": [ + "moonshotai/Kimi-K3/mlp.down_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 768, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 33792, + "n": 7168, + "out": "bf16" + }, + "sources": [ + "moonshotai/Kimi-K3/mlp.down_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 1024, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 33792, + "n": 7168, + "out": "bf16" + }, + "sources": [ + "moonshotai/Kimi-K3/mlp.down_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 1536, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 33792, + "n": 7168, + "out": "bf16" + }, + "sources": [ + "moonshotai/Kimi-K3/mlp.down_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 2048, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 33792, + "n": 7168, + "out": "bf16" + }, + "sources": [ + "moonshotai/Kimi-K3/mlp.down_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 3072, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 33792, + "n": 7168, + "out": "bf16" + }, + "sources": [ + "moonshotai/Kimi-K3/mlp.down_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 4096, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 33792, + "n": 7168, + "out": "bf16" + }, + "sources": [ + "moonshotai/Kimi-K3/mlp.down_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 6144, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 33792, + "n": 7168, + "out": "bf16" + }, + "sources": [ + "moonshotai/Kimi-K3/mlp.down_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 8192, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 33792, + "n": 7168, + "out": "bf16" + }, + "sources": [ + "moonshotai/Kimi-K3/mlp.down_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 12288, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 33792, + "n": 7168, + "out": "bf16" + }, + "sources": [ + "moonshotai/Kimi-K3/mlp.down_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 16384, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 33792, + "n": 7168, + "out": "bf16" + }, + "sources": [ + "moonshotai/Kimi-K3/mlp.down_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 24576, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 33792, + "n": 7168, + "out": "bf16" + }, + "sources": [ + "moonshotai/Kimi-K3/mlp.down_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 32768, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 33792, + "n": 7168, + "out": "bf16" + }, + "sources": [ + "moonshotai/Kimi-K3/mlp.down_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 49152, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 33792, + "n": 7168, + "out": "bf16" + }, + "sources": [ + "moonshotai/Kimi-K3/mlp.down_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 65536, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 33792, + "n": 7168, + "out": "bf16" + }, + "sources": [ + "moonshotai/Kimi-K3/mlp.down_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 1, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 7168, + "n": 67584, + "out": "bf16" + }, + "sources": [ + "moonshotai/Kimi-K3/mlp.gate_proj+mlp.up_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 2, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 7168, + "n": 67584, + "out": "bf16" + }, + "sources": [ + "moonshotai/Kimi-K3/mlp.gate_proj+mlp.up_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 3, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 7168, + "n": 67584, + "out": "bf16" + }, + "sources": [ + "moonshotai/Kimi-K3/mlp.gate_proj+mlp.up_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 4, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 7168, + "n": 67584, + "out": "bf16" + }, + "sources": [ + "moonshotai/Kimi-K3/mlp.gate_proj+mlp.up_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 6, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 7168, + "n": 67584, + "out": "bf16" + }, + "sources": [ + "moonshotai/Kimi-K3/mlp.gate_proj+mlp.up_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 8, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 7168, + "n": 67584, + "out": "bf16" + }, + "sources": [ + "moonshotai/Kimi-K3/mlp.gate_proj+mlp.up_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 12, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 7168, + "n": 67584, + "out": "bf16" + }, + "sources": [ + "moonshotai/Kimi-K3/mlp.gate_proj+mlp.up_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 16, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 7168, + "n": 67584, + "out": "bf16" + }, + "sources": [ + "moonshotai/Kimi-K3/mlp.gate_proj+mlp.up_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 24, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 7168, + "n": 67584, + "out": "bf16" + }, + "sources": [ + "moonshotai/Kimi-K3/mlp.gate_proj+mlp.up_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 32, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 7168, + "n": 67584, + "out": "bf16" + }, + "sources": [ + "moonshotai/Kimi-K3/mlp.gate_proj+mlp.up_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 48, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 7168, + "n": 67584, + "out": "bf16" + }, + "sources": [ + "moonshotai/Kimi-K3/mlp.gate_proj+mlp.up_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 64, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 7168, + "n": 67584, + "out": "bf16" + }, + "sources": [ + "moonshotai/Kimi-K3/mlp.gate_proj+mlp.up_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 96, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 7168, + "n": 67584, + "out": "bf16" + }, + "sources": [ + "moonshotai/Kimi-K3/mlp.gate_proj+mlp.up_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 128, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 7168, + "n": 67584, + "out": "bf16" + }, + "sources": [ + "moonshotai/Kimi-K3/mlp.gate_proj+mlp.up_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 192, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 7168, + "n": 67584, + "out": "bf16" + }, + "sources": [ + "moonshotai/Kimi-K3/mlp.gate_proj+mlp.up_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 256, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 7168, + "n": 67584, + "out": "bf16" + }, + "sources": [ + "moonshotai/Kimi-K3/mlp.gate_proj+mlp.up_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 384, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 7168, + "n": 67584, + "out": "bf16" + }, + "sources": [ + "moonshotai/Kimi-K3/mlp.gate_proj+mlp.up_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 512, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 7168, + "n": 67584, + "out": "bf16" + }, + "sources": [ + "moonshotai/Kimi-K3/mlp.gate_proj+mlp.up_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 768, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 7168, + "n": 67584, + "out": "bf16" + }, + "sources": [ + "moonshotai/Kimi-K3/mlp.gate_proj+mlp.up_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 1024, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 7168, + "n": 67584, + "out": "bf16" + }, + "sources": [ + "moonshotai/Kimi-K3/mlp.gate_proj+mlp.up_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 1536, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 7168, + "n": 67584, + "out": "bf16" + }, + "sources": [ + "moonshotai/Kimi-K3/mlp.gate_proj+mlp.up_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 2048, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 7168, + "n": 67584, + "out": "bf16" + }, + "sources": [ + "moonshotai/Kimi-K3/mlp.gate_proj+mlp.up_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 3072, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 7168, + "n": 67584, + "out": "bf16" + }, + "sources": [ + "moonshotai/Kimi-K3/mlp.gate_proj+mlp.up_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 4096, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 7168, + "n": 67584, + "out": "bf16" + }, + "sources": [ + "moonshotai/Kimi-K3/mlp.gate_proj+mlp.up_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 6144, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 7168, + "n": 67584, + "out": "bf16" + }, + "sources": [ + "moonshotai/Kimi-K3/mlp.gate_proj+mlp.up_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 8192, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 7168, + "n": 67584, + "out": "bf16" + }, + "sources": [ + "moonshotai/Kimi-K3/mlp.gate_proj+mlp.up_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 12288, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 7168, + "n": 67584, + "out": "bf16" + }, + "sources": [ + "moonshotai/Kimi-K3/mlp.gate_proj+mlp.up_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 16384, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 7168, + "n": 67584, + "out": "bf16" + }, + "sources": [ + "moonshotai/Kimi-K3/mlp.gate_proj+mlp.up_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 24576, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 7168, + "n": 67584, + "out": "bf16" + }, + "sources": [ + "moonshotai/Kimi-K3/mlp.gate_proj+mlp.up_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 32768, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 7168, + "n": 67584, + "out": "bf16" + }, + "sources": [ + "moonshotai/Kimi-K3/mlp.gate_proj+mlp.up_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 49152, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 7168, + "n": 67584, + "out": "bf16" + }, + "sources": [ + "moonshotai/Kimi-K3/mlp.gate_proj+mlp.up_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 65536, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 7168, + "n": 67584, + "out": "bf16" + }, + "sources": [ + "moonshotai/Kimi-K3/mlp.gate_proj+mlp.up_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 1, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 7168, + "n": 224, + "out": "bf16" + }, + "sources": [ + "moonshotai/Kimi-K3/self_attn.f_a_proj+self_attn.b_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 2, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 7168, + "n": 224, + "out": "bf16" + }, + "sources": [ + "moonshotai/Kimi-K3/self_attn.f_a_proj+self_attn.b_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 3, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 7168, + "n": 224, + "out": "bf16" + }, + "sources": [ + "moonshotai/Kimi-K3/self_attn.f_a_proj+self_attn.b_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 4, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 7168, + "n": 224, + "out": "bf16" + }, + "sources": [ + "moonshotai/Kimi-K3/self_attn.f_a_proj+self_attn.b_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 6, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 7168, + "n": 224, + "out": "bf16" + }, + "sources": [ + "moonshotai/Kimi-K3/self_attn.f_a_proj+self_attn.b_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 8, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 7168, + "n": 224, + "out": "bf16" + }, + "sources": [ + "moonshotai/Kimi-K3/self_attn.f_a_proj+self_attn.b_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 12, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 7168, + "n": 224, + "out": "bf16" + }, + "sources": [ + "moonshotai/Kimi-K3/self_attn.f_a_proj+self_attn.b_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 16, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 7168, + "n": 224, + "out": "bf16" + }, + "sources": [ + "moonshotai/Kimi-K3/self_attn.f_a_proj+self_attn.b_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 24, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 7168, + "n": 224, + "out": "bf16" + }, + "sources": [ + "moonshotai/Kimi-K3/self_attn.f_a_proj+self_attn.b_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 32, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 7168, + "n": 224, + "out": "bf16" + }, + "sources": [ + "moonshotai/Kimi-K3/self_attn.f_a_proj+self_attn.b_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 48, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 7168, + "n": 224, + "out": "bf16" + }, + "sources": [ + "moonshotai/Kimi-K3/self_attn.f_a_proj+self_attn.b_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 64, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 7168, + "n": 224, + "out": "bf16" + }, + "sources": [ + "moonshotai/Kimi-K3/self_attn.f_a_proj+self_attn.b_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 96, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 7168, + "n": 224, + "out": "bf16" + }, + "sources": [ + "moonshotai/Kimi-K3/self_attn.f_a_proj+self_attn.b_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 128, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 7168, + "n": 224, + "out": "bf16" + }, + "sources": [ + "moonshotai/Kimi-K3/self_attn.f_a_proj+self_attn.b_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 192, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 7168, + "n": 224, + "out": "bf16" + }, + "sources": [ + "moonshotai/Kimi-K3/self_attn.f_a_proj+self_attn.b_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 256, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 7168, + "n": 224, + "out": "bf16" + }, + "sources": [ + "moonshotai/Kimi-K3/self_attn.f_a_proj+self_attn.b_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 384, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 7168, + "n": 224, + "out": "bf16" + }, + "sources": [ + "moonshotai/Kimi-K3/self_attn.f_a_proj+self_attn.b_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 512, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 7168, + "n": 224, + "out": "bf16" + }, + "sources": [ + "moonshotai/Kimi-K3/self_attn.f_a_proj+self_attn.b_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 768, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 7168, + "n": 224, + "out": "bf16" + }, + "sources": [ + "moonshotai/Kimi-K3/self_attn.f_a_proj+self_attn.b_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 1024, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 7168, + "n": 224, + "out": "bf16" + }, + "sources": [ + "moonshotai/Kimi-K3/self_attn.f_a_proj+self_attn.b_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 1536, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 7168, + "n": 224, + "out": "bf16" + }, + "sources": [ + "moonshotai/Kimi-K3/self_attn.f_a_proj+self_attn.b_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 2048, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 7168, + "n": 224, + "out": "bf16" + }, + "sources": [ + "moonshotai/Kimi-K3/self_attn.f_a_proj+self_attn.b_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 3072, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 7168, + "n": 224, + "out": "bf16" + }, + "sources": [ + "moonshotai/Kimi-K3/self_attn.f_a_proj+self_attn.b_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 4096, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 7168, + "n": 224, + "out": "bf16" + }, + "sources": [ + "moonshotai/Kimi-K3/self_attn.f_a_proj+self_attn.b_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 6144, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 7168, + "n": 224, + "out": "bf16" + }, + "sources": [ + "moonshotai/Kimi-K3/self_attn.f_a_proj+self_attn.b_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 8192, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 7168, + "n": 224, + "out": "bf16" + }, + "sources": [ + "moonshotai/Kimi-K3/self_attn.f_a_proj+self_attn.b_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 12288, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 7168, + "n": 224, + "out": "bf16" + }, + "sources": [ + "moonshotai/Kimi-K3/self_attn.f_a_proj+self_attn.b_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 16384, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 7168, + "n": 224, + "out": "bf16" + }, + "sources": [ + "moonshotai/Kimi-K3/self_attn.f_a_proj+self_attn.b_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 24576, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 7168, + "n": 224, + "out": "bf16" + }, + "sources": [ + "moonshotai/Kimi-K3/self_attn.f_a_proj+self_attn.b_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 32768, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 7168, + "n": 224, + "out": "bf16" + }, + "sources": [ + "moonshotai/Kimi-K3/self_attn.f_a_proj+self_attn.b_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 49152, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 7168, + "n": 224, + "out": "bf16" + }, + "sources": [ + "moonshotai/Kimi-K3/self_attn.f_a_proj+self_attn.b_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 65536, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 7168, + "n": 224, + "out": "bf16" + }, + "sources": [ + "moonshotai/Kimi-K3/self_attn.f_a_proj+self_attn.b_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 1, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 128, + "n": 12288, + "out": "bf16" + }, + "sources": [ + "moonshotai/Kimi-K3/self_attn.f_b_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 2, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 128, + "n": 12288, + "out": "bf16" + }, + "sources": [ + "moonshotai/Kimi-K3/self_attn.f_b_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 3, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 128, + "n": 12288, + "out": "bf16" + }, + "sources": [ + "moonshotai/Kimi-K3/self_attn.f_b_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 4, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 128, + "n": 12288, + "out": "bf16" + }, + "sources": [ + "moonshotai/Kimi-K3/self_attn.f_b_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 6, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 128, + "n": 12288, + "out": "bf16" + }, + "sources": [ + "moonshotai/Kimi-K3/self_attn.f_b_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 8, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 128, + "n": 12288, + "out": "bf16" + }, + "sources": [ + "moonshotai/Kimi-K3/self_attn.f_b_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 12, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 128, + "n": 12288, + "out": "bf16" + }, + "sources": [ + "moonshotai/Kimi-K3/self_attn.f_b_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 16, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 128, + "n": 12288, + "out": "bf16" + }, + "sources": [ + "moonshotai/Kimi-K3/self_attn.f_b_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 24, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 128, + "n": 12288, + "out": "bf16" + }, + "sources": [ + "moonshotai/Kimi-K3/self_attn.f_b_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 32, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 128, + "n": 12288, + "out": "bf16" + }, + "sources": [ + "moonshotai/Kimi-K3/self_attn.f_b_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 48, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 128, + "n": 12288, + "out": "bf16" + }, + "sources": [ + "moonshotai/Kimi-K3/self_attn.f_b_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 64, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 128, + "n": 12288, + "out": "bf16" + }, + "sources": [ + "moonshotai/Kimi-K3/self_attn.f_b_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 96, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 128, + "n": 12288, + "out": "bf16" + }, + "sources": [ + "moonshotai/Kimi-K3/self_attn.f_b_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 128, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 128, + "n": 12288, + "out": "bf16" + }, + "sources": [ + "moonshotai/Kimi-K3/self_attn.f_b_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 192, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 128, + "n": 12288, + "out": "bf16" + }, + "sources": [ + "moonshotai/Kimi-K3/self_attn.f_b_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 256, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 128, + "n": 12288, + "out": "bf16" + }, + "sources": [ + "moonshotai/Kimi-K3/self_attn.f_b_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 384, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 128, + "n": 12288, + "out": "bf16" + }, + "sources": [ + "moonshotai/Kimi-K3/self_attn.f_b_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 512, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 128, + "n": 12288, + "out": "bf16" + }, + "sources": [ + "moonshotai/Kimi-K3/self_attn.f_b_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 768, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 128, + "n": 12288, + "out": "bf16" + }, + "sources": [ + "moonshotai/Kimi-K3/self_attn.f_b_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 1024, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 128, + "n": 12288, + "out": "bf16" + }, + "sources": [ + "moonshotai/Kimi-K3/self_attn.f_b_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 1536, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 128, + "n": 12288, + "out": "bf16" + }, + "sources": [ + "moonshotai/Kimi-K3/self_attn.f_b_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 2048, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 128, + "n": 12288, + "out": "bf16" + }, + "sources": [ + "moonshotai/Kimi-K3/self_attn.f_b_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 3072, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 128, + "n": 12288, + "out": "bf16" + }, + "sources": [ + "moonshotai/Kimi-K3/self_attn.f_b_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 4096, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 128, + "n": 12288, + "out": "bf16" + }, + "sources": [ + "moonshotai/Kimi-K3/self_attn.f_b_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 6144, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 128, + "n": 12288, + "out": "bf16" + }, + "sources": [ + "moonshotai/Kimi-K3/self_attn.f_b_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 8192, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 128, + "n": 12288, + "out": "bf16" + }, + "sources": [ + "moonshotai/Kimi-K3/self_attn.f_b_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 12288, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 128, + "n": 12288, + "out": "bf16" + }, + "sources": [ + "moonshotai/Kimi-K3/self_attn.f_b_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 16384, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 128, + "n": 12288, + "out": "bf16" + }, + "sources": [ + "moonshotai/Kimi-K3/self_attn.f_b_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 24576, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 128, + "n": 12288, + "out": "bf16" + }, + "sources": [ + "moonshotai/Kimi-K3/self_attn.f_b_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 32768, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 128, + "n": 12288, + "out": "bf16" + }, + "sources": [ + "moonshotai/Kimi-K3/self_attn.f_b_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 49152, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 128, + "n": 12288, + "out": "bf16" + }, + "sources": [ + "moonshotai/Kimi-K3/self_attn.f_b_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 65536, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 128, + "n": 12288, + "out": "bf16" + }, + "sources": [ + "moonshotai/Kimi-K3/self_attn.f_b_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 1, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 512, + "n": 24576, + "out": "bf16" + }, + "sources": [ + "moonshotai/Kimi-K3/self_attn.kv_b_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 2, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 512, + "n": 24576, + "out": "bf16" + }, + "sources": [ + "moonshotai/Kimi-K3/self_attn.kv_b_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 3, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 512, + "n": 24576, + "out": "bf16" + }, + "sources": [ + "moonshotai/Kimi-K3/self_attn.kv_b_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 4, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 512, + "n": 24576, + "out": "bf16" + }, + "sources": [ + "moonshotai/Kimi-K3/self_attn.kv_b_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 6, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 512, + "n": 24576, + "out": "bf16" + }, + "sources": [ + "moonshotai/Kimi-K3/self_attn.kv_b_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 8, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 512, + "n": 24576, + "out": "bf16" + }, + "sources": [ + "moonshotai/Kimi-K3/self_attn.kv_b_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 12, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 512, + "n": 24576, + "out": "bf16" + }, + "sources": [ + "moonshotai/Kimi-K3/self_attn.kv_b_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 16, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 512, + "n": 24576, + "out": "bf16" + }, + "sources": [ + "moonshotai/Kimi-K3/self_attn.kv_b_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 24, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 512, + "n": 24576, + "out": "bf16" + }, + "sources": [ + "moonshotai/Kimi-K3/self_attn.kv_b_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 32, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 512, + "n": 24576, + "out": "bf16" + }, + "sources": [ + "moonshotai/Kimi-K3/self_attn.kv_b_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 48, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 512, + "n": 24576, + "out": "bf16" + }, + "sources": [ + "moonshotai/Kimi-K3/self_attn.kv_b_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 64, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 512, + "n": 24576, + "out": "bf16" + }, + "sources": [ + "moonshotai/Kimi-K3/self_attn.kv_b_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 96, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 512, + "n": 24576, + "out": "bf16" + }, + "sources": [ + "moonshotai/Kimi-K3/self_attn.kv_b_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 128, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 512, + "n": 24576, + "out": "bf16" + }, + "sources": [ + "moonshotai/Kimi-K3/self_attn.kv_b_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 192, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 512, + "n": 24576, + "out": "bf16" + }, + "sources": [ + "moonshotai/Kimi-K3/self_attn.kv_b_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 256, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 512, + "n": 24576, + "out": "bf16" + }, + "sources": [ + "moonshotai/Kimi-K3/self_attn.kv_b_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 384, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 512, + "n": 24576, + "out": "bf16" + }, + "sources": [ + "moonshotai/Kimi-K3/self_attn.kv_b_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 512, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 512, + "n": 24576, + "out": "bf16" + }, + "sources": [ + "moonshotai/Kimi-K3/self_attn.kv_b_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 768, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 512, + "n": 24576, + "out": "bf16" + }, + "sources": [ + "moonshotai/Kimi-K3/self_attn.kv_b_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 1024, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 512, + "n": 24576, + "out": "bf16" + }, + "sources": [ + "moonshotai/Kimi-K3/self_attn.kv_b_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 1536, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 512, + "n": 24576, + "out": "bf16" + }, + "sources": [ + "moonshotai/Kimi-K3/self_attn.kv_b_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 2048, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 512, + "n": 24576, + "out": "bf16" + }, + "sources": [ + "moonshotai/Kimi-K3/self_attn.kv_b_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 3072, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 512, + "n": 24576, + "out": "bf16" + }, + "sources": [ + "moonshotai/Kimi-K3/self_attn.kv_b_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 4096, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 512, + "n": 24576, + "out": "bf16" + }, + "sources": [ + "moonshotai/Kimi-K3/self_attn.kv_b_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 6144, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 512, + "n": 24576, + "out": "bf16" + }, + "sources": [ + "moonshotai/Kimi-K3/self_attn.kv_b_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 8192, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 512, + "n": 24576, + "out": "bf16" + }, + "sources": [ + "moonshotai/Kimi-K3/self_attn.kv_b_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 12288, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 512, + "n": 24576, + "out": "bf16" + }, + "sources": [ + "moonshotai/Kimi-K3/self_attn.kv_b_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 16384, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 512, + "n": 24576, + "out": "bf16" + }, + "sources": [ + "moonshotai/Kimi-K3/self_attn.kv_b_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 24576, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 512, + "n": 24576, + "out": "bf16" + }, + "sources": [ + "moonshotai/Kimi-K3/self_attn.kv_b_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 32768, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 512, + "n": 24576, + "out": "bf16" + }, + "sources": [ + "moonshotai/Kimi-K3/self_attn.kv_b_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 49152, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 512, + "n": 24576, + "out": "bf16" + }, + "sources": [ + "moonshotai/Kimi-K3/self_attn.kv_b_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 65536, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 512, + "n": 24576, + "out": "bf16" + }, + "sources": [ + "moonshotai/Kimi-K3/self_attn.kv_b_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 1, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 12288, + "n": 7168, + "out": "bf16" + }, + "sources": [ + "moonshotai/Kimi-K3/self_attn.o_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 2, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 12288, + "n": 7168, + "out": "bf16" + }, + "sources": [ + "moonshotai/Kimi-K3/self_attn.o_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 3, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 12288, + "n": 7168, + "out": "bf16" + }, + "sources": [ + "moonshotai/Kimi-K3/self_attn.o_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 4, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 12288, + "n": 7168, + "out": "bf16" + }, + "sources": [ + "moonshotai/Kimi-K3/self_attn.o_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 6, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 12288, + "n": 7168, + "out": "bf16" + }, + "sources": [ + "moonshotai/Kimi-K3/self_attn.o_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 8, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 12288, + "n": 7168, + "out": "bf16" + }, + "sources": [ + "moonshotai/Kimi-K3/self_attn.o_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 12, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 12288, + "n": 7168, + "out": "bf16" + }, + "sources": [ + "moonshotai/Kimi-K3/self_attn.o_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 16, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 12288, + "n": 7168, + "out": "bf16" + }, + "sources": [ + "moonshotai/Kimi-K3/self_attn.o_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 24, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 12288, + "n": 7168, + "out": "bf16" + }, + "sources": [ + "moonshotai/Kimi-K3/self_attn.o_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 32, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 12288, + "n": 7168, + "out": "bf16" + }, + "sources": [ + "moonshotai/Kimi-K3/self_attn.o_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 48, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 12288, + "n": 7168, + "out": "bf16" + }, + "sources": [ + "moonshotai/Kimi-K3/self_attn.o_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 64, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 12288, + "n": 7168, + "out": "bf16" + }, + "sources": [ + "moonshotai/Kimi-K3/self_attn.o_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 96, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 12288, + "n": 7168, + "out": "bf16" + }, + "sources": [ + "moonshotai/Kimi-K3/self_attn.o_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 128, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 12288, + "n": 7168, + "out": "bf16" + }, + "sources": [ + "moonshotai/Kimi-K3/self_attn.o_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 192, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 12288, + "n": 7168, + "out": "bf16" + }, + "sources": [ + "moonshotai/Kimi-K3/self_attn.o_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 256, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 12288, + "n": 7168, + "out": "bf16" + }, + "sources": [ + "moonshotai/Kimi-K3/self_attn.o_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 384, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 12288, + "n": 7168, + "out": "bf16" + }, + "sources": [ + "moonshotai/Kimi-K3/self_attn.o_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 512, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 12288, + "n": 7168, + "out": "bf16" + }, + "sources": [ + "moonshotai/Kimi-K3/self_attn.o_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 768, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 12288, + "n": 7168, + "out": "bf16" + }, + "sources": [ + "moonshotai/Kimi-K3/self_attn.o_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 1024, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 12288, + "n": 7168, + "out": "bf16" + }, + "sources": [ + "moonshotai/Kimi-K3/self_attn.o_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 1536, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 12288, + "n": 7168, + "out": "bf16" + }, + "sources": [ + "moonshotai/Kimi-K3/self_attn.o_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 2048, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 12288, + "n": 7168, + "out": "bf16" + }, + "sources": [ + "moonshotai/Kimi-K3/self_attn.o_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 3072, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 12288, + "n": 7168, + "out": "bf16" + }, + "sources": [ + "moonshotai/Kimi-K3/self_attn.o_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 4096, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 12288, + "n": 7168, + "out": "bf16" + }, + "sources": [ + "moonshotai/Kimi-K3/self_attn.o_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 6144, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 12288, + "n": 7168, + "out": "bf16" + }, + "sources": [ + "moonshotai/Kimi-K3/self_attn.o_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 8192, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 12288, + "n": 7168, + "out": "bf16" + }, + "sources": [ + "moonshotai/Kimi-K3/self_attn.o_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 12288, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 12288, + "n": 7168, + "out": "bf16" + }, + "sources": [ + "moonshotai/Kimi-K3/self_attn.o_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 16384, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 12288, + "n": 7168, + "out": "bf16" + }, + "sources": [ + "moonshotai/Kimi-K3/self_attn.o_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 24576, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 12288, + "n": 7168, + "out": "bf16" + }, + "sources": [ + "moonshotai/Kimi-K3/self_attn.o_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 32768, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 12288, + "n": 7168, + "out": "bf16" + }, + "sources": [ + "moonshotai/Kimi-K3/self_attn.o_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 49152, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 12288, + "n": 7168, + "out": "bf16" + }, + "sources": [ + "moonshotai/Kimi-K3/self_attn.o_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 65536, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 12288, + "n": 7168, + "out": "bf16" + }, + "sources": [ + "moonshotai/Kimi-K3/self_attn.o_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 1, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 7168, + "n": 2112, + "out": "bf16" + }, + "sources": [ + "moonshotai/Kimi-K3/self_attn.q_a_proj+self_attn.kv_a_proj_with_mqa" + ] + }, + { + "type": "gemm", + "args": { + "m": 2, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 7168, + "n": 2112, + "out": "bf16" + }, + "sources": [ + "moonshotai/Kimi-K3/self_attn.q_a_proj+self_attn.kv_a_proj_with_mqa" + ] + }, + { + "type": "gemm", + "args": { + "m": 3, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 7168, + "n": 2112, + "out": "bf16" + }, + "sources": [ + "moonshotai/Kimi-K3/self_attn.q_a_proj+self_attn.kv_a_proj_with_mqa" + ] + }, + { + "type": "gemm", + "args": { + "m": 4, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 7168, + "n": 2112, + "out": "bf16" + }, + "sources": [ + "moonshotai/Kimi-K3/self_attn.q_a_proj+self_attn.kv_a_proj_with_mqa" + ] + }, + { + "type": "gemm", + "args": { + "m": 6, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 7168, + "n": 2112, + "out": "bf16" + }, + "sources": [ + "moonshotai/Kimi-K3/self_attn.q_a_proj+self_attn.kv_a_proj_with_mqa" + ] + }, + { + "type": "gemm", + "args": { + "m": 8, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 7168, + "n": 2112, + "out": "bf16" + }, + "sources": [ + "moonshotai/Kimi-K3/self_attn.q_a_proj+self_attn.kv_a_proj_with_mqa" + ] + }, + { + "type": "gemm", + "args": { + "m": 12, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 7168, + "n": 2112, + "out": "bf16" + }, + "sources": [ + "moonshotai/Kimi-K3/self_attn.q_a_proj+self_attn.kv_a_proj_with_mqa" + ] + }, + { + "type": "gemm", + "args": { + "m": 16, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 7168, + "n": 2112, + "out": "bf16" + }, + "sources": [ + "moonshotai/Kimi-K3/self_attn.q_a_proj+self_attn.kv_a_proj_with_mqa" + ] + }, + { + "type": "gemm", + "args": { + "m": 24, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 7168, + "n": 2112, + "out": "bf16" + }, + "sources": [ + "moonshotai/Kimi-K3/self_attn.q_a_proj+self_attn.kv_a_proj_with_mqa" + ] + }, + { + "type": "gemm", + "args": { + "m": 32, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 7168, + "n": 2112, + "out": "bf16" + }, + "sources": [ + "moonshotai/Kimi-K3/self_attn.q_a_proj+self_attn.kv_a_proj_with_mqa" + ] + }, + { + "type": "gemm", + "args": { + "m": 48, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 7168, + "n": 2112, + "out": "bf16" + }, + "sources": [ + "moonshotai/Kimi-K3/self_attn.q_a_proj+self_attn.kv_a_proj_with_mqa" + ] + }, + { + "type": "gemm", + "args": { + "m": 64, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 7168, + "n": 2112, + "out": "bf16" + }, + "sources": [ + "moonshotai/Kimi-K3/self_attn.q_a_proj+self_attn.kv_a_proj_with_mqa" + ] + }, + { + "type": "gemm", + "args": { + "m": 96, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 7168, + "n": 2112, + "out": "bf16" + }, + "sources": [ + "moonshotai/Kimi-K3/self_attn.q_a_proj+self_attn.kv_a_proj_with_mqa" + ] + }, + { + "type": "gemm", + "args": { + "m": 128, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 7168, + "n": 2112, + "out": "bf16" + }, + "sources": [ + "moonshotai/Kimi-K3/self_attn.q_a_proj+self_attn.kv_a_proj_with_mqa" + ] + }, + { + "type": "gemm", + "args": { + "m": 192, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 7168, + "n": 2112, + "out": "bf16" + }, + "sources": [ + "moonshotai/Kimi-K3/self_attn.q_a_proj+self_attn.kv_a_proj_with_mqa" + ] + }, + { + "type": "gemm", + "args": { + "m": 256, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 7168, + "n": 2112, + "out": "bf16" + }, + "sources": [ + "moonshotai/Kimi-K3/self_attn.q_a_proj+self_attn.kv_a_proj_with_mqa" + ] + }, + { + "type": "gemm", + "args": { + "m": 384, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 7168, + "n": 2112, + "out": "bf16" + }, + "sources": [ + "moonshotai/Kimi-K3/self_attn.q_a_proj+self_attn.kv_a_proj_with_mqa" + ] + }, + { + "type": "gemm", + "args": { + "m": 512, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 7168, + "n": 2112, + "out": "bf16" + }, + "sources": [ + "moonshotai/Kimi-K3/self_attn.q_a_proj+self_attn.kv_a_proj_with_mqa" + ] + }, + { + "type": "gemm", + "args": { + "m": 768, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 7168, + "n": 2112, + "out": "bf16" + }, + "sources": [ + "moonshotai/Kimi-K3/self_attn.q_a_proj+self_attn.kv_a_proj_with_mqa" + ] + }, + { + "type": "gemm", + "args": { + "m": 1024, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 7168, + "n": 2112, + "out": "bf16" + }, + "sources": [ + "moonshotai/Kimi-K3/self_attn.q_a_proj+self_attn.kv_a_proj_with_mqa" + ] + }, + { + "type": "gemm", + "args": { + "m": 1536, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 7168, + "n": 2112, + "out": "bf16" + }, + "sources": [ + "moonshotai/Kimi-K3/self_attn.q_a_proj+self_attn.kv_a_proj_with_mqa" + ] + }, + { + "type": "gemm", + "args": { + "m": 2048, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 7168, + "n": 2112, + "out": "bf16" + }, + "sources": [ + "moonshotai/Kimi-K3/self_attn.q_a_proj+self_attn.kv_a_proj_with_mqa" + ] + }, + { + "type": "gemm", + "args": { + "m": 3072, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 7168, + "n": 2112, + "out": "bf16" + }, + "sources": [ + "moonshotai/Kimi-K3/self_attn.q_a_proj+self_attn.kv_a_proj_with_mqa" + ] + }, + { + "type": "gemm", + "args": { + "m": 4096, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 7168, + "n": 2112, + "out": "bf16" + }, + "sources": [ + "moonshotai/Kimi-K3/self_attn.q_a_proj+self_attn.kv_a_proj_with_mqa" + ] + }, + { + "type": "gemm", + "args": { + "m": 6144, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 7168, + "n": 2112, + "out": "bf16" + }, + "sources": [ + "moonshotai/Kimi-K3/self_attn.q_a_proj+self_attn.kv_a_proj_with_mqa" + ] + }, + { + "type": "gemm", + "args": { + "m": 8192, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 7168, + "n": 2112, + "out": "bf16" + }, + "sources": [ + "moonshotai/Kimi-K3/self_attn.q_a_proj+self_attn.kv_a_proj_with_mqa" + ] + }, + { + "type": "gemm", + "args": { + "m": 12288, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 7168, + "n": 2112, + "out": "bf16" + }, + "sources": [ + "moonshotai/Kimi-K3/self_attn.q_a_proj+self_attn.kv_a_proj_with_mqa" + ] + }, + { + "type": "gemm", + "args": { + "m": 16384, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 7168, + "n": 2112, + "out": "bf16" + }, + "sources": [ + "moonshotai/Kimi-K3/self_attn.q_a_proj+self_attn.kv_a_proj_with_mqa" + ] + }, + { + "type": "gemm", + "args": { + "m": 24576, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 7168, + "n": 2112, + "out": "bf16" + }, + "sources": [ + "moonshotai/Kimi-K3/self_attn.q_a_proj+self_attn.kv_a_proj_with_mqa" + ] + }, + { + "type": "gemm", + "args": { + "m": 32768, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 7168, + "n": 2112, + "out": "bf16" + }, + "sources": [ + "moonshotai/Kimi-K3/self_attn.q_a_proj+self_attn.kv_a_proj_with_mqa" + ] + }, + { + "type": "gemm", + "args": { + "m": 49152, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 7168, + "n": 2112, + "out": "bf16" + }, + "sources": [ + "moonshotai/Kimi-K3/self_attn.q_a_proj+self_attn.kv_a_proj_with_mqa" + ] + }, + { + "type": "gemm", + "args": { + "m": 65536, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 7168, + "n": 2112, + "out": "bf16" + }, + "sources": [ + "moonshotai/Kimi-K3/self_attn.q_a_proj+self_attn.kv_a_proj_with_mqa" + ] + }, + { + "type": "gemm", + "args": { + "m": 1, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 7168, + "n": 14400, + "out": "bf16" + }, + "sources": [ + "moonshotai/Kimi-K3/self_attn.q_a_proj+self_attn.kv_a_proj_with_mqa+self_attn.g_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 2, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 7168, + "n": 14400, + "out": "bf16" + }, + "sources": [ + "moonshotai/Kimi-K3/self_attn.q_a_proj+self_attn.kv_a_proj_with_mqa+self_attn.g_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 3, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 7168, + "n": 14400, + "out": "bf16" + }, + "sources": [ + "moonshotai/Kimi-K3/self_attn.q_a_proj+self_attn.kv_a_proj_with_mqa+self_attn.g_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 4, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 7168, + "n": 14400, + "out": "bf16" + }, + "sources": [ + "moonshotai/Kimi-K3/self_attn.q_a_proj+self_attn.kv_a_proj_with_mqa+self_attn.g_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 6, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 7168, + "n": 14400, + "out": "bf16" + }, + "sources": [ + "moonshotai/Kimi-K3/self_attn.q_a_proj+self_attn.kv_a_proj_with_mqa+self_attn.g_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 8, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 7168, + "n": 14400, + "out": "bf16" + }, + "sources": [ + "moonshotai/Kimi-K3/self_attn.q_a_proj+self_attn.kv_a_proj_with_mqa+self_attn.g_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 12, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 7168, + "n": 14400, + "out": "bf16" + }, + "sources": [ + "moonshotai/Kimi-K3/self_attn.q_a_proj+self_attn.kv_a_proj_with_mqa+self_attn.g_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 16, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 7168, + "n": 14400, + "out": "bf16" + }, + "sources": [ + "moonshotai/Kimi-K3/self_attn.q_a_proj+self_attn.kv_a_proj_with_mqa+self_attn.g_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 24, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 7168, + "n": 14400, + "out": "bf16" + }, + "sources": [ + "moonshotai/Kimi-K3/self_attn.q_a_proj+self_attn.kv_a_proj_with_mqa+self_attn.g_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 32, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 7168, + "n": 14400, + "out": "bf16" + }, + "sources": [ + "moonshotai/Kimi-K3/self_attn.q_a_proj+self_attn.kv_a_proj_with_mqa+self_attn.g_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 48, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 7168, + "n": 14400, + "out": "bf16" + }, + "sources": [ + "moonshotai/Kimi-K3/self_attn.q_a_proj+self_attn.kv_a_proj_with_mqa+self_attn.g_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 64, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 7168, + "n": 14400, + "out": "bf16" + }, + "sources": [ + "moonshotai/Kimi-K3/self_attn.q_a_proj+self_attn.kv_a_proj_with_mqa+self_attn.g_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 96, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 7168, + "n": 14400, + "out": "bf16" + }, + "sources": [ + "moonshotai/Kimi-K3/self_attn.q_a_proj+self_attn.kv_a_proj_with_mqa+self_attn.g_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 128, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 7168, + "n": 14400, + "out": "bf16" + }, + "sources": [ + "moonshotai/Kimi-K3/self_attn.q_a_proj+self_attn.kv_a_proj_with_mqa+self_attn.g_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 192, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 7168, + "n": 14400, + "out": "bf16" + }, + "sources": [ + "moonshotai/Kimi-K3/self_attn.q_a_proj+self_attn.kv_a_proj_with_mqa+self_attn.g_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 256, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 7168, + "n": 14400, + "out": "bf16" + }, + "sources": [ + "moonshotai/Kimi-K3/self_attn.q_a_proj+self_attn.kv_a_proj_with_mqa+self_attn.g_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 384, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 7168, + "n": 14400, + "out": "bf16" + }, + "sources": [ + "moonshotai/Kimi-K3/self_attn.q_a_proj+self_attn.kv_a_proj_with_mqa+self_attn.g_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 512, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 7168, + "n": 14400, + "out": "bf16" + }, + "sources": [ + "moonshotai/Kimi-K3/self_attn.q_a_proj+self_attn.kv_a_proj_with_mqa+self_attn.g_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 768, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 7168, + "n": 14400, + "out": "bf16" + }, + "sources": [ + "moonshotai/Kimi-K3/self_attn.q_a_proj+self_attn.kv_a_proj_with_mqa+self_attn.g_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 1024, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 7168, + "n": 14400, + "out": "bf16" + }, + "sources": [ + "moonshotai/Kimi-K3/self_attn.q_a_proj+self_attn.kv_a_proj_with_mqa+self_attn.g_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 1536, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 7168, + "n": 14400, + "out": "bf16" + }, + "sources": [ + "moonshotai/Kimi-K3/self_attn.q_a_proj+self_attn.kv_a_proj_with_mqa+self_attn.g_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 2048, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 7168, + "n": 14400, + "out": "bf16" + }, + "sources": [ + "moonshotai/Kimi-K3/self_attn.q_a_proj+self_attn.kv_a_proj_with_mqa+self_attn.g_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 3072, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 7168, + "n": 14400, + "out": "bf16" + }, + "sources": [ + "moonshotai/Kimi-K3/self_attn.q_a_proj+self_attn.kv_a_proj_with_mqa+self_attn.g_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 4096, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 7168, + "n": 14400, + "out": "bf16" + }, + "sources": [ + "moonshotai/Kimi-K3/self_attn.q_a_proj+self_attn.kv_a_proj_with_mqa+self_attn.g_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 6144, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 7168, + "n": 14400, + "out": "bf16" + }, + "sources": [ + "moonshotai/Kimi-K3/self_attn.q_a_proj+self_attn.kv_a_proj_with_mqa+self_attn.g_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 8192, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 7168, + "n": 14400, + "out": "bf16" + }, + "sources": [ + "moonshotai/Kimi-K3/self_attn.q_a_proj+self_attn.kv_a_proj_with_mqa+self_attn.g_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 12288, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 7168, + "n": 14400, + "out": "bf16" + }, + "sources": [ + "moonshotai/Kimi-K3/self_attn.q_a_proj+self_attn.kv_a_proj_with_mqa+self_attn.g_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 16384, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 7168, + "n": 14400, + "out": "bf16" + }, + "sources": [ + "moonshotai/Kimi-K3/self_attn.q_a_proj+self_attn.kv_a_proj_with_mqa+self_attn.g_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 24576, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 7168, + "n": 14400, + "out": "bf16" + }, + "sources": [ + "moonshotai/Kimi-K3/self_attn.q_a_proj+self_attn.kv_a_proj_with_mqa+self_attn.g_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 32768, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 7168, + "n": 14400, + "out": "bf16" + }, + "sources": [ + "moonshotai/Kimi-K3/self_attn.q_a_proj+self_attn.kv_a_proj_with_mqa+self_attn.g_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 49152, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 7168, + "n": 14400, + "out": "bf16" + }, + "sources": [ + "moonshotai/Kimi-K3/self_attn.q_a_proj+self_attn.kv_a_proj_with_mqa+self_attn.g_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 65536, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 7168, + "n": 14400, + "out": "bf16" + }, + "sources": [ + "moonshotai/Kimi-K3/self_attn.q_a_proj+self_attn.kv_a_proj_with_mqa+self_attn.g_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 1, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 1536, + "n": 18432, + "out": "bf16" + }, + "sources": [ + "moonshotai/Kimi-K3/self_attn.q_b_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 2, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 1536, + "n": 18432, + "out": "bf16" + }, + "sources": [ + "moonshotai/Kimi-K3/self_attn.q_b_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 3, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 1536, + "n": 18432, + "out": "bf16" + }, + "sources": [ + "moonshotai/Kimi-K3/self_attn.q_b_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 4, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 1536, + "n": 18432, + "out": "bf16" + }, + "sources": [ + "moonshotai/Kimi-K3/self_attn.q_b_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 6, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 1536, + "n": 18432, + "out": "bf16" + }, + "sources": [ + "moonshotai/Kimi-K3/self_attn.q_b_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 8, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 1536, + "n": 18432, + "out": "bf16" + }, + "sources": [ + "moonshotai/Kimi-K3/self_attn.q_b_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 12, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 1536, + "n": 18432, + "out": "bf16" + }, + "sources": [ + "moonshotai/Kimi-K3/self_attn.q_b_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 16, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 1536, + "n": 18432, + "out": "bf16" + }, + "sources": [ + "moonshotai/Kimi-K3/self_attn.q_b_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 24, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 1536, + "n": 18432, + "out": "bf16" + }, + "sources": [ + "moonshotai/Kimi-K3/self_attn.q_b_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 32, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 1536, + "n": 18432, + "out": "bf16" + }, + "sources": [ + "moonshotai/Kimi-K3/self_attn.q_b_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 48, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 1536, + "n": 18432, + "out": "bf16" + }, + "sources": [ + "moonshotai/Kimi-K3/self_attn.q_b_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 64, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 1536, + "n": 18432, + "out": "bf16" + }, + "sources": [ + "moonshotai/Kimi-K3/self_attn.q_b_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 96, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 1536, + "n": 18432, + "out": "bf16" + }, + "sources": [ + "moonshotai/Kimi-K3/self_attn.q_b_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 128, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 1536, + "n": 18432, + "out": "bf16" + }, + "sources": [ + "moonshotai/Kimi-K3/self_attn.q_b_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 192, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 1536, + "n": 18432, + "out": "bf16" + }, + "sources": [ + "moonshotai/Kimi-K3/self_attn.q_b_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 256, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 1536, + "n": 18432, + "out": "bf16" + }, + "sources": [ + "moonshotai/Kimi-K3/self_attn.q_b_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 384, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 1536, + "n": 18432, + "out": "bf16" + }, + "sources": [ + "moonshotai/Kimi-K3/self_attn.q_b_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 512, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 1536, + "n": 18432, + "out": "bf16" + }, + "sources": [ + "moonshotai/Kimi-K3/self_attn.q_b_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 768, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 1536, + "n": 18432, + "out": "bf16" + }, + "sources": [ + "moonshotai/Kimi-K3/self_attn.q_b_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 1024, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 1536, + "n": 18432, + "out": "bf16" + }, + "sources": [ + "moonshotai/Kimi-K3/self_attn.q_b_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 1536, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 1536, + "n": 18432, + "out": "bf16" + }, + "sources": [ + "moonshotai/Kimi-K3/self_attn.q_b_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 2048, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 1536, + "n": 18432, + "out": "bf16" + }, + "sources": [ + "moonshotai/Kimi-K3/self_attn.q_b_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 3072, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 1536, + "n": 18432, + "out": "bf16" + }, + "sources": [ + "moonshotai/Kimi-K3/self_attn.q_b_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 4096, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 1536, + "n": 18432, + "out": "bf16" + }, + "sources": [ + "moonshotai/Kimi-K3/self_attn.q_b_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 6144, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 1536, + "n": 18432, + "out": "bf16" + }, + "sources": [ + "moonshotai/Kimi-K3/self_attn.q_b_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 8192, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 1536, + "n": 18432, + "out": "bf16" + }, + "sources": [ + "moonshotai/Kimi-K3/self_attn.q_b_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 12288, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 1536, + "n": 18432, + "out": "bf16" + }, + "sources": [ + "moonshotai/Kimi-K3/self_attn.q_b_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 16384, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 1536, + "n": 18432, + "out": "bf16" + }, + "sources": [ + "moonshotai/Kimi-K3/self_attn.q_b_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 24576, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 1536, + "n": 18432, + "out": "bf16" + }, + "sources": [ + "moonshotai/Kimi-K3/self_attn.q_b_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 32768, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 1536, + "n": 18432, + "out": "bf16" + }, + "sources": [ + "moonshotai/Kimi-K3/self_attn.q_b_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 49152, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 1536, + "n": 18432, + "out": "bf16" + }, + "sources": [ + "moonshotai/Kimi-K3/self_attn.q_b_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 65536, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 1536, + "n": 18432, + "out": "bf16" + }, + "sources": [ + "moonshotai/Kimi-K3/self_attn.q_b_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 1, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 7168, + "n": 36864, + "out": "bf16" + }, + "sources": [ + "moonshotai/Kimi-K3/self_attn.q_proj+self_attn.k_proj+self_attn.v_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 2, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 7168, + "n": 36864, + "out": "bf16" + }, + "sources": [ + "moonshotai/Kimi-K3/self_attn.q_proj+self_attn.k_proj+self_attn.v_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 3, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 7168, + "n": 36864, + "out": "bf16" + }, + "sources": [ + "moonshotai/Kimi-K3/self_attn.q_proj+self_attn.k_proj+self_attn.v_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 4, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 7168, + "n": 36864, + "out": "bf16" + }, + "sources": [ + "moonshotai/Kimi-K3/self_attn.q_proj+self_attn.k_proj+self_attn.v_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 6, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 7168, + "n": 36864, + "out": "bf16" + }, + "sources": [ + "moonshotai/Kimi-K3/self_attn.q_proj+self_attn.k_proj+self_attn.v_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 8, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 7168, + "n": 36864, + "out": "bf16" + }, + "sources": [ + "moonshotai/Kimi-K3/self_attn.q_proj+self_attn.k_proj+self_attn.v_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 12, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 7168, + "n": 36864, + "out": "bf16" + }, + "sources": [ + "moonshotai/Kimi-K3/self_attn.q_proj+self_attn.k_proj+self_attn.v_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 16, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 7168, + "n": 36864, + "out": "bf16" + }, + "sources": [ + "moonshotai/Kimi-K3/self_attn.q_proj+self_attn.k_proj+self_attn.v_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 24, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 7168, + "n": 36864, + "out": "bf16" + }, + "sources": [ + "moonshotai/Kimi-K3/self_attn.q_proj+self_attn.k_proj+self_attn.v_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 32, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 7168, + "n": 36864, + "out": "bf16" + }, + "sources": [ + "moonshotai/Kimi-K3/self_attn.q_proj+self_attn.k_proj+self_attn.v_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 48, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 7168, + "n": 36864, + "out": "bf16" + }, + "sources": [ + "moonshotai/Kimi-K3/self_attn.q_proj+self_attn.k_proj+self_attn.v_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 64, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 7168, + "n": 36864, + "out": "bf16" + }, + "sources": [ + "moonshotai/Kimi-K3/self_attn.q_proj+self_attn.k_proj+self_attn.v_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 96, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 7168, + "n": 36864, + "out": "bf16" + }, + "sources": [ + "moonshotai/Kimi-K3/self_attn.q_proj+self_attn.k_proj+self_attn.v_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 128, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 7168, + "n": 36864, + "out": "bf16" + }, + "sources": [ + "moonshotai/Kimi-K3/self_attn.q_proj+self_attn.k_proj+self_attn.v_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 192, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 7168, + "n": 36864, + "out": "bf16" + }, + "sources": [ + "moonshotai/Kimi-K3/self_attn.q_proj+self_attn.k_proj+self_attn.v_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 256, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 7168, + "n": 36864, + "out": "bf16" + }, + "sources": [ + "moonshotai/Kimi-K3/self_attn.q_proj+self_attn.k_proj+self_attn.v_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 384, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 7168, + "n": 36864, + "out": "bf16" + }, + "sources": [ + "moonshotai/Kimi-K3/self_attn.q_proj+self_attn.k_proj+self_attn.v_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 512, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 7168, + "n": 36864, + "out": "bf16" + }, + "sources": [ + "moonshotai/Kimi-K3/self_attn.q_proj+self_attn.k_proj+self_attn.v_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 768, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 7168, + "n": 36864, + "out": "bf16" + }, + "sources": [ + "moonshotai/Kimi-K3/self_attn.q_proj+self_attn.k_proj+self_attn.v_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 1024, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 7168, + "n": 36864, + "out": "bf16" + }, + "sources": [ + "moonshotai/Kimi-K3/self_attn.q_proj+self_attn.k_proj+self_attn.v_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 1536, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 7168, + "n": 36864, + "out": "bf16" + }, + "sources": [ + "moonshotai/Kimi-K3/self_attn.q_proj+self_attn.k_proj+self_attn.v_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 2048, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 7168, + "n": 36864, + "out": "bf16" + }, + "sources": [ + "moonshotai/Kimi-K3/self_attn.q_proj+self_attn.k_proj+self_attn.v_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 3072, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 7168, + "n": 36864, + "out": "bf16" + }, + "sources": [ + "moonshotai/Kimi-K3/self_attn.q_proj+self_attn.k_proj+self_attn.v_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 4096, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 7168, + "n": 36864, + "out": "bf16" + }, + "sources": [ + "moonshotai/Kimi-K3/self_attn.q_proj+self_attn.k_proj+self_attn.v_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 6144, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 7168, + "n": 36864, + "out": "bf16" + }, + "sources": [ + "moonshotai/Kimi-K3/self_attn.q_proj+self_attn.k_proj+self_attn.v_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 8192, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 7168, + "n": 36864, + "out": "bf16" + }, + "sources": [ + "moonshotai/Kimi-K3/self_attn.q_proj+self_attn.k_proj+self_attn.v_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 12288, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 7168, + "n": 36864, + "out": "bf16" + }, + "sources": [ + "moonshotai/Kimi-K3/self_attn.q_proj+self_attn.k_proj+self_attn.v_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 16384, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 7168, + "n": 36864, + "out": "bf16" + }, + "sources": [ + "moonshotai/Kimi-K3/self_attn.q_proj+self_attn.k_proj+self_attn.v_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 24576, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 7168, + "n": 36864, + "out": "bf16" + }, + "sources": [ + "moonshotai/Kimi-K3/self_attn.q_proj+self_attn.k_proj+self_attn.v_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 32768, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 7168, + "n": 36864, + "out": "bf16" + }, + "sources": [ + "moonshotai/Kimi-K3/self_attn.q_proj+self_attn.k_proj+self_attn.v_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 49152, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 7168, + "n": 36864, + "out": "bf16" + }, + "sources": [ + "moonshotai/Kimi-K3/self_attn.q_proj+self_attn.k_proj+self_attn.v_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 65536, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 7168, + "n": 36864, + "out": "bf16" + }, + "sources": [ + "moonshotai/Kimi-K3/self_attn.q_proj+self_attn.k_proj+self_attn.v_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 1, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 7168, + "n": 49152, + "out": "bf16" + }, + "sources": [ + "moonshotai/Kimi-K3/self_attn.q_proj+self_attn.k_proj+self_attn.v_proj+self_attn.g_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 2, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 7168, + "n": 49152, + "out": "bf16" + }, + "sources": [ + "moonshotai/Kimi-K3/self_attn.q_proj+self_attn.k_proj+self_attn.v_proj+self_attn.g_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 3, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 7168, + "n": 49152, + "out": "bf16" + }, + "sources": [ + "moonshotai/Kimi-K3/self_attn.q_proj+self_attn.k_proj+self_attn.v_proj+self_attn.g_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 4, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 7168, + "n": 49152, + "out": "bf16" + }, + "sources": [ + "moonshotai/Kimi-K3/self_attn.q_proj+self_attn.k_proj+self_attn.v_proj+self_attn.g_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 6, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 7168, + "n": 49152, + "out": "bf16" + }, + "sources": [ + "moonshotai/Kimi-K3/self_attn.q_proj+self_attn.k_proj+self_attn.v_proj+self_attn.g_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 8, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 7168, + "n": 49152, + "out": "bf16" + }, + "sources": [ + "moonshotai/Kimi-K3/self_attn.q_proj+self_attn.k_proj+self_attn.v_proj+self_attn.g_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 12, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 7168, + "n": 49152, + "out": "bf16" + }, + "sources": [ + "moonshotai/Kimi-K3/self_attn.q_proj+self_attn.k_proj+self_attn.v_proj+self_attn.g_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 16, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 7168, + "n": 49152, + "out": "bf16" + }, + "sources": [ + "moonshotai/Kimi-K3/self_attn.q_proj+self_attn.k_proj+self_attn.v_proj+self_attn.g_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 24, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 7168, + "n": 49152, + "out": "bf16" + }, + "sources": [ + "moonshotai/Kimi-K3/self_attn.q_proj+self_attn.k_proj+self_attn.v_proj+self_attn.g_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 32, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 7168, + "n": 49152, + "out": "bf16" + }, + "sources": [ + "moonshotai/Kimi-K3/self_attn.q_proj+self_attn.k_proj+self_attn.v_proj+self_attn.g_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 48, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 7168, + "n": 49152, + "out": "bf16" + }, + "sources": [ + "moonshotai/Kimi-K3/self_attn.q_proj+self_attn.k_proj+self_attn.v_proj+self_attn.g_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 64, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 7168, + "n": 49152, + "out": "bf16" + }, + "sources": [ + "moonshotai/Kimi-K3/self_attn.q_proj+self_attn.k_proj+self_attn.v_proj+self_attn.g_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 96, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 7168, + "n": 49152, + "out": "bf16" + }, + "sources": [ + "moonshotai/Kimi-K3/self_attn.q_proj+self_attn.k_proj+self_attn.v_proj+self_attn.g_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 128, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 7168, + "n": 49152, + "out": "bf16" + }, + "sources": [ + "moonshotai/Kimi-K3/self_attn.q_proj+self_attn.k_proj+self_attn.v_proj+self_attn.g_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 192, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 7168, + "n": 49152, + "out": "bf16" + }, + "sources": [ + "moonshotai/Kimi-K3/self_attn.q_proj+self_attn.k_proj+self_attn.v_proj+self_attn.g_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 256, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 7168, + "n": 49152, + "out": "bf16" + }, + "sources": [ + "moonshotai/Kimi-K3/self_attn.q_proj+self_attn.k_proj+self_attn.v_proj+self_attn.g_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 384, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 7168, + "n": 49152, + "out": "bf16" + }, + "sources": [ + "moonshotai/Kimi-K3/self_attn.q_proj+self_attn.k_proj+self_attn.v_proj+self_attn.g_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 512, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 7168, + "n": 49152, + "out": "bf16" + }, + "sources": [ + "moonshotai/Kimi-K3/self_attn.q_proj+self_attn.k_proj+self_attn.v_proj+self_attn.g_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 768, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 7168, + "n": 49152, + "out": "bf16" + }, + "sources": [ + "moonshotai/Kimi-K3/self_attn.q_proj+self_attn.k_proj+self_attn.v_proj+self_attn.g_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 1024, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 7168, + "n": 49152, + "out": "bf16" + }, + "sources": [ + "moonshotai/Kimi-K3/self_attn.q_proj+self_attn.k_proj+self_attn.v_proj+self_attn.g_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 1536, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 7168, + "n": 49152, + "out": "bf16" + }, + "sources": [ + "moonshotai/Kimi-K3/self_attn.q_proj+self_attn.k_proj+self_attn.v_proj+self_attn.g_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 2048, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 7168, + "n": 49152, + "out": "bf16" + }, + "sources": [ + "moonshotai/Kimi-K3/self_attn.q_proj+self_attn.k_proj+self_attn.v_proj+self_attn.g_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 3072, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 7168, + "n": 49152, + "out": "bf16" + }, + "sources": [ + "moonshotai/Kimi-K3/self_attn.q_proj+self_attn.k_proj+self_attn.v_proj+self_attn.g_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 4096, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 7168, + "n": 49152, + "out": "bf16" + }, + "sources": [ + "moonshotai/Kimi-K3/self_attn.q_proj+self_attn.k_proj+self_attn.v_proj+self_attn.g_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 6144, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 7168, + "n": 49152, + "out": "bf16" + }, + "sources": [ + "moonshotai/Kimi-K3/self_attn.q_proj+self_attn.k_proj+self_attn.v_proj+self_attn.g_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 8192, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 7168, + "n": 49152, + "out": "bf16" + }, + "sources": [ + "moonshotai/Kimi-K3/self_attn.q_proj+self_attn.k_proj+self_attn.v_proj+self_attn.g_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 12288, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 7168, + "n": 49152, + "out": "bf16" + }, + "sources": [ + "moonshotai/Kimi-K3/self_attn.q_proj+self_attn.k_proj+self_attn.v_proj+self_attn.g_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 16384, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 7168, + "n": 49152, + "out": "bf16" + }, + "sources": [ + "moonshotai/Kimi-K3/self_attn.q_proj+self_attn.k_proj+self_attn.v_proj+self_attn.g_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 24576, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 7168, + "n": 49152, + "out": "bf16" + }, + "sources": [ + "moonshotai/Kimi-K3/self_attn.q_proj+self_attn.k_proj+self_attn.v_proj+self_attn.g_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 32768, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 7168, + "n": 49152, + "out": "bf16" + }, + "sources": [ + "moonshotai/Kimi-K3/self_attn.q_proj+self_attn.k_proj+self_attn.v_proj+self_attn.g_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 49152, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 7168, + "n": 49152, + "out": "bf16" + }, + "sources": [ + "moonshotai/Kimi-K3/self_attn.q_proj+self_attn.k_proj+self_attn.v_proj+self_attn.g_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 65536, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 7168, + "n": 49152, + "out": "bf16" + }, + "sources": [ + "moonshotai/Kimi-K3/self_attn.q_proj+self_attn.k_proj+self_attn.v_proj+self_attn.g_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 1, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 7168, + "n": 49376, + "out": "bf16" + }, + "sources": [ + "moonshotai/Kimi-K3/self_attn.q_proj+self_attn.k_proj+self_attn.v_proj+self_attn.g_proj+self_attn.f_a_proj+self_attn.b_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 2, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 7168, + "n": 49376, + "out": "bf16" + }, + "sources": [ + "moonshotai/Kimi-K3/self_attn.q_proj+self_attn.k_proj+self_attn.v_proj+self_attn.g_proj+self_attn.f_a_proj+self_attn.b_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 3, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 7168, + "n": 49376, + "out": "bf16" + }, + "sources": [ + "moonshotai/Kimi-K3/self_attn.q_proj+self_attn.k_proj+self_attn.v_proj+self_attn.g_proj+self_attn.f_a_proj+self_attn.b_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 4, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 7168, + "n": 49376, + "out": "bf16" + }, + "sources": [ + "moonshotai/Kimi-K3/self_attn.q_proj+self_attn.k_proj+self_attn.v_proj+self_attn.g_proj+self_attn.f_a_proj+self_attn.b_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 6, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 7168, + "n": 49376, + "out": "bf16" + }, + "sources": [ + "moonshotai/Kimi-K3/self_attn.q_proj+self_attn.k_proj+self_attn.v_proj+self_attn.g_proj+self_attn.f_a_proj+self_attn.b_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 8, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 7168, + "n": 49376, + "out": "bf16" + }, + "sources": [ + "moonshotai/Kimi-K3/self_attn.q_proj+self_attn.k_proj+self_attn.v_proj+self_attn.g_proj+self_attn.f_a_proj+self_attn.b_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 12, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 7168, + "n": 49376, + "out": "bf16" + }, + "sources": [ + "moonshotai/Kimi-K3/self_attn.q_proj+self_attn.k_proj+self_attn.v_proj+self_attn.g_proj+self_attn.f_a_proj+self_attn.b_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 16, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 7168, + "n": 49376, + "out": "bf16" + }, + "sources": [ + "moonshotai/Kimi-K3/self_attn.q_proj+self_attn.k_proj+self_attn.v_proj+self_attn.g_proj+self_attn.f_a_proj+self_attn.b_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 24, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 7168, + "n": 49376, + "out": "bf16" + }, + "sources": [ + "moonshotai/Kimi-K3/self_attn.q_proj+self_attn.k_proj+self_attn.v_proj+self_attn.g_proj+self_attn.f_a_proj+self_attn.b_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 32, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 7168, + "n": 49376, + "out": "bf16" + }, + "sources": [ + "moonshotai/Kimi-K3/self_attn.q_proj+self_attn.k_proj+self_attn.v_proj+self_attn.g_proj+self_attn.f_a_proj+self_attn.b_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 48, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 7168, + "n": 49376, + "out": "bf16" + }, + "sources": [ + "moonshotai/Kimi-K3/self_attn.q_proj+self_attn.k_proj+self_attn.v_proj+self_attn.g_proj+self_attn.f_a_proj+self_attn.b_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 64, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 7168, + "n": 49376, + "out": "bf16" + }, + "sources": [ + "moonshotai/Kimi-K3/self_attn.q_proj+self_attn.k_proj+self_attn.v_proj+self_attn.g_proj+self_attn.f_a_proj+self_attn.b_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 96, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 7168, + "n": 49376, + "out": "bf16" + }, + "sources": [ + "moonshotai/Kimi-K3/self_attn.q_proj+self_attn.k_proj+self_attn.v_proj+self_attn.g_proj+self_attn.f_a_proj+self_attn.b_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 128, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 7168, + "n": 49376, + "out": "bf16" + }, + "sources": [ + "moonshotai/Kimi-K3/self_attn.q_proj+self_attn.k_proj+self_attn.v_proj+self_attn.g_proj+self_attn.f_a_proj+self_attn.b_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 192, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 7168, + "n": 49376, + "out": "bf16" + }, + "sources": [ + "moonshotai/Kimi-K3/self_attn.q_proj+self_attn.k_proj+self_attn.v_proj+self_attn.g_proj+self_attn.f_a_proj+self_attn.b_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 256, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 7168, + "n": 49376, + "out": "bf16" + }, + "sources": [ + "moonshotai/Kimi-K3/self_attn.q_proj+self_attn.k_proj+self_attn.v_proj+self_attn.g_proj+self_attn.f_a_proj+self_attn.b_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 384, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 7168, + "n": 49376, + "out": "bf16" + }, + "sources": [ + "moonshotai/Kimi-K3/self_attn.q_proj+self_attn.k_proj+self_attn.v_proj+self_attn.g_proj+self_attn.f_a_proj+self_attn.b_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 512, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 7168, + "n": 49376, + "out": "bf16" + }, + "sources": [ + "moonshotai/Kimi-K3/self_attn.q_proj+self_attn.k_proj+self_attn.v_proj+self_attn.g_proj+self_attn.f_a_proj+self_attn.b_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 768, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 7168, + "n": 49376, + "out": "bf16" + }, + "sources": [ + "moonshotai/Kimi-K3/self_attn.q_proj+self_attn.k_proj+self_attn.v_proj+self_attn.g_proj+self_attn.f_a_proj+self_attn.b_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 1024, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 7168, + "n": 49376, + "out": "bf16" + }, + "sources": [ + "moonshotai/Kimi-K3/self_attn.q_proj+self_attn.k_proj+self_attn.v_proj+self_attn.g_proj+self_attn.f_a_proj+self_attn.b_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 1536, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 7168, + "n": 49376, + "out": "bf16" + }, + "sources": [ + "moonshotai/Kimi-K3/self_attn.q_proj+self_attn.k_proj+self_attn.v_proj+self_attn.g_proj+self_attn.f_a_proj+self_attn.b_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 2048, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 7168, + "n": 49376, + "out": "bf16" + }, + "sources": [ + "moonshotai/Kimi-K3/self_attn.q_proj+self_attn.k_proj+self_attn.v_proj+self_attn.g_proj+self_attn.f_a_proj+self_attn.b_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 3072, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 7168, + "n": 49376, + "out": "bf16" + }, + "sources": [ + "moonshotai/Kimi-K3/self_attn.q_proj+self_attn.k_proj+self_attn.v_proj+self_attn.g_proj+self_attn.f_a_proj+self_attn.b_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 4096, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 7168, + "n": 49376, + "out": "bf16" + }, + "sources": [ + "moonshotai/Kimi-K3/self_attn.q_proj+self_attn.k_proj+self_attn.v_proj+self_attn.g_proj+self_attn.f_a_proj+self_attn.b_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 6144, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 7168, + "n": 49376, + "out": "bf16" + }, + "sources": [ + "moonshotai/Kimi-K3/self_attn.q_proj+self_attn.k_proj+self_attn.v_proj+self_attn.g_proj+self_attn.f_a_proj+self_attn.b_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 8192, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 7168, + "n": 49376, + "out": "bf16" + }, + "sources": [ + "moonshotai/Kimi-K3/self_attn.q_proj+self_attn.k_proj+self_attn.v_proj+self_attn.g_proj+self_attn.f_a_proj+self_attn.b_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 12288, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 7168, + "n": 49376, + "out": "bf16" + }, + "sources": [ + "moonshotai/Kimi-K3/self_attn.q_proj+self_attn.k_proj+self_attn.v_proj+self_attn.g_proj+self_attn.f_a_proj+self_attn.b_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 16384, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 7168, + "n": 49376, + "out": "bf16" + }, + "sources": [ + "moonshotai/Kimi-K3/self_attn.q_proj+self_attn.k_proj+self_attn.v_proj+self_attn.g_proj+self_attn.f_a_proj+self_attn.b_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 24576, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 7168, + "n": 49376, + "out": "bf16" + }, + "sources": [ + "moonshotai/Kimi-K3/self_attn.q_proj+self_attn.k_proj+self_attn.v_proj+self_attn.g_proj+self_attn.f_a_proj+self_attn.b_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 32768, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 7168, + "n": 49376, + "out": "bf16" + }, + "sources": [ + "moonshotai/Kimi-K3/self_attn.q_proj+self_attn.k_proj+self_attn.v_proj+self_attn.g_proj+self_attn.f_a_proj+self_attn.b_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 49152, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 7168, + "n": 49376, + "out": "bf16" + }, + "sources": [ + "moonshotai/Kimi-K3/self_attn.q_proj+self_attn.k_proj+self_attn.v_proj+self_attn.g_proj+self_attn.f_a_proj+self_attn.b_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 65536, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 7168, + "n": 49376, + "out": "bf16" + }, + "sources": [ + "moonshotai/Kimi-K3/self_attn.q_proj+self_attn.k_proj+self_attn.v_proj+self_attn.g_proj+self_attn.f_a_proj+self_attn.b_proj" + ] + } +] diff --git a/operatorx/testlists/gemm_minimax_m3.json b/operatorx/testlists/gemm_minimax_m3.json new file mode 100644 index 0000000000..1453100727 --- /dev/null +++ b/operatorx/testlists/gemm_minimax_m3.json @@ -0,0 +1,18946 @@ +[ + { + "type": "gemm", + "args": { + "m": 1, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 6144, + "n": 128, + "out": "bf16" + }, + "sources": [ + "MiniMaxAI/MiniMax-M3-MXFP8/block_sparse_moe.gate", + "amd/MiniMax-M3-MXFP4/block_sparse_moe.gate", + "amd/MiniMax-M3-MXFP4/self_attn.index_k_proj", + "nvidia/MiniMax-M3-NVFP4/block_sparse_moe.gate" + ] + }, + { + "type": "gemm", + "args": { + "m": 2, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 6144, + "n": 128, + "out": "bf16" + }, + "sources": [ + "MiniMaxAI/MiniMax-M3-MXFP8/block_sparse_moe.gate", + "amd/MiniMax-M3-MXFP4/block_sparse_moe.gate", + "amd/MiniMax-M3-MXFP4/self_attn.index_k_proj", + "nvidia/MiniMax-M3-NVFP4/block_sparse_moe.gate" + ] + }, + { + "type": "gemm", + "args": { + "m": 3, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 6144, + "n": 128, + "out": "bf16" + }, + "sources": [ + "MiniMaxAI/MiniMax-M3-MXFP8/block_sparse_moe.gate", + "amd/MiniMax-M3-MXFP4/block_sparse_moe.gate", + "amd/MiniMax-M3-MXFP4/self_attn.index_k_proj", + "nvidia/MiniMax-M3-NVFP4/block_sparse_moe.gate" + ] + }, + { + "type": "gemm", + "args": { + "m": 4, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 6144, + "n": 128, + "out": "bf16" + }, + "sources": [ + "MiniMaxAI/MiniMax-M3-MXFP8/block_sparse_moe.gate", + "amd/MiniMax-M3-MXFP4/block_sparse_moe.gate", + "amd/MiniMax-M3-MXFP4/self_attn.index_k_proj", + "nvidia/MiniMax-M3-NVFP4/block_sparse_moe.gate" + ] + }, + { + "type": "gemm", + "args": { + "m": 6, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 6144, + "n": 128, + "out": "bf16" + }, + "sources": [ + "MiniMaxAI/MiniMax-M3-MXFP8/block_sparse_moe.gate", + "amd/MiniMax-M3-MXFP4/block_sparse_moe.gate", + "amd/MiniMax-M3-MXFP4/self_attn.index_k_proj", + "nvidia/MiniMax-M3-NVFP4/block_sparse_moe.gate" + ] + }, + { + "type": "gemm", + "args": { + "m": 8, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 6144, + "n": 128, + "out": "bf16" + }, + "sources": [ + "MiniMaxAI/MiniMax-M3-MXFP8/block_sparse_moe.gate", + "amd/MiniMax-M3-MXFP4/block_sparse_moe.gate", + "amd/MiniMax-M3-MXFP4/self_attn.index_k_proj", + "nvidia/MiniMax-M3-NVFP4/block_sparse_moe.gate" + ] + }, + { + "type": "gemm", + "args": { + "m": 12, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 6144, + "n": 128, + "out": "bf16" + }, + "sources": [ + "MiniMaxAI/MiniMax-M3-MXFP8/block_sparse_moe.gate", + "amd/MiniMax-M3-MXFP4/block_sparse_moe.gate", + "amd/MiniMax-M3-MXFP4/self_attn.index_k_proj", + "nvidia/MiniMax-M3-NVFP4/block_sparse_moe.gate" + ] + }, + { + "type": "gemm", + "args": { + "m": 16, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 6144, + "n": 128, + "out": "bf16" + }, + "sources": [ + "MiniMaxAI/MiniMax-M3-MXFP8/block_sparse_moe.gate", + "amd/MiniMax-M3-MXFP4/block_sparse_moe.gate", + "amd/MiniMax-M3-MXFP4/self_attn.index_k_proj", + "nvidia/MiniMax-M3-NVFP4/block_sparse_moe.gate" + ] + }, + { + "type": "gemm", + "args": { + "m": 24, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 6144, + "n": 128, + "out": "bf16" + }, + "sources": [ + "MiniMaxAI/MiniMax-M3-MXFP8/block_sparse_moe.gate", + "amd/MiniMax-M3-MXFP4/block_sparse_moe.gate", + "amd/MiniMax-M3-MXFP4/self_attn.index_k_proj", + "nvidia/MiniMax-M3-NVFP4/block_sparse_moe.gate" + ] + }, + { + "type": "gemm", + "args": { + "m": 32, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 6144, + "n": 128, + "out": "bf16" + }, + "sources": [ + "MiniMaxAI/MiniMax-M3-MXFP8/block_sparse_moe.gate", + "amd/MiniMax-M3-MXFP4/block_sparse_moe.gate", + "amd/MiniMax-M3-MXFP4/self_attn.index_k_proj", + "nvidia/MiniMax-M3-NVFP4/block_sparse_moe.gate" + ] + }, + { + "type": "gemm", + "args": { + "m": 48, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 6144, + "n": 128, + "out": "bf16" + }, + "sources": [ + "MiniMaxAI/MiniMax-M3-MXFP8/block_sparse_moe.gate", + "amd/MiniMax-M3-MXFP4/block_sparse_moe.gate", + "amd/MiniMax-M3-MXFP4/self_attn.index_k_proj", + "nvidia/MiniMax-M3-NVFP4/block_sparse_moe.gate" + ] + }, + { + "type": "gemm", + "args": { + "m": 64, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 6144, + "n": 128, + "out": "bf16" + }, + "sources": [ + "MiniMaxAI/MiniMax-M3-MXFP8/block_sparse_moe.gate", + "amd/MiniMax-M3-MXFP4/block_sparse_moe.gate", + "amd/MiniMax-M3-MXFP4/self_attn.index_k_proj", + "nvidia/MiniMax-M3-NVFP4/block_sparse_moe.gate" + ] + }, + { + "type": "gemm", + "args": { + "m": 96, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 6144, + "n": 128, + "out": "bf16" + }, + "sources": [ + "MiniMaxAI/MiniMax-M3-MXFP8/block_sparse_moe.gate", + "amd/MiniMax-M3-MXFP4/block_sparse_moe.gate", + "amd/MiniMax-M3-MXFP4/self_attn.index_k_proj", + "nvidia/MiniMax-M3-NVFP4/block_sparse_moe.gate" + ] + }, + { + "type": "gemm", + "args": { + "m": 128, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 6144, + "n": 128, + "out": "bf16" + }, + "sources": [ + "MiniMaxAI/MiniMax-M3-MXFP8/block_sparse_moe.gate", + "amd/MiniMax-M3-MXFP4/block_sparse_moe.gate", + "amd/MiniMax-M3-MXFP4/self_attn.index_k_proj", + "nvidia/MiniMax-M3-NVFP4/block_sparse_moe.gate" + ] + }, + { + "type": "gemm", + "args": { + "m": 192, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 6144, + "n": 128, + "out": "bf16" + }, + "sources": [ + "MiniMaxAI/MiniMax-M3-MXFP8/block_sparse_moe.gate", + "amd/MiniMax-M3-MXFP4/block_sparse_moe.gate", + "amd/MiniMax-M3-MXFP4/self_attn.index_k_proj", + "nvidia/MiniMax-M3-NVFP4/block_sparse_moe.gate" + ] + }, + { + "type": "gemm", + "args": { + "m": 256, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 6144, + "n": 128, + "out": "bf16" + }, + "sources": [ + "MiniMaxAI/MiniMax-M3-MXFP8/block_sparse_moe.gate", + "amd/MiniMax-M3-MXFP4/block_sparse_moe.gate", + "amd/MiniMax-M3-MXFP4/self_attn.index_k_proj", + "nvidia/MiniMax-M3-NVFP4/block_sparse_moe.gate" + ] + }, + { + "type": "gemm", + "args": { + "m": 384, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 6144, + "n": 128, + "out": "bf16" + }, + "sources": [ + "MiniMaxAI/MiniMax-M3-MXFP8/block_sparse_moe.gate", + "amd/MiniMax-M3-MXFP4/block_sparse_moe.gate", + "amd/MiniMax-M3-MXFP4/self_attn.index_k_proj", + "nvidia/MiniMax-M3-NVFP4/block_sparse_moe.gate" + ] + }, + { + "type": "gemm", + "args": { + "m": 512, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 6144, + "n": 128, + "out": "bf16" + }, + "sources": [ + "MiniMaxAI/MiniMax-M3-MXFP8/block_sparse_moe.gate", + "amd/MiniMax-M3-MXFP4/block_sparse_moe.gate", + "amd/MiniMax-M3-MXFP4/self_attn.index_k_proj", + "nvidia/MiniMax-M3-NVFP4/block_sparse_moe.gate" + ] + }, + { + "type": "gemm", + "args": { + "m": 768, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 6144, + "n": 128, + "out": "bf16" + }, + "sources": [ + "MiniMaxAI/MiniMax-M3-MXFP8/block_sparse_moe.gate", + "amd/MiniMax-M3-MXFP4/block_sparse_moe.gate", + "amd/MiniMax-M3-MXFP4/self_attn.index_k_proj", + "nvidia/MiniMax-M3-NVFP4/block_sparse_moe.gate" + ] + }, + { + "type": "gemm", + "args": { + "m": 1024, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 6144, + "n": 128, + "out": "bf16" + }, + "sources": [ + "MiniMaxAI/MiniMax-M3-MXFP8/block_sparse_moe.gate", + "amd/MiniMax-M3-MXFP4/block_sparse_moe.gate", + "amd/MiniMax-M3-MXFP4/self_attn.index_k_proj", + "nvidia/MiniMax-M3-NVFP4/block_sparse_moe.gate" + ] + }, + { + "type": "gemm", + "args": { + "m": 1536, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 6144, + "n": 128, + "out": "bf16" + }, + "sources": [ + "MiniMaxAI/MiniMax-M3-MXFP8/block_sparse_moe.gate", + "amd/MiniMax-M3-MXFP4/block_sparse_moe.gate", + "amd/MiniMax-M3-MXFP4/self_attn.index_k_proj", + "nvidia/MiniMax-M3-NVFP4/block_sparse_moe.gate" + ] + }, + { + "type": "gemm", + "args": { + "m": 2048, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 6144, + "n": 128, + "out": "bf16" + }, + "sources": [ + "MiniMaxAI/MiniMax-M3-MXFP8/block_sparse_moe.gate", + "amd/MiniMax-M3-MXFP4/block_sparse_moe.gate", + "amd/MiniMax-M3-MXFP4/self_attn.index_k_proj", + "nvidia/MiniMax-M3-NVFP4/block_sparse_moe.gate" + ] + }, + { + "type": "gemm", + "args": { + "m": 3072, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 6144, + "n": 128, + "out": "bf16" + }, + "sources": [ + "MiniMaxAI/MiniMax-M3-MXFP8/block_sparse_moe.gate", + "amd/MiniMax-M3-MXFP4/block_sparse_moe.gate", + "amd/MiniMax-M3-MXFP4/self_attn.index_k_proj", + "nvidia/MiniMax-M3-NVFP4/block_sparse_moe.gate" + ] + }, + { + "type": "gemm", + "args": { + "m": 4096, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 6144, + "n": 128, + "out": "bf16" + }, + "sources": [ + "MiniMaxAI/MiniMax-M3-MXFP8/block_sparse_moe.gate", + "amd/MiniMax-M3-MXFP4/block_sparse_moe.gate", + "amd/MiniMax-M3-MXFP4/self_attn.index_k_proj", + "nvidia/MiniMax-M3-NVFP4/block_sparse_moe.gate" + ] + }, + { + "type": "gemm", + "args": { + "m": 6144, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 6144, + "n": 128, + "out": "bf16" + }, + "sources": [ + "MiniMaxAI/MiniMax-M3-MXFP8/block_sparse_moe.gate", + "amd/MiniMax-M3-MXFP4/block_sparse_moe.gate", + "amd/MiniMax-M3-MXFP4/self_attn.index_k_proj", + "nvidia/MiniMax-M3-NVFP4/block_sparse_moe.gate" + ] + }, + { + "type": "gemm", + "args": { + "m": 8192, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 6144, + "n": 128, + "out": "bf16" + }, + "sources": [ + "MiniMaxAI/MiniMax-M3-MXFP8/block_sparse_moe.gate", + "amd/MiniMax-M3-MXFP4/block_sparse_moe.gate", + "amd/MiniMax-M3-MXFP4/self_attn.index_k_proj", + "nvidia/MiniMax-M3-NVFP4/block_sparse_moe.gate" + ] + }, + { + "type": "gemm", + "args": { + "m": 12288, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 6144, + "n": 128, + "out": "bf16" + }, + "sources": [ + "MiniMaxAI/MiniMax-M3-MXFP8/block_sparse_moe.gate", + "amd/MiniMax-M3-MXFP4/block_sparse_moe.gate", + "amd/MiniMax-M3-MXFP4/self_attn.index_k_proj", + "nvidia/MiniMax-M3-NVFP4/block_sparse_moe.gate" + ] + }, + { + "type": "gemm", + "args": { + "m": 16384, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 6144, + "n": 128, + "out": "bf16" + }, + "sources": [ + "MiniMaxAI/MiniMax-M3-MXFP8/block_sparse_moe.gate", + "amd/MiniMax-M3-MXFP4/block_sparse_moe.gate", + "amd/MiniMax-M3-MXFP4/self_attn.index_k_proj", + "nvidia/MiniMax-M3-NVFP4/block_sparse_moe.gate" + ] + }, + { + "type": "gemm", + "args": { + "m": 24576, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 6144, + "n": 128, + "out": "bf16" + }, + "sources": [ + "MiniMaxAI/MiniMax-M3-MXFP8/block_sparse_moe.gate", + "amd/MiniMax-M3-MXFP4/block_sparse_moe.gate", + "amd/MiniMax-M3-MXFP4/self_attn.index_k_proj", + "nvidia/MiniMax-M3-NVFP4/block_sparse_moe.gate" + ] + }, + { + "type": "gemm", + "args": { + "m": 32768, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 6144, + "n": 128, + "out": "bf16" + }, + "sources": [ + "MiniMaxAI/MiniMax-M3-MXFP8/block_sparse_moe.gate", + "amd/MiniMax-M3-MXFP4/block_sparse_moe.gate", + "amd/MiniMax-M3-MXFP4/self_attn.index_k_proj", + "nvidia/MiniMax-M3-NVFP4/block_sparse_moe.gate" + ] + }, + { + "type": "gemm", + "args": { + "m": 49152, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 6144, + "n": 128, + "out": "bf16" + }, + "sources": [ + "MiniMaxAI/MiniMax-M3-MXFP8/block_sparse_moe.gate", + "amd/MiniMax-M3-MXFP4/block_sparse_moe.gate", + "amd/MiniMax-M3-MXFP4/self_attn.index_k_proj", + "nvidia/MiniMax-M3-NVFP4/block_sparse_moe.gate" + ] + }, + { + "type": "gemm", + "args": { + "m": 65536, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 6144, + "n": 128, + "out": "bf16" + }, + "sources": [ + "MiniMaxAI/MiniMax-M3-MXFP8/block_sparse_moe.gate", + "amd/MiniMax-M3-MXFP4/block_sparse_moe.gate", + "amd/MiniMax-M3-MXFP4/self_attn.index_k_proj", + "nvidia/MiniMax-M3-NVFP4/block_sparse_moe.gate" + ] + }, + { + "type": "gemm", + "args": { + "m": 1, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 6144, + "n": 128, + "out": "fp32" + }, + "sources": [ + "MiniMaxAI/MiniMax-M3-MXFP8/block_sparse_moe.gate", + "amd/MiniMax-M3-MXFP4/block_sparse_moe.gate", + "nvidia/MiniMax-M3-NVFP4/block_sparse_moe.gate" + ] + }, + { + "type": "gemm", + "args": { + "m": 2, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 6144, + "n": 128, + "out": "fp32" + }, + "sources": [ + "MiniMaxAI/MiniMax-M3-MXFP8/block_sparse_moe.gate", + "amd/MiniMax-M3-MXFP4/block_sparse_moe.gate", + "nvidia/MiniMax-M3-NVFP4/block_sparse_moe.gate" + ] + }, + { + "type": "gemm", + "args": { + "m": 3, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 6144, + "n": 128, + "out": "fp32" + }, + "sources": [ + "MiniMaxAI/MiniMax-M3-MXFP8/block_sparse_moe.gate", + "amd/MiniMax-M3-MXFP4/block_sparse_moe.gate", + "nvidia/MiniMax-M3-NVFP4/block_sparse_moe.gate" + ] + }, + { + "type": "gemm", + "args": { + "m": 4, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 6144, + "n": 128, + "out": "fp32" + }, + "sources": [ + "MiniMaxAI/MiniMax-M3-MXFP8/block_sparse_moe.gate", + "amd/MiniMax-M3-MXFP4/block_sparse_moe.gate", + "nvidia/MiniMax-M3-NVFP4/block_sparse_moe.gate" + ] + }, + { + "type": "gemm", + "args": { + "m": 6, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 6144, + "n": 128, + "out": "fp32" + }, + "sources": [ + "MiniMaxAI/MiniMax-M3-MXFP8/block_sparse_moe.gate", + "amd/MiniMax-M3-MXFP4/block_sparse_moe.gate", + "nvidia/MiniMax-M3-NVFP4/block_sparse_moe.gate" + ] + }, + { + "type": "gemm", + "args": { + "m": 8, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 6144, + "n": 128, + "out": "fp32" + }, + "sources": [ + "MiniMaxAI/MiniMax-M3-MXFP8/block_sparse_moe.gate", + "amd/MiniMax-M3-MXFP4/block_sparse_moe.gate", + "nvidia/MiniMax-M3-NVFP4/block_sparse_moe.gate" + ] + }, + { + "type": "gemm", + "args": { + "m": 12, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 6144, + "n": 128, + "out": "fp32" + }, + "sources": [ + "MiniMaxAI/MiniMax-M3-MXFP8/block_sparse_moe.gate", + "amd/MiniMax-M3-MXFP4/block_sparse_moe.gate", + "nvidia/MiniMax-M3-NVFP4/block_sparse_moe.gate" + ] + }, + { + "type": "gemm", + "args": { + "m": 16, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 6144, + "n": 128, + "out": "fp32" + }, + "sources": [ + "MiniMaxAI/MiniMax-M3-MXFP8/block_sparse_moe.gate", + "amd/MiniMax-M3-MXFP4/block_sparse_moe.gate", + "nvidia/MiniMax-M3-NVFP4/block_sparse_moe.gate" + ] + }, + { + "type": "gemm", + "args": { + "m": 24, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 6144, + "n": 128, + "out": "fp32" + }, + "sources": [ + "MiniMaxAI/MiniMax-M3-MXFP8/block_sparse_moe.gate", + "amd/MiniMax-M3-MXFP4/block_sparse_moe.gate", + "nvidia/MiniMax-M3-NVFP4/block_sparse_moe.gate" + ] + }, + { + "type": "gemm", + "args": { + "m": 32, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 6144, + "n": 128, + "out": "fp32" + }, + "sources": [ + "MiniMaxAI/MiniMax-M3-MXFP8/block_sparse_moe.gate", + "amd/MiniMax-M3-MXFP4/block_sparse_moe.gate", + "nvidia/MiniMax-M3-NVFP4/block_sparse_moe.gate" + ] + }, + { + "type": "gemm", + "args": { + "m": 48, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 6144, + "n": 128, + "out": "fp32" + }, + "sources": [ + "MiniMaxAI/MiniMax-M3-MXFP8/block_sparse_moe.gate", + "amd/MiniMax-M3-MXFP4/block_sparse_moe.gate", + "nvidia/MiniMax-M3-NVFP4/block_sparse_moe.gate" + ] + }, + { + "type": "gemm", + "args": { + "m": 64, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 6144, + "n": 128, + "out": "fp32" + }, + "sources": [ + "MiniMaxAI/MiniMax-M3-MXFP8/block_sparse_moe.gate", + "amd/MiniMax-M3-MXFP4/block_sparse_moe.gate", + "nvidia/MiniMax-M3-NVFP4/block_sparse_moe.gate" + ] + }, + { + "type": "gemm", + "args": { + "m": 96, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 6144, + "n": 128, + "out": "fp32" + }, + "sources": [ + "MiniMaxAI/MiniMax-M3-MXFP8/block_sparse_moe.gate", + "amd/MiniMax-M3-MXFP4/block_sparse_moe.gate", + "nvidia/MiniMax-M3-NVFP4/block_sparse_moe.gate" + ] + }, + { + "type": "gemm", + "args": { + "m": 128, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 6144, + "n": 128, + "out": "fp32" + }, + "sources": [ + "MiniMaxAI/MiniMax-M3-MXFP8/block_sparse_moe.gate", + "amd/MiniMax-M3-MXFP4/block_sparse_moe.gate", + "nvidia/MiniMax-M3-NVFP4/block_sparse_moe.gate" + ] + }, + { + "type": "gemm", + "args": { + "m": 192, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 6144, + "n": 128, + "out": "fp32" + }, + "sources": [ + "MiniMaxAI/MiniMax-M3-MXFP8/block_sparse_moe.gate", + "amd/MiniMax-M3-MXFP4/block_sparse_moe.gate", + "nvidia/MiniMax-M3-NVFP4/block_sparse_moe.gate" + ] + }, + { + "type": "gemm", + "args": { + "m": 256, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 6144, + "n": 128, + "out": "fp32" + }, + "sources": [ + "MiniMaxAI/MiniMax-M3-MXFP8/block_sparse_moe.gate", + "amd/MiniMax-M3-MXFP4/block_sparse_moe.gate", + "nvidia/MiniMax-M3-NVFP4/block_sparse_moe.gate" + ] + }, + { + "type": "gemm", + "args": { + "m": 384, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 6144, + "n": 128, + "out": "fp32" + }, + "sources": [ + "MiniMaxAI/MiniMax-M3-MXFP8/block_sparse_moe.gate", + "amd/MiniMax-M3-MXFP4/block_sparse_moe.gate", + "nvidia/MiniMax-M3-NVFP4/block_sparse_moe.gate" + ] + }, + { + "type": "gemm", + "args": { + "m": 512, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 6144, + "n": 128, + "out": "fp32" + }, + "sources": [ + "MiniMaxAI/MiniMax-M3-MXFP8/block_sparse_moe.gate", + "amd/MiniMax-M3-MXFP4/block_sparse_moe.gate", + "nvidia/MiniMax-M3-NVFP4/block_sparse_moe.gate" + ] + }, + { + "type": "gemm", + "args": { + "m": 768, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 6144, + "n": 128, + "out": "fp32" + }, + "sources": [ + "MiniMaxAI/MiniMax-M3-MXFP8/block_sparse_moe.gate", + "amd/MiniMax-M3-MXFP4/block_sparse_moe.gate", + "nvidia/MiniMax-M3-NVFP4/block_sparse_moe.gate" + ] + }, + { + "type": "gemm", + "args": { + "m": 1024, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 6144, + "n": 128, + "out": "fp32" + }, + "sources": [ + "MiniMaxAI/MiniMax-M3-MXFP8/block_sparse_moe.gate", + "amd/MiniMax-M3-MXFP4/block_sparse_moe.gate", + "nvidia/MiniMax-M3-NVFP4/block_sparse_moe.gate" + ] + }, + { + "type": "gemm", + "args": { + "m": 1536, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 6144, + "n": 128, + "out": "fp32" + }, + "sources": [ + "MiniMaxAI/MiniMax-M3-MXFP8/block_sparse_moe.gate", + "amd/MiniMax-M3-MXFP4/block_sparse_moe.gate", + "nvidia/MiniMax-M3-NVFP4/block_sparse_moe.gate" + ] + }, + { + "type": "gemm", + "args": { + "m": 2048, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 6144, + "n": 128, + "out": "fp32" + }, + "sources": [ + "MiniMaxAI/MiniMax-M3-MXFP8/block_sparse_moe.gate", + "amd/MiniMax-M3-MXFP4/block_sparse_moe.gate", + "nvidia/MiniMax-M3-NVFP4/block_sparse_moe.gate" + ] + }, + { + "type": "gemm", + "args": { + "m": 3072, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 6144, + "n": 128, + "out": "fp32" + }, + "sources": [ + "MiniMaxAI/MiniMax-M3-MXFP8/block_sparse_moe.gate", + "amd/MiniMax-M3-MXFP4/block_sparse_moe.gate", + "nvidia/MiniMax-M3-NVFP4/block_sparse_moe.gate" + ] + }, + { + "type": "gemm", + "args": { + "m": 4096, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 6144, + "n": 128, + "out": "fp32" + }, + "sources": [ + "MiniMaxAI/MiniMax-M3-MXFP8/block_sparse_moe.gate", + "amd/MiniMax-M3-MXFP4/block_sparse_moe.gate", + "nvidia/MiniMax-M3-NVFP4/block_sparse_moe.gate" + ] + }, + { + "type": "gemm", + "args": { + "m": 6144, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 6144, + "n": 128, + "out": "fp32" + }, + "sources": [ + "MiniMaxAI/MiniMax-M3-MXFP8/block_sparse_moe.gate", + "amd/MiniMax-M3-MXFP4/block_sparse_moe.gate", + "nvidia/MiniMax-M3-NVFP4/block_sparse_moe.gate" + ] + }, + { + "type": "gemm", + "args": { + "m": 8192, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 6144, + "n": 128, + "out": "fp32" + }, + "sources": [ + "MiniMaxAI/MiniMax-M3-MXFP8/block_sparse_moe.gate", + "amd/MiniMax-M3-MXFP4/block_sparse_moe.gate", + "nvidia/MiniMax-M3-NVFP4/block_sparse_moe.gate" + ] + }, + { + "type": "gemm", + "args": { + "m": 12288, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 6144, + "n": 128, + "out": "fp32" + }, + "sources": [ + "MiniMaxAI/MiniMax-M3-MXFP8/block_sparse_moe.gate", + "amd/MiniMax-M3-MXFP4/block_sparse_moe.gate", + "nvidia/MiniMax-M3-NVFP4/block_sparse_moe.gate" + ] + }, + { + "type": "gemm", + "args": { + "m": 16384, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 6144, + "n": 128, + "out": "fp32" + }, + "sources": [ + "MiniMaxAI/MiniMax-M3-MXFP8/block_sparse_moe.gate", + "amd/MiniMax-M3-MXFP4/block_sparse_moe.gate", + "nvidia/MiniMax-M3-NVFP4/block_sparse_moe.gate" + ] + }, + { + "type": "gemm", + "args": { + "m": 24576, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 6144, + "n": 128, + "out": "fp32" + }, + "sources": [ + "MiniMaxAI/MiniMax-M3-MXFP8/block_sparse_moe.gate", + "amd/MiniMax-M3-MXFP4/block_sparse_moe.gate", + "nvidia/MiniMax-M3-NVFP4/block_sparse_moe.gate" + ] + }, + { + "type": "gemm", + "args": { + "m": 32768, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 6144, + "n": 128, + "out": "fp32" + }, + "sources": [ + "MiniMaxAI/MiniMax-M3-MXFP8/block_sparse_moe.gate", + "amd/MiniMax-M3-MXFP4/block_sparse_moe.gate", + "nvidia/MiniMax-M3-NVFP4/block_sparse_moe.gate" + ] + }, + { + "type": "gemm", + "args": { + "m": 49152, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 6144, + "n": 128, + "out": "fp32" + }, + "sources": [ + "MiniMaxAI/MiniMax-M3-MXFP8/block_sparse_moe.gate", + "amd/MiniMax-M3-MXFP4/block_sparse_moe.gate", + "nvidia/MiniMax-M3-NVFP4/block_sparse_moe.gate" + ] + }, + { + "type": "gemm", + "args": { + "m": 65536, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 6144, + "n": 128, + "out": "fp32" + }, + "sources": [ + "MiniMaxAI/MiniMax-M3-MXFP8/block_sparse_moe.gate", + "amd/MiniMax-M3-MXFP4/block_sparse_moe.gate", + "nvidia/MiniMax-M3-NVFP4/block_sparse_moe.gate" + ] + }, + { + "type": "gemm", + "args": { + "m": 1, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "fp32" + }, + "k": 6144, + "n": 128, + "out": "fp32" + }, + "sources": [ + "MiniMaxAI/MiniMax-M3-MXFP8/block_sparse_moe.gate", + "amd/MiniMax-M3-MXFP4/block_sparse_moe.gate", + "nvidia/MiniMax-M3-NVFP4/block_sparse_moe.gate" + ] + }, + { + "type": "gemm", + "args": { + "m": 2, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "fp32" + }, + "k": 6144, + "n": 128, + "out": "fp32" + }, + "sources": [ + "MiniMaxAI/MiniMax-M3-MXFP8/block_sparse_moe.gate", + "amd/MiniMax-M3-MXFP4/block_sparse_moe.gate", + "nvidia/MiniMax-M3-NVFP4/block_sparse_moe.gate" + ] + }, + { + "type": "gemm", + "args": { + "m": 3, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "fp32" + }, + "k": 6144, + "n": 128, + "out": "fp32" + }, + "sources": [ + "MiniMaxAI/MiniMax-M3-MXFP8/block_sparse_moe.gate", + "amd/MiniMax-M3-MXFP4/block_sparse_moe.gate", + "nvidia/MiniMax-M3-NVFP4/block_sparse_moe.gate" + ] + }, + { + "type": "gemm", + "args": { + "m": 4, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "fp32" + }, + "k": 6144, + "n": 128, + "out": "fp32" + }, + "sources": [ + "MiniMaxAI/MiniMax-M3-MXFP8/block_sparse_moe.gate", + "amd/MiniMax-M3-MXFP4/block_sparse_moe.gate", + "nvidia/MiniMax-M3-NVFP4/block_sparse_moe.gate" + ] + }, + { + "type": "gemm", + "args": { + "m": 6, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "fp32" + }, + "k": 6144, + "n": 128, + "out": "fp32" + }, + "sources": [ + "MiniMaxAI/MiniMax-M3-MXFP8/block_sparse_moe.gate", + "amd/MiniMax-M3-MXFP4/block_sparse_moe.gate", + "nvidia/MiniMax-M3-NVFP4/block_sparse_moe.gate" + ] + }, + { + "type": "gemm", + "args": { + "m": 8, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "fp32" + }, + "k": 6144, + "n": 128, + "out": "fp32" + }, + "sources": [ + "MiniMaxAI/MiniMax-M3-MXFP8/block_sparse_moe.gate", + "amd/MiniMax-M3-MXFP4/block_sparse_moe.gate", + "nvidia/MiniMax-M3-NVFP4/block_sparse_moe.gate" + ] + }, + { + "type": "gemm", + "args": { + "m": 12, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "fp32" + }, + "k": 6144, + "n": 128, + "out": "fp32" + }, + "sources": [ + "MiniMaxAI/MiniMax-M3-MXFP8/block_sparse_moe.gate", + "amd/MiniMax-M3-MXFP4/block_sparse_moe.gate", + "nvidia/MiniMax-M3-NVFP4/block_sparse_moe.gate" + ] + }, + { + "type": "gemm", + "args": { + "m": 16, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "fp32" + }, + "k": 6144, + "n": 128, + "out": "fp32" + }, + "sources": [ + "MiniMaxAI/MiniMax-M3-MXFP8/block_sparse_moe.gate", + "amd/MiniMax-M3-MXFP4/block_sparse_moe.gate", + "nvidia/MiniMax-M3-NVFP4/block_sparse_moe.gate" + ] + }, + { + "type": "gemm", + "args": { + "m": 24, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "fp32" + }, + "k": 6144, + "n": 128, + "out": "fp32" + }, + "sources": [ + "MiniMaxAI/MiniMax-M3-MXFP8/block_sparse_moe.gate", + "amd/MiniMax-M3-MXFP4/block_sparse_moe.gate", + "nvidia/MiniMax-M3-NVFP4/block_sparse_moe.gate" + ] + }, + { + "type": "gemm", + "args": { + "m": 32, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "fp32" + }, + "k": 6144, + "n": 128, + "out": "fp32" + }, + "sources": [ + "MiniMaxAI/MiniMax-M3-MXFP8/block_sparse_moe.gate", + "amd/MiniMax-M3-MXFP4/block_sparse_moe.gate", + "nvidia/MiniMax-M3-NVFP4/block_sparse_moe.gate" + ] + }, + { + "type": "gemm", + "args": { + "m": 48, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "fp32" + }, + "k": 6144, + "n": 128, + "out": "fp32" + }, + "sources": [ + "MiniMaxAI/MiniMax-M3-MXFP8/block_sparse_moe.gate", + "amd/MiniMax-M3-MXFP4/block_sparse_moe.gate", + "nvidia/MiniMax-M3-NVFP4/block_sparse_moe.gate" + ] + }, + { + "type": "gemm", + "args": { + "m": 64, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "fp32" + }, + "k": 6144, + "n": 128, + "out": "fp32" + }, + "sources": [ + "MiniMaxAI/MiniMax-M3-MXFP8/block_sparse_moe.gate", + "amd/MiniMax-M3-MXFP4/block_sparse_moe.gate", + "nvidia/MiniMax-M3-NVFP4/block_sparse_moe.gate" + ] + }, + { + "type": "gemm", + "args": { + "m": 96, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "fp32" + }, + "k": 6144, + "n": 128, + "out": "fp32" + }, + "sources": [ + "MiniMaxAI/MiniMax-M3-MXFP8/block_sparse_moe.gate", + "amd/MiniMax-M3-MXFP4/block_sparse_moe.gate", + "nvidia/MiniMax-M3-NVFP4/block_sparse_moe.gate" + ] + }, + { + "type": "gemm", + "args": { + "m": 128, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "fp32" + }, + "k": 6144, + "n": 128, + "out": "fp32" + }, + "sources": [ + "MiniMaxAI/MiniMax-M3-MXFP8/block_sparse_moe.gate", + "amd/MiniMax-M3-MXFP4/block_sparse_moe.gate", + "nvidia/MiniMax-M3-NVFP4/block_sparse_moe.gate" + ] + }, + { + "type": "gemm", + "args": { + "m": 192, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "fp32" + }, + "k": 6144, + "n": 128, + "out": "fp32" + }, + "sources": [ + "MiniMaxAI/MiniMax-M3-MXFP8/block_sparse_moe.gate", + "amd/MiniMax-M3-MXFP4/block_sparse_moe.gate", + "nvidia/MiniMax-M3-NVFP4/block_sparse_moe.gate" + ] + }, + { + "type": "gemm", + "args": { + "m": 256, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "fp32" + }, + "k": 6144, + "n": 128, + "out": "fp32" + }, + "sources": [ + "MiniMaxAI/MiniMax-M3-MXFP8/block_sparse_moe.gate", + "amd/MiniMax-M3-MXFP4/block_sparse_moe.gate", + "nvidia/MiniMax-M3-NVFP4/block_sparse_moe.gate" + ] + }, + { + "type": "gemm", + "args": { + "m": 384, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "fp32" + }, + "k": 6144, + "n": 128, + "out": "fp32" + }, + "sources": [ + "MiniMaxAI/MiniMax-M3-MXFP8/block_sparse_moe.gate", + "amd/MiniMax-M3-MXFP4/block_sparse_moe.gate", + "nvidia/MiniMax-M3-NVFP4/block_sparse_moe.gate" + ] + }, + { + "type": "gemm", + "args": { + "m": 512, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "fp32" + }, + "k": 6144, + "n": 128, + "out": "fp32" + }, + "sources": [ + "MiniMaxAI/MiniMax-M3-MXFP8/block_sparse_moe.gate", + "amd/MiniMax-M3-MXFP4/block_sparse_moe.gate", + "nvidia/MiniMax-M3-NVFP4/block_sparse_moe.gate" + ] + }, + { + "type": "gemm", + "args": { + "m": 768, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "fp32" + }, + "k": 6144, + "n": 128, + "out": "fp32" + }, + "sources": [ + "MiniMaxAI/MiniMax-M3-MXFP8/block_sparse_moe.gate", + "amd/MiniMax-M3-MXFP4/block_sparse_moe.gate", + "nvidia/MiniMax-M3-NVFP4/block_sparse_moe.gate" + ] + }, + { + "type": "gemm", + "args": { + "m": 1024, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "fp32" + }, + "k": 6144, + "n": 128, + "out": "fp32" + }, + "sources": [ + "MiniMaxAI/MiniMax-M3-MXFP8/block_sparse_moe.gate", + "amd/MiniMax-M3-MXFP4/block_sparse_moe.gate", + "nvidia/MiniMax-M3-NVFP4/block_sparse_moe.gate" + ] + }, + { + "type": "gemm", + "args": { + "m": 1536, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "fp32" + }, + "k": 6144, + "n": 128, + "out": "fp32" + }, + "sources": [ + "MiniMaxAI/MiniMax-M3-MXFP8/block_sparse_moe.gate", + "amd/MiniMax-M3-MXFP4/block_sparse_moe.gate", + "nvidia/MiniMax-M3-NVFP4/block_sparse_moe.gate" + ] + }, + { + "type": "gemm", + "args": { + "m": 2048, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "fp32" + }, + "k": 6144, + "n": 128, + "out": "fp32" + }, + "sources": [ + "MiniMaxAI/MiniMax-M3-MXFP8/block_sparse_moe.gate", + "amd/MiniMax-M3-MXFP4/block_sparse_moe.gate", + "nvidia/MiniMax-M3-NVFP4/block_sparse_moe.gate" + ] + }, + { + "type": "gemm", + "args": { + "m": 3072, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "fp32" + }, + "k": 6144, + "n": 128, + "out": "fp32" + }, + "sources": [ + "MiniMaxAI/MiniMax-M3-MXFP8/block_sparse_moe.gate", + "amd/MiniMax-M3-MXFP4/block_sparse_moe.gate", + "nvidia/MiniMax-M3-NVFP4/block_sparse_moe.gate" + ] + }, + { + "type": "gemm", + "args": { + "m": 4096, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "fp32" + }, + "k": 6144, + "n": 128, + "out": "fp32" + }, + "sources": [ + "MiniMaxAI/MiniMax-M3-MXFP8/block_sparse_moe.gate", + "amd/MiniMax-M3-MXFP4/block_sparse_moe.gate", + "nvidia/MiniMax-M3-NVFP4/block_sparse_moe.gate" + ] + }, + { + "type": "gemm", + "args": { + "m": 6144, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "fp32" + }, + "k": 6144, + "n": 128, + "out": "fp32" + }, + "sources": [ + "MiniMaxAI/MiniMax-M3-MXFP8/block_sparse_moe.gate", + "amd/MiniMax-M3-MXFP4/block_sparse_moe.gate", + "nvidia/MiniMax-M3-NVFP4/block_sparse_moe.gate" + ] + }, + { + "type": "gemm", + "args": { + "m": 8192, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "fp32" + }, + "k": 6144, + "n": 128, + "out": "fp32" + }, + "sources": [ + "MiniMaxAI/MiniMax-M3-MXFP8/block_sparse_moe.gate", + "amd/MiniMax-M3-MXFP4/block_sparse_moe.gate", + "nvidia/MiniMax-M3-NVFP4/block_sparse_moe.gate" + ] + }, + { + "type": "gemm", + "args": { + "m": 12288, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "fp32" + }, + "k": 6144, + "n": 128, + "out": "fp32" + }, + "sources": [ + "MiniMaxAI/MiniMax-M3-MXFP8/block_sparse_moe.gate", + "amd/MiniMax-M3-MXFP4/block_sparse_moe.gate", + "nvidia/MiniMax-M3-NVFP4/block_sparse_moe.gate" + ] + }, + { + "type": "gemm", + "args": { + "m": 16384, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "fp32" + }, + "k": 6144, + "n": 128, + "out": "fp32" + }, + "sources": [ + "MiniMaxAI/MiniMax-M3-MXFP8/block_sparse_moe.gate", + "amd/MiniMax-M3-MXFP4/block_sparse_moe.gate", + "nvidia/MiniMax-M3-NVFP4/block_sparse_moe.gate" + ] + }, + { + "type": "gemm", + "args": { + "m": 24576, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "fp32" + }, + "k": 6144, + "n": 128, + "out": "fp32" + }, + "sources": [ + "MiniMaxAI/MiniMax-M3-MXFP8/block_sparse_moe.gate", + "amd/MiniMax-M3-MXFP4/block_sparse_moe.gate", + "nvidia/MiniMax-M3-NVFP4/block_sparse_moe.gate" + ] + }, + { + "type": "gemm", + "args": { + "m": 32768, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "fp32" + }, + "k": 6144, + "n": 128, + "out": "fp32" + }, + "sources": [ + "MiniMaxAI/MiniMax-M3-MXFP8/block_sparse_moe.gate", + "amd/MiniMax-M3-MXFP4/block_sparse_moe.gate", + "nvidia/MiniMax-M3-NVFP4/block_sparse_moe.gate" + ] + }, + { + "type": "gemm", + "args": { + "m": 49152, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "fp32" + }, + "k": 6144, + "n": 128, + "out": "fp32" + }, + "sources": [ + "MiniMaxAI/MiniMax-M3-MXFP8/block_sparse_moe.gate", + "amd/MiniMax-M3-MXFP4/block_sparse_moe.gate", + "nvidia/MiniMax-M3-NVFP4/block_sparse_moe.gate" + ] + }, + { + "type": "gemm", + "args": { + "m": 65536, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "fp32" + }, + "k": 6144, + "n": 128, + "out": "fp32" + }, + "sources": [ + "MiniMaxAI/MiniMax-M3-MXFP8/block_sparse_moe.gate", + "amd/MiniMax-M3-MXFP4/block_sparse_moe.gate", + "nvidia/MiniMax-M3-NVFP4/block_sparse_moe.gate" + ] + }, + { + "type": "gemm", + "args": { + "m": 1, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + }, + "k": 3072, + "n": 6144, + "out": "bf16" + }, + "sources": [ + "MiniMaxAI/MiniMax-M3-MXFP8/block_sparse_moe.shared_experts.down_proj", + "nvidia/MiniMax-M3-NVFP4/block_sparse_moe.shared_experts.down_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 2, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + }, + "k": 3072, + "n": 6144, + "out": "bf16" + }, + "sources": [ + "MiniMaxAI/MiniMax-M3-MXFP8/block_sparse_moe.shared_experts.down_proj", + "nvidia/MiniMax-M3-NVFP4/block_sparse_moe.shared_experts.down_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 3, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + }, + "k": 3072, + "n": 6144, + "out": "bf16" + }, + "sources": [ + "MiniMaxAI/MiniMax-M3-MXFP8/block_sparse_moe.shared_experts.down_proj", + "nvidia/MiniMax-M3-NVFP4/block_sparse_moe.shared_experts.down_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 4, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + }, + "k": 3072, + "n": 6144, + "out": "bf16" + }, + "sources": [ + "MiniMaxAI/MiniMax-M3-MXFP8/block_sparse_moe.shared_experts.down_proj", + "nvidia/MiniMax-M3-NVFP4/block_sparse_moe.shared_experts.down_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 6, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + }, + "k": 3072, + "n": 6144, + "out": "bf16" + }, + "sources": [ + "MiniMaxAI/MiniMax-M3-MXFP8/block_sparse_moe.shared_experts.down_proj", + "nvidia/MiniMax-M3-NVFP4/block_sparse_moe.shared_experts.down_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 8, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + }, + "k": 3072, + "n": 6144, + "out": "bf16" + }, + "sources": [ + "MiniMaxAI/MiniMax-M3-MXFP8/block_sparse_moe.shared_experts.down_proj", + "nvidia/MiniMax-M3-NVFP4/block_sparse_moe.shared_experts.down_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 12, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + }, + "k": 3072, + "n": 6144, + "out": "bf16" + }, + "sources": [ + "MiniMaxAI/MiniMax-M3-MXFP8/block_sparse_moe.shared_experts.down_proj", + "nvidia/MiniMax-M3-NVFP4/block_sparse_moe.shared_experts.down_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 16, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + }, + "k": 3072, + "n": 6144, + "out": "bf16" + }, + "sources": [ + "MiniMaxAI/MiniMax-M3-MXFP8/block_sparse_moe.shared_experts.down_proj", + "nvidia/MiniMax-M3-NVFP4/block_sparse_moe.shared_experts.down_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 24, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + }, + "k": 3072, + "n": 6144, + "out": "bf16" + }, + "sources": [ + "MiniMaxAI/MiniMax-M3-MXFP8/block_sparse_moe.shared_experts.down_proj", + "nvidia/MiniMax-M3-NVFP4/block_sparse_moe.shared_experts.down_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 32, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + }, + "k": 3072, + "n": 6144, + "out": "bf16" + }, + "sources": [ + "MiniMaxAI/MiniMax-M3-MXFP8/block_sparse_moe.shared_experts.down_proj", + "nvidia/MiniMax-M3-NVFP4/block_sparse_moe.shared_experts.down_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 48, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + }, + "k": 3072, + "n": 6144, + "out": "bf16" + }, + "sources": [ + "MiniMaxAI/MiniMax-M3-MXFP8/block_sparse_moe.shared_experts.down_proj", + "nvidia/MiniMax-M3-NVFP4/block_sparse_moe.shared_experts.down_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 64, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + }, + "k": 3072, + "n": 6144, + "out": "bf16" + }, + "sources": [ + "MiniMaxAI/MiniMax-M3-MXFP8/block_sparse_moe.shared_experts.down_proj", + "nvidia/MiniMax-M3-NVFP4/block_sparse_moe.shared_experts.down_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 96, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + }, + "k": 3072, + "n": 6144, + "out": "bf16" + }, + "sources": [ + "MiniMaxAI/MiniMax-M3-MXFP8/block_sparse_moe.shared_experts.down_proj", + "nvidia/MiniMax-M3-NVFP4/block_sparse_moe.shared_experts.down_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 128, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + }, + "k": 3072, + "n": 6144, + "out": "bf16" + }, + "sources": [ + "MiniMaxAI/MiniMax-M3-MXFP8/block_sparse_moe.shared_experts.down_proj", + "nvidia/MiniMax-M3-NVFP4/block_sparse_moe.shared_experts.down_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 192, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + }, + "k": 3072, + "n": 6144, + "out": "bf16" + }, + "sources": [ + "MiniMaxAI/MiniMax-M3-MXFP8/block_sparse_moe.shared_experts.down_proj", + "nvidia/MiniMax-M3-NVFP4/block_sparse_moe.shared_experts.down_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 256, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + }, + "k": 3072, + "n": 6144, + "out": "bf16" + }, + "sources": [ + "MiniMaxAI/MiniMax-M3-MXFP8/block_sparse_moe.shared_experts.down_proj", + "nvidia/MiniMax-M3-NVFP4/block_sparse_moe.shared_experts.down_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 384, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + }, + "k": 3072, + "n": 6144, + "out": "bf16" + }, + "sources": [ + "MiniMaxAI/MiniMax-M3-MXFP8/block_sparse_moe.shared_experts.down_proj", + "nvidia/MiniMax-M3-NVFP4/block_sparse_moe.shared_experts.down_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 512, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + }, + "k": 3072, + "n": 6144, + "out": "bf16" + }, + "sources": [ + "MiniMaxAI/MiniMax-M3-MXFP8/block_sparse_moe.shared_experts.down_proj", + "nvidia/MiniMax-M3-NVFP4/block_sparse_moe.shared_experts.down_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 768, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + }, + "k": 3072, + "n": 6144, + "out": "bf16" + }, + "sources": [ + "MiniMaxAI/MiniMax-M3-MXFP8/block_sparse_moe.shared_experts.down_proj", + "nvidia/MiniMax-M3-NVFP4/block_sparse_moe.shared_experts.down_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 1024, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + }, + "k": 3072, + "n": 6144, + "out": "bf16" + }, + "sources": [ + "MiniMaxAI/MiniMax-M3-MXFP8/block_sparse_moe.shared_experts.down_proj", + "nvidia/MiniMax-M3-NVFP4/block_sparse_moe.shared_experts.down_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + }, + "k": 3072, + "n": 6144, + "out": "bf16" + }, + "sources": [ + "MiniMaxAI/MiniMax-M3-MXFP8/block_sparse_moe.shared_experts.down_proj", + "nvidia/MiniMax-M3-NVFP4/block_sparse_moe.shared_experts.down_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 2048, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + }, + "k": 3072, + "n": 6144, + "out": "bf16" + }, + "sources": [ + "MiniMaxAI/MiniMax-M3-MXFP8/block_sparse_moe.shared_experts.down_proj", + "nvidia/MiniMax-M3-NVFP4/block_sparse_moe.shared_experts.down_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 3072, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + }, + "k": 3072, + "n": 6144, + "out": "bf16" + }, + "sources": [ + "MiniMaxAI/MiniMax-M3-MXFP8/block_sparse_moe.shared_experts.down_proj", + "nvidia/MiniMax-M3-NVFP4/block_sparse_moe.shared_experts.down_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 4096, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + }, + "k": 3072, + "n": 6144, + "out": "bf16" + }, + "sources": [ + "MiniMaxAI/MiniMax-M3-MXFP8/block_sparse_moe.shared_experts.down_proj", + "nvidia/MiniMax-M3-NVFP4/block_sparse_moe.shared_experts.down_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 6144, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + }, + "k": 3072, + "n": 6144, + "out": "bf16" + }, + "sources": [ + "MiniMaxAI/MiniMax-M3-MXFP8/block_sparse_moe.shared_experts.down_proj", + "nvidia/MiniMax-M3-NVFP4/block_sparse_moe.shared_experts.down_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 8192, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + }, + "k": 3072, + "n": 6144, + "out": "bf16" + }, + "sources": [ + "MiniMaxAI/MiniMax-M3-MXFP8/block_sparse_moe.shared_experts.down_proj", + "nvidia/MiniMax-M3-NVFP4/block_sparse_moe.shared_experts.down_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 12288, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + }, + "k": 3072, + "n": 6144, + "out": "bf16" + }, + "sources": [ + "MiniMaxAI/MiniMax-M3-MXFP8/block_sparse_moe.shared_experts.down_proj", + "nvidia/MiniMax-M3-NVFP4/block_sparse_moe.shared_experts.down_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 16384, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + }, + "k": 3072, + "n": 6144, + "out": "bf16" + }, + "sources": [ + "MiniMaxAI/MiniMax-M3-MXFP8/block_sparse_moe.shared_experts.down_proj", + "nvidia/MiniMax-M3-NVFP4/block_sparse_moe.shared_experts.down_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 24576, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + }, + "k": 3072, + "n": 6144, + "out": "bf16" + }, + "sources": [ + "MiniMaxAI/MiniMax-M3-MXFP8/block_sparse_moe.shared_experts.down_proj", + "nvidia/MiniMax-M3-NVFP4/block_sparse_moe.shared_experts.down_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 32768, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + }, + "k": 3072, + "n": 6144, + "out": "bf16" + }, + "sources": [ + "MiniMaxAI/MiniMax-M3-MXFP8/block_sparse_moe.shared_experts.down_proj", + "nvidia/MiniMax-M3-NVFP4/block_sparse_moe.shared_experts.down_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 49152, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + }, + "k": 3072, + "n": 6144, + "out": "bf16" + }, + "sources": [ + "MiniMaxAI/MiniMax-M3-MXFP8/block_sparse_moe.shared_experts.down_proj", + "nvidia/MiniMax-M3-NVFP4/block_sparse_moe.shared_experts.down_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 65536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + }, + "k": 3072, + "n": 6144, + "out": "bf16" + }, + "sources": [ + "MiniMaxAI/MiniMax-M3-MXFP8/block_sparse_moe.shared_experts.down_proj", + "nvidia/MiniMax-M3-NVFP4/block_sparse_moe.shared_experts.down_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 1, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + }, + "k": 6144, + "n": 6144, + "out": "bf16" + }, + "sources": [ + "MiniMaxAI/MiniMax-M3-MXFP8/block_sparse_moe.shared_experts.gate_proj+block_sparse_moe.shared_experts.up_proj", + "nvidia/MiniMax-M3-NVFP4/block_sparse_moe.shared_experts.gate_proj+block_sparse_moe.shared_experts.up_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 2, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + }, + "k": 6144, + "n": 6144, + "out": "bf16" + }, + "sources": [ + "MiniMaxAI/MiniMax-M3-MXFP8/block_sparse_moe.shared_experts.gate_proj+block_sparse_moe.shared_experts.up_proj", + "nvidia/MiniMax-M3-NVFP4/block_sparse_moe.shared_experts.gate_proj+block_sparse_moe.shared_experts.up_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 3, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + }, + "k": 6144, + "n": 6144, + "out": "bf16" + }, + "sources": [ + "MiniMaxAI/MiniMax-M3-MXFP8/block_sparse_moe.shared_experts.gate_proj+block_sparse_moe.shared_experts.up_proj", + "nvidia/MiniMax-M3-NVFP4/block_sparse_moe.shared_experts.gate_proj+block_sparse_moe.shared_experts.up_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 4, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + }, + "k": 6144, + "n": 6144, + "out": "bf16" + }, + "sources": [ + "MiniMaxAI/MiniMax-M3-MXFP8/block_sparse_moe.shared_experts.gate_proj+block_sparse_moe.shared_experts.up_proj", + "nvidia/MiniMax-M3-NVFP4/block_sparse_moe.shared_experts.gate_proj+block_sparse_moe.shared_experts.up_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 6, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + }, + "k": 6144, + "n": 6144, + "out": "bf16" + }, + "sources": [ + "MiniMaxAI/MiniMax-M3-MXFP8/block_sparse_moe.shared_experts.gate_proj+block_sparse_moe.shared_experts.up_proj", + "nvidia/MiniMax-M3-NVFP4/block_sparse_moe.shared_experts.gate_proj+block_sparse_moe.shared_experts.up_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 8, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + }, + "k": 6144, + "n": 6144, + "out": "bf16" + }, + "sources": [ + "MiniMaxAI/MiniMax-M3-MXFP8/block_sparse_moe.shared_experts.gate_proj+block_sparse_moe.shared_experts.up_proj", + "nvidia/MiniMax-M3-NVFP4/block_sparse_moe.shared_experts.gate_proj+block_sparse_moe.shared_experts.up_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 12, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + }, + "k": 6144, + "n": 6144, + "out": "bf16" + }, + "sources": [ + "MiniMaxAI/MiniMax-M3-MXFP8/block_sparse_moe.shared_experts.gate_proj+block_sparse_moe.shared_experts.up_proj", + "nvidia/MiniMax-M3-NVFP4/block_sparse_moe.shared_experts.gate_proj+block_sparse_moe.shared_experts.up_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 16, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + }, + "k": 6144, + "n": 6144, + "out": "bf16" + }, + "sources": [ + "MiniMaxAI/MiniMax-M3-MXFP8/block_sparse_moe.shared_experts.gate_proj+block_sparse_moe.shared_experts.up_proj", + "nvidia/MiniMax-M3-NVFP4/block_sparse_moe.shared_experts.gate_proj+block_sparse_moe.shared_experts.up_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 24, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + }, + "k": 6144, + "n": 6144, + "out": "bf16" + }, + "sources": [ + "MiniMaxAI/MiniMax-M3-MXFP8/block_sparse_moe.shared_experts.gate_proj+block_sparse_moe.shared_experts.up_proj", + "nvidia/MiniMax-M3-NVFP4/block_sparse_moe.shared_experts.gate_proj+block_sparse_moe.shared_experts.up_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 32, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + }, + "k": 6144, + "n": 6144, + "out": "bf16" + }, + "sources": [ + "MiniMaxAI/MiniMax-M3-MXFP8/block_sparse_moe.shared_experts.gate_proj+block_sparse_moe.shared_experts.up_proj", + "nvidia/MiniMax-M3-NVFP4/block_sparse_moe.shared_experts.gate_proj+block_sparse_moe.shared_experts.up_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 48, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + }, + "k": 6144, + "n": 6144, + "out": "bf16" + }, + "sources": [ + "MiniMaxAI/MiniMax-M3-MXFP8/block_sparse_moe.shared_experts.gate_proj+block_sparse_moe.shared_experts.up_proj", + "nvidia/MiniMax-M3-NVFP4/block_sparse_moe.shared_experts.gate_proj+block_sparse_moe.shared_experts.up_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 64, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + }, + "k": 6144, + "n": 6144, + "out": "bf16" + }, + "sources": [ + "MiniMaxAI/MiniMax-M3-MXFP8/block_sparse_moe.shared_experts.gate_proj+block_sparse_moe.shared_experts.up_proj", + "nvidia/MiniMax-M3-NVFP4/block_sparse_moe.shared_experts.gate_proj+block_sparse_moe.shared_experts.up_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 96, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + }, + "k": 6144, + "n": 6144, + "out": "bf16" + }, + "sources": [ + "MiniMaxAI/MiniMax-M3-MXFP8/block_sparse_moe.shared_experts.gate_proj+block_sparse_moe.shared_experts.up_proj", + "nvidia/MiniMax-M3-NVFP4/block_sparse_moe.shared_experts.gate_proj+block_sparse_moe.shared_experts.up_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 128, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + }, + "k": 6144, + "n": 6144, + "out": "bf16" + }, + "sources": [ + "MiniMaxAI/MiniMax-M3-MXFP8/block_sparse_moe.shared_experts.gate_proj+block_sparse_moe.shared_experts.up_proj", + "nvidia/MiniMax-M3-NVFP4/block_sparse_moe.shared_experts.gate_proj+block_sparse_moe.shared_experts.up_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 192, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + }, + "k": 6144, + "n": 6144, + "out": "bf16" + }, + "sources": [ + "MiniMaxAI/MiniMax-M3-MXFP8/block_sparse_moe.shared_experts.gate_proj+block_sparse_moe.shared_experts.up_proj", + "nvidia/MiniMax-M3-NVFP4/block_sparse_moe.shared_experts.gate_proj+block_sparse_moe.shared_experts.up_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 256, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + }, + "k": 6144, + "n": 6144, + "out": "bf16" + }, + "sources": [ + "MiniMaxAI/MiniMax-M3-MXFP8/block_sparse_moe.shared_experts.gate_proj+block_sparse_moe.shared_experts.up_proj", + "nvidia/MiniMax-M3-NVFP4/block_sparse_moe.shared_experts.gate_proj+block_sparse_moe.shared_experts.up_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 384, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + }, + "k": 6144, + "n": 6144, + "out": "bf16" + }, + "sources": [ + "MiniMaxAI/MiniMax-M3-MXFP8/block_sparse_moe.shared_experts.gate_proj+block_sparse_moe.shared_experts.up_proj", + "nvidia/MiniMax-M3-NVFP4/block_sparse_moe.shared_experts.gate_proj+block_sparse_moe.shared_experts.up_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 512, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + }, + "k": 6144, + "n": 6144, + "out": "bf16" + }, + "sources": [ + "MiniMaxAI/MiniMax-M3-MXFP8/block_sparse_moe.shared_experts.gate_proj+block_sparse_moe.shared_experts.up_proj", + "nvidia/MiniMax-M3-NVFP4/block_sparse_moe.shared_experts.gate_proj+block_sparse_moe.shared_experts.up_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 768, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + }, + "k": 6144, + "n": 6144, + "out": "bf16" + }, + "sources": [ + "MiniMaxAI/MiniMax-M3-MXFP8/block_sparse_moe.shared_experts.gate_proj+block_sparse_moe.shared_experts.up_proj", + "nvidia/MiniMax-M3-NVFP4/block_sparse_moe.shared_experts.gate_proj+block_sparse_moe.shared_experts.up_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 1024, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + }, + "k": 6144, + "n": 6144, + "out": "bf16" + }, + "sources": [ + "MiniMaxAI/MiniMax-M3-MXFP8/block_sparse_moe.shared_experts.gate_proj+block_sparse_moe.shared_experts.up_proj", + "nvidia/MiniMax-M3-NVFP4/block_sparse_moe.shared_experts.gate_proj+block_sparse_moe.shared_experts.up_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + }, + "k": 6144, + "n": 6144, + "out": "bf16" + }, + "sources": [ + "MiniMaxAI/MiniMax-M3-MXFP8/block_sparse_moe.shared_experts.gate_proj+block_sparse_moe.shared_experts.up_proj", + "nvidia/MiniMax-M3-NVFP4/block_sparse_moe.shared_experts.gate_proj+block_sparse_moe.shared_experts.up_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 2048, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + }, + "k": 6144, + "n": 6144, + "out": "bf16" + }, + "sources": [ + "MiniMaxAI/MiniMax-M3-MXFP8/block_sparse_moe.shared_experts.gate_proj+block_sparse_moe.shared_experts.up_proj", + "nvidia/MiniMax-M3-NVFP4/block_sparse_moe.shared_experts.gate_proj+block_sparse_moe.shared_experts.up_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 3072, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + }, + "k": 6144, + "n": 6144, + "out": "bf16" + }, + "sources": [ + "MiniMaxAI/MiniMax-M3-MXFP8/block_sparse_moe.shared_experts.gate_proj+block_sparse_moe.shared_experts.up_proj", + "nvidia/MiniMax-M3-NVFP4/block_sparse_moe.shared_experts.gate_proj+block_sparse_moe.shared_experts.up_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 4096, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + }, + "k": 6144, + "n": 6144, + "out": "bf16" + }, + "sources": [ + "MiniMaxAI/MiniMax-M3-MXFP8/block_sparse_moe.shared_experts.gate_proj+block_sparse_moe.shared_experts.up_proj", + "nvidia/MiniMax-M3-NVFP4/block_sparse_moe.shared_experts.gate_proj+block_sparse_moe.shared_experts.up_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 6144, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + }, + "k": 6144, + "n": 6144, + "out": "bf16" + }, + "sources": [ + "MiniMaxAI/MiniMax-M3-MXFP8/block_sparse_moe.shared_experts.gate_proj+block_sparse_moe.shared_experts.up_proj", + "nvidia/MiniMax-M3-NVFP4/block_sparse_moe.shared_experts.gate_proj+block_sparse_moe.shared_experts.up_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 8192, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + }, + "k": 6144, + "n": 6144, + "out": "bf16" + }, + "sources": [ + "MiniMaxAI/MiniMax-M3-MXFP8/block_sparse_moe.shared_experts.gate_proj+block_sparse_moe.shared_experts.up_proj", + "nvidia/MiniMax-M3-NVFP4/block_sparse_moe.shared_experts.gate_proj+block_sparse_moe.shared_experts.up_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 12288, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + }, + "k": 6144, + "n": 6144, + "out": "bf16" + }, + "sources": [ + "MiniMaxAI/MiniMax-M3-MXFP8/block_sparse_moe.shared_experts.gate_proj+block_sparse_moe.shared_experts.up_proj", + "nvidia/MiniMax-M3-NVFP4/block_sparse_moe.shared_experts.gate_proj+block_sparse_moe.shared_experts.up_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 16384, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + }, + "k": 6144, + "n": 6144, + "out": "bf16" + }, + "sources": [ + "MiniMaxAI/MiniMax-M3-MXFP8/block_sparse_moe.shared_experts.gate_proj+block_sparse_moe.shared_experts.up_proj", + "nvidia/MiniMax-M3-NVFP4/block_sparse_moe.shared_experts.gate_proj+block_sparse_moe.shared_experts.up_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 24576, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + }, + "k": 6144, + "n": 6144, + "out": "bf16" + }, + "sources": [ + "MiniMaxAI/MiniMax-M3-MXFP8/block_sparse_moe.shared_experts.gate_proj+block_sparse_moe.shared_experts.up_proj", + "nvidia/MiniMax-M3-NVFP4/block_sparse_moe.shared_experts.gate_proj+block_sparse_moe.shared_experts.up_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 32768, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + }, + "k": 6144, + "n": 6144, + "out": "bf16" + }, + "sources": [ + "MiniMaxAI/MiniMax-M3-MXFP8/block_sparse_moe.shared_experts.gate_proj+block_sparse_moe.shared_experts.up_proj", + "nvidia/MiniMax-M3-NVFP4/block_sparse_moe.shared_experts.gate_proj+block_sparse_moe.shared_experts.up_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 49152, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + }, + "k": 6144, + "n": 6144, + "out": "bf16" + }, + "sources": [ + "MiniMaxAI/MiniMax-M3-MXFP8/block_sparse_moe.shared_experts.gate_proj+block_sparse_moe.shared_experts.up_proj", + "nvidia/MiniMax-M3-NVFP4/block_sparse_moe.shared_experts.gate_proj+block_sparse_moe.shared_experts.up_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 65536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + }, + "k": 6144, + "n": 6144, + "out": "bf16" + }, + "sources": [ + "MiniMaxAI/MiniMax-M3-MXFP8/block_sparse_moe.shared_experts.gate_proj+block_sparse_moe.shared_experts.up_proj", + "nvidia/MiniMax-M3-NVFP4/block_sparse_moe.shared_experts.gate_proj+block_sparse_moe.shared_experts.up_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 1, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 6144, + "n": 200064, + "out": "bf16" + }, + "sources": [ + "MiniMaxAI/MiniMax-M3-MXFP8/lm_head", + "amd/MiniMax-M3-MXFP4/lm_head", + "nvidia/MiniMax-M3-NVFP4/lm_head" + ] + }, + { + "type": "gemm", + "args": { + "m": 2, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 6144, + "n": 200064, + "out": "bf16" + }, + "sources": [ + "MiniMaxAI/MiniMax-M3-MXFP8/lm_head", + "amd/MiniMax-M3-MXFP4/lm_head", + "nvidia/MiniMax-M3-NVFP4/lm_head" + ] + }, + { + "type": "gemm", + "args": { + "m": 3, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 6144, + "n": 200064, + "out": "bf16" + }, + "sources": [ + "MiniMaxAI/MiniMax-M3-MXFP8/lm_head", + "amd/MiniMax-M3-MXFP4/lm_head", + "nvidia/MiniMax-M3-NVFP4/lm_head" + ] + }, + { + "type": "gemm", + "args": { + "m": 4, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 6144, + "n": 200064, + "out": "bf16" + }, + "sources": [ + "MiniMaxAI/MiniMax-M3-MXFP8/lm_head", + "amd/MiniMax-M3-MXFP4/lm_head", + "nvidia/MiniMax-M3-NVFP4/lm_head" + ] + }, + { + "type": "gemm", + "args": { + "m": 6, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 6144, + "n": 200064, + "out": "bf16" + }, + "sources": [ + "MiniMaxAI/MiniMax-M3-MXFP8/lm_head", + "amd/MiniMax-M3-MXFP4/lm_head", + "nvidia/MiniMax-M3-NVFP4/lm_head" + ] + }, + { + "type": "gemm", + "args": { + "m": 8, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 6144, + "n": 200064, + "out": "bf16" + }, + "sources": [ + "MiniMaxAI/MiniMax-M3-MXFP8/lm_head", + "amd/MiniMax-M3-MXFP4/lm_head", + "nvidia/MiniMax-M3-NVFP4/lm_head" + ] + }, + { + "type": "gemm", + "args": { + "m": 12, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 6144, + "n": 200064, + "out": "bf16" + }, + "sources": [ + "MiniMaxAI/MiniMax-M3-MXFP8/lm_head", + "amd/MiniMax-M3-MXFP4/lm_head", + "nvidia/MiniMax-M3-NVFP4/lm_head" + ] + }, + { + "type": "gemm", + "args": { + "m": 16, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 6144, + "n": 200064, + "out": "bf16" + }, + "sources": [ + "MiniMaxAI/MiniMax-M3-MXFP8/lm_head", + "amd/MiniMax-M3-MXFP4/lm_head", + "nvidia/MiniMax-M3-NVFP4/lm_head" + ] + }, + { + "type": "gemm", + "args": { + "m": 24, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 6144, + "n": 200064, + "out": "bf16" + }, + "sources": [ + "MiniMaxAI/MiniMax-M3-MXFP8/lm_head", + "amd/MiniMax-M3-MXFP4/lm_head", + "nvidia/MiniMax-M3-NVFP4/lm_head" + ] + }, + { + "type": "gemm", + "args": { + "m": 32, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 6144, + "n": 200064, + "out": "bf16" + }, + "sources": [ + "MiniMaxAI/MiniMax-M3-MXFP8/lm_head", + "amd/MiniMax-M3-MXFP4/lm_head", + "nvidia/MiniMax-M3-NVFP4/lm_head" + ] + }, + { + "type": "gemm", + "args": { + "m": 48, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 6144, + "n": 200064, + "out": "bf16" + }, + "sources": [ + "MiniMaxAI/MiniMax-M3-MXFP8/lm_head", + "amd/MiniMax-M3-MXFP4/lm_head", + "nvidia/MiniMax-M3-NVFP4/lm_head" + ] + }, + { + "type": "gemm", + "args": { + "m": 64, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 6144, + "n": 200064, + "out": "bf16" + }, + "sources": [ + "MiniMaxAI/MiniMax-M3-MXFP8/lm_head", + "amd/MiniMax-M3-MXFP4/lm_head", + "nvidia/MiniMax-M3-NVFP4/lm_head" + ] + }, + { + "type": "gemm", + "args": { + "m": 96, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 6144, + "n": 200064, + "out": "bf16" + }, + "sources": [ + "MiniMaxAI/MiniMax-M3-MXFP8/lm_head", + "amd/MiniMax-M3-MXFP4/lm_head", + "nvidia/MiniMax-M3-NVFP4/lm_head" + ] + }, + { + "type": "gemm", + "args": { + "m": 128, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 6144, + "n": 200064, + "out": "bf16" + }, + "sources": [ + "MiniMaxAI/MiniMax-M3-MXFP8/lm_head", + "amd/MiniMax-M3-MXFP4/lm_head", + "nvidia/MiniMax-M3-NVFP4/lm_head" + ] + }, + { + "type": "gemm", + "args": { + "m": 192, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 6144, + "n": 200064, + "out": "bf16" + }, + "sources": [ + "MiniMaxAI/MiniMax-M3-MXFP8/lm_head", + "amd/MiniMax-M3-MXFP4/lm_head", + "nvidia/MiniMax-M3-NVFP4/lm_head" + ] + }, + { + "type": "gemm", + "args": { + "m": 256, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 6144, + "n": 200064, + "out": "bf16" + }, + "sources": [ + "MiniMaxAI/MiniMax-M3-MXFP8/lm_head", + "amd/MiniMax-M3-MXFP4/lm_head", + "nvidia/MiniMax-M3-NVFP4/lm_head" + ] + }, + { + "type": "gemm", + "args": { + "m": 384, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 6144, + "n": 200064, + "out": "bf16" + }, + "sources": [ + "MiniMaxAI/MiniMax-M3-MXFP8/lm_head", + "amd/MiniMax-M3-MXFP4/lm_head", + "nvidia/MiniMax-M3-NVFP4/lm_head" + ] + }, + { + "type": "gemm", + "args": { + "m": 512, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 6144, + "n": 200064, + "out": "bf16" + }, + "sources": [ + "MiniMaxAI/MiniMax-M3-MXFP8/lm_head", + "amd/MiniMax-M3-MXFP4/lm_head", + "nvidia/MiniMax-M3-NVFP4/lm_head" + ] + }, + { + "type": "gemm", + "args": { + "m": 768, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 6144, + "n": 200064, + "out": "bf16" + }, + "sources": [ + "MiniMaxAI/MiniMax-M3-MXFP8/lm_head", + "amd/MiniMax-M3-MXFP4/lm_head", + "nvidia/MiniMax-M3-NVFP4/lm_head" + ] + }, + { + "type": "gemm", + "args": { + "m": 1024, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 6144, + "n": 200064, + "out": "bf16" + }, + "sources": [ + "MiniMaxAI/MiniMax-M3-MXFP8/lm_head", + "amd/MiniMax-M3-MXFP4/lm_head", + "nvidia/MiniMax-M3-NVFP4/lm_head" + ] + }, + { + "type": "gemm", + "args": { + "m": 1536, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 6144, + "n": 200064, + "out": "bf16" + }, + "sources": [ + "MiniMaxAI/MiniMax-M3-MXFP8/lm_head", + "amd/MiniMax-M3-MXFP4/lm_head", + "nvidia/MiniMax-M3-NVFP4/lm_head" + ] + }, + { + "type": "gemm", + "args": { + "m": 2048, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 6144, + "n": 200064, + "out": "bf16" + }, + "sources": [ + "MiniMaxAI/MiniMax-M3-MXFP8/lm_head", + "amd/MiniMax-M3-MXFP4/lm_head", + "nvidia/MiniMax-M3-NVFP4/lm_head" + ] + }, + { + "type": "gemm", + "args": { + "m": 3072, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 6144, + "n": 200064, + "out": "bf16" + }, + "sources": [ + "MiniMaxAI/MiniMax-M3-MXFP8/lm_head", + "amd/MiniMax-M3-MXFP4/lm_head", + "nvidia/MiniMax-M3-NVFP4/lm_head" + ] + }, + { + "type": "gemm", + "args": { + "m": 4096, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 6144, + "n": 200064, + "out": "bf16" + }, + "sources": [ + "MiniMaxAI/MiniMax-M3-MXFP8/lm_head", + "amd/MiniMax-M3-MXFP4/lm_head", + "nvidia/MiniMax-M3-NVFP4/lm_head" + ] + }, + { + "type": "gemm", + "args": { + "m": 6144, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 6144, + "n": 200064, + "out": "bf16" + }, + "sources": [ + "MiniMaxAI/MiniMax-M3-MXFP8/lm_head", + "amd/MiniMax-M3-MXFP4/lm_head", + "nvidia/MiniMax-M3-NVFP4/lm_head" + ] + }, + { + "type": "gemm", + "args": { + "m": 8192, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 6144, + "n": 200064, + "out": "bf16" + }, + "sources": [ + "MiniMaxAI/MiniMax-M3-MXFP8/lm_head", + "amd/MiniMax-M3-MXFP4/lm_head", + "nvidia/MiniMax-M3-NVFP4/lm_head" + ] + }, + { + "type": "gemm", + "args": { + "m": 12288, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 6144, + "n": 200064, + "out": "bf16" + }, + "sources": [ + "MiniMaxAI/MiniMax-M3-MXFP8/lm_head", + "amd/MiniMax-M3-MXFP4/lm_head", + "nvidia/MiniMax-M3-NVFP4/lm_head" + ] + }, + { + "type": "gemm", + "args": { + "m": 16384, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 6144, + "n": 200064, + "out": "bf16" + }, + "sources": [ + "MiniMaxAI/MiniMax-M3-MXFP8/lm_head", + "amd/MiniMax-M3-MXFP4/lm_head", + "nvidia/MiniMax-M3-NVFP4/lm_head" + ] + }, + { + "type": "gemm", + "args": { + "m": 24576, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 6144, + "n": 200064, + "out": "bf16" + }, + "sources": [ + "MiniMaxAI/MiniMax-M3-MXFP8/lm_head", + "amd/MiniMax-M3-MXFP4/lm_head", + "nvidia/MiniMax-M3-NVFP4/lm_head" + ] + }, + { + "type": "gemm", + "args": { + "m": 32768, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 6144, + "n": 200064, + "out": "bf16" + }, + "sources": [ + "MiniMaxAI/MiniMax-M3-MXFP8/lm_head", + "amd/MiniMax-M3-MXFP4/lm_head", + "nvidia/MiniMax-M3-NVFP4/lm_head" + ] + }, + { + "type": "gemm", + "args": { + "m": 49152, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 6144, + "n": 200064, + "out": "bf16" + }, + "sources": [ + "MiniMaxAI/MiniMax-M3-MXFP8/lm_head", + "amd/MiniMax-M3-MXFP4/lm_head", + "nvidia/MiniMax-M3-NVFP4/lm_head" + ] + }, + { + "type": "gemm", + "args": { + "m": 65536, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 6144, + "n": 200064, + "out": "bf16" + }, + "sources": [ + "MiniMaxAI/MiniMax-M3-MXFP8/lm_head", + "amd/MiniMax-M3-MXFP4/lm_head", + "nvidia/MiniMax-M3-NVFP4/lm_head" + ] + }, + { + "type": "gemm", + "args": { + "m": 1, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + }, + "k": 12288, + "n": 6144, + "out": "bf16" + }, + "sources": [ + "MiniMaxAI/MiniMax-M3-MXFP8/mlp.down_proj", + "nvidia/MiniMax-M3-NVFP4/mlp.down_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 2, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + }, + "k": 12288, + "n": 6144, + "out": "bf16" + }, + "sources": [ + "MiniMaxAI/MiniMax-M3-MXFP8/mlp.down_proj", + "nvidia/MiniMax-M3-NVFP4/mlp.down_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 3, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + }, + "k": 12288, + "n": 6144, + "out": "bf16" + }, + "sources": [ + "MiniMaxAI/MiniMax-M3-MXFP8/mlp.down_proj", + "nvidia/MiniMax-M3-NVFP4/mlp.down_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 4, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + }, + "k": 12288, + "n": 6144, + "out": "bf16" + }, + "sources": [ + "MiniMaxAI/MiniMax-M3-MXFP8/mlp.down_proj", + "nvidia/MiniMax-M3-NVFP4/mlp.down_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 6, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + }, + "k": 12288, + "n": 6144, + "out": "bf16" + }, + "sources": [ + "MiniMaxAI/MiniMax-M3-MXFP8/mlp.down_proj", + "nvidia/MiniMax-M3-NVFP4/mlp.down_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 8, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + }, + "k": 12288, + "n": 6144, + "out": "bf16" + }, + "sources": [ + "MiniMaxAI/MiniMax-M3-MXFP8/mlp.down_proj", + "nvidia/MiniMax-M3-NVFP4/mlp.down_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 12, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + }, + "k": 12288, + "n": 6144, + "out": "bf16" + }, + "sources": [ + "MiniMaxAI/MiniMax-M3-MXFP8/mlp.down_proj", + "nvidia/MiniMax-M3-NVFP4/mlp.down_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 16, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + }, + "k": 12288, + "n": 6144, + "out": "bf16" + }, + "sources": [ + "MiniMaxAI/MiniMax-M3-MXFP8/mlp.down_proj", + "nvidia/MiniMax-M3-NVFP4/mlp.down_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 24, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + }, + "k": 12288, + "n": 6144, + "out": "bf16" + }, + "sources": [ + "MiniMaxAI/MiniMax-M3-MXFP8/mlp.down_proj", + "nvidia/MiniMax-M3-NVFP4/mlp.down_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 32, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + }, + "k": 12288, + "n": 6144, + "out": "bf16" + }, + "sources": [ + "MiniMaxAI/MiniMax-M3-MXFP8/mlp.down_proj", + "nvidia/MiniMax-M3-NVFP4/mlp.down_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 48, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + }, + "k": 12288, + "n": 6144, + "out": "bf16" + }, + "sources": [ + "MiniMaxAI/MiniMax-M3-MXFP8/mlp.down_proj", + "nvidia/MiniMax-M3-NVFP4/mlp.down_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 64, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + }, + "k": 12288, + "n": 6144, + "out": "bf16" + }, + "sources": [ + "MiniMaxAI/MiniMax-M3-MXFP8/mlp.down_proj", + "nvidia/MiniMax-M3-NVFP4/mlp.down_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 96, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + }, + "k": 12288, + "n": 6144, + "out": "bf16" + }, + "sources": [ + "MiniMaxAI/MiniMax-M3-MXFP8/mlp.down_proj", + "nvidia/MiniMax-M3-NVFP4/mlp.down_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 128, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + }, + "k": 12288, + "n": 6144, + "out": "bf16" + }, + "sources": [ + "MiniMaxAI/MiniMax-M3-MXFP8/mlp.down_proj", + "nvidia/MiniMax-M3-NVFP4/mlp.down_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 192, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + }, + "k": 12288, + "n": 6144, + "out": "bf16" + }, + "sources": [ + "MiniMaxAI/MiniMax-M3-MXFP8/mlp.down_proj", + "nvidia/MiniMax-M3-NVFP4/mlp.down_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 256, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + }, + "k": 12288, + "n": 6144, + "out": "bf16" + }, + "sources": [ + "MiniMaxAI/MiniMax-M3-MXFP8/mlp.down_proj", + "nvidia/MiniMax-M3-NVFP4/mlp.down_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 384, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + }, + "k": 12288, + "n": 6144, + "out": "bf16" + }, + "sources": [ + "MiniMaxAI/MiniMax-M3-MXFP8/mlp.down_proj", + "nvidia/MiniMax-M3-NVFP4/mlp.down_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 512, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + }, + "k": 12288, + "n": 6144, + "out": "bf16" + }, + "sources": [ + "MiniMaxAI/MiniMax-M3-MXFP8/mlp.down_proj", + "nvidia/MiniMax-M3-NVFP4/mlp.down_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 768, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + }, + "k": 12288, + "n": 6144, + "out": "bf16" + }, + "sources": [ + "MiniMaxAI/MiniMax-M3-MXFP8/mlp.down_proj", + "nvidia/MiniMax-M3-NVFP4/mlp.down_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 1024, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + }, + "k": 12288, + "n": 6144, + "out": "bf16" + }, + "sources": [ + "MiniMaxAI/MiniMax-M3-MXFP8/mlp.down_proj", + "nvidia/MiniMax-M3-NVFP4/mlp.down_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + }, + "k": 12288, + "n": 6144, + "out": "bf16" + }, + "sources": [ + "MiniMaxAI/MiniMax-M3-MXFP8/mlp.down_proj", + "nvidia/MiniMax-M3-NVFP4/mlp.down_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 2048, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + }, + "k": 12288, + "n": 6144, + "out": "bf16" + }, + "sources": [ + "MiniMaxAI/MiniMax-M3-MXFP8/mlp.down_proj", + "nvidia/MiniMax-M3-NVFP4/mlp.down_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 3072, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + }, + "k": 12288, + "n": 6144, + "out": "bf16" + }, + "sources": [ + "MiniMaxAI/MiniMax-M3-MXFP8/mlp.down_proj", + "nvidia/MiniMax-M3-NVFP4/mlp.down_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 4096, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + }, + "k": 12288, + "n": 6144, + "out": "bf16" + }, + "sources": [ + "MiniMaxAI/MiniMax-M3-MXFP8/mlp.down_proj", + "nvidia/MiniMax-M3-NVFP4/mlp.down_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 6144, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + }, + "k": 12288, + "n": 6144, + "out": "bf16" + }, + "sources": [ + "MiniMaxAI/MiniMax-M3-MXFP8/mlp.down_proj", + "nvidia/MiniMax-M3-NVFP4/mlp.down_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 8192, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + }, + "k": 12288, + "n": 6144, + "out": "bf16" + }, + "sources": [ + "MiniMaxAI/MiniMax-M3-MXFP8/mlp.down_proj", + "nvidia/MiniMax-M3-NVFP4/mlp.down_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 12288, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + }, + "k": 12288, + "n": 6144, + "out": "bf16" + }, + "sources": [ + "MiniMaxAI/MiniMax-M3-MXFP8/mlp.down_proj", + "nvidia/MiniMax-M3-NVFP4/mlp.down_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 16384, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + }, + "k": 12288, + "n": 6144, + "out": "bf16" + }, + "sources": [ + "MiniMaxAI/MiniMax-M3-MXFP8/mlp.down_proj", + "nvidia/MiniMax-M3-NVFP4/mlp.down_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 24576, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + }, + "k": 12288, + "n": 6144, + "out": "bf16" + }, + "sources": [ + "MiniMaxAI/MiniMax-M3-MXFP8/mlp.down_proj", + "nvidia/MiniMax-M3-NVFP4/mlp.down_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 32768, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + }, + "k": 12288, + "n": 6144, + "out": "bf16" + }, + "sources": [ + "MiniMaxAI/MiniMax-M3-MXFP8/mlp.down_proj", + "nvidia/MiniMax-M3-NVFP4/mlp.down_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 49152, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + }, + "k": 12288, + "n": 6144, + "out": "bf16" + }, + "sources": [ + "MiniMaxAI/MiniMax-M3-MXFP8/mlp.down_proj", + "nvidia/MiniMax-M3-NVFP4/mlp.down_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 65536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + }, + "k": 12288, + "n": 6144, + "out": "bf16" + }, + "sources": [ + "MiniMaxAI/MiniMax-M3-MXFP8/mlp.down_proj", + "nvidia/MiniMax-M3-NVFP4/mlp.down_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 1, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + }, + "k": 6144, + "n": 24576, + "out": "bf16" + }, + "sources": [ + "MiniMaxAI/MiniMax-M3-MXFP8/mlp.gate_proj+mlp.up_proj", + "nvidia/MiniMax-M3-NVFP4/mlp.gate_proj+mlp.up_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 2, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + }, + "k": 6144, + "n": 24576, + "out": "bf16" + }, + "sources": [ + "MiniMaxAI/MiniMax-M3-MXFP8/mlp.gate_proj+mlp.up_proj", + "nvidia/MiniMax-M3-NVFP4/mlp.gate_proj+mlp.up_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 3, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + }, + "k": 6144, + "n": 24576, + "out": "bf16" + }, + "sources": [ + "MiniMaxAI/MiniMax-M3-MXFP8/mlp.gate_proj+mlp.up_proj", + "nvidia/MiniMax-M3-NVFP4/mlp.gate_proj+mlp.up_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 4, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + }, + "k": 6144, + "n": 24576, + "out": "bf16" + }, + "sources": [ + "MiniMaxAI/MiniMax-M3-MXFP8/mlp.gate_proj+mlp.up_proj", + "nvidia/MiniMax-M3-NVFP4/mlp.gate_proj+mlp.up_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 6, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + }, + "k": 6144, + "n": 24576, + "out": "bf16" + }, + "sources": [ + "MiniMaxAI/MiniMax-M3-MXFP8/mlp.gate_proj+mlp.up_proj", + "nvidia/MiniMax-M3-NVFP4/mlp.gate_proj+mlp.up_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 8, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + }, + "k": 6144, + "n": 24576, + "out": "bf16" + }, + "sources": [ + "MiniMaxAI/MiniMax-M3-MXFP8/mlp.gate_proj+mlp.up_proj", + "nvidia/MiniMax-M3-NVFP4/mlp.gate_proj+mlp.up_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 12, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + }, + "k": 6144, + "n": 24576, + "out": "bf16" + }, + "sources": [ + "MiniMaxAI/MiniMax-M3-MXFP8/mlp.gate_proj+mlp.up_proj", + "nvidia/MiniMax-M3-NVFP4/mlp.gate_proj+mlp.up_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 16, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + }, + "k": 6144, + "n": 24576, + "out": "bf16" + }, + "sources": [ + "MiniMaxAI/MiniMax-M3-MXFP8/mlp.gate_proj+mlp.up_proj", + "nvidia/MiniMax-M3-NVFP4/mlp.gate_proj+mlp.up_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 24, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + }, + "k": 6144, + "n": 24576, + "out": "bf16" + }, + "sources": [ + "MiniMaxAI/MiniMax-M3-MXFP8/mlp.gate_proj+mlp.up_proj", + "nvidia/MiniMax-M3-NVFP4/mlp.gate_proj+mlp.up_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 32, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + }, + "k": 6144, + "n": 24576, + "out": "bf16" + }, + "sources": [ + "MiniMaxAI/MiniMax-M3-MXFP8/mlp.gate_proj+mlp.up_proj", + "nvidia/MiniMax-M3-NVFP4/mlp.gate_proj+mlp.up_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 48, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + }, + "k": 6144, + "n": 24576, + "out": "bf16" + }, + "sources": [ + "MiniMaxAI/MiniMax-M3-MXFP8/mlp.gate_proj+mlp.up_proj", + "nvidia/MiniMax-M3-NVFP4/mlp.gate_proj+mlp.up_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 64, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + }, + "k": 6144, + "n": 24576, + "out": "bf16" + }, + "sources": [ + "MiniMaxAI/MiniMax-M3-MXFP8/mlp.gate_proj+mlp.up_proj", + "nvidia/MiniMax-M3-NVFP4/mlp.gate_proj+mlp.up_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 96, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + }, + "k": 6144, + "n": 24576, + "out": "bf16" + }, + "sources": [ + "MiniMaxAI/MiniMax-M3-MXFP8/mlp.gate_proj+mlp.up_proj", + "nvidia/MiniMax-M3-NVFP4/mlp.gate_proj+mlp.up_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 128, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + }, + "k": 6144, + "n": 24576, + "out": "bf16" + }, + "sources": [ + "MiniMaxAI/MiniMax-M3-MXFP8/mlp.gate_proj+mlp.up_proj", + "nvidia/MiniMax-M3-NVFP4/mlp.gate_proj+mlp.up_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 192, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + }, + "k": 6144, + "n": 24576, + "out": "bf16" + }, + "sources": [ + "MiniMaxAI/MiniMax-M3-MXFP8/mlp.gate_proj+mlp.up_proj", + "nvidia/MiniMax-M3-NVFP4/mlp.gate_proj+mlp.up_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 256, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + }, + "k": 6144, + "n": 24576, + "out": "bf16" + }, + "sources": [ + "MiniMaxAI/MiniMax-M3-MXFP8/mlp.gate_proj+mlp.up_proj", + "nvidia/MiniMax-M3-NVFP4/mlp.gate_proj+mlp.up_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 384, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + }, + "k": 6144, + "n": 24576, + "out": "bf16" + }, + "sources": [ + "MiniMaxAI/MiniMax-M3-MXFP8/mlp.gate_proj+mlp.up_proj", + "nvidia/MiniMax-M3-NVFP4/mlp.gate_proj+mlp.up_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 512, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + }, + "k": 6144, + "n": 24576, + "out": "bf16" + }, + "sources": [ + "MiniMaxAI/MiniMax-M3-MXFP8/mlp.gate_proj+mlp.up_proj", + "nvidia/MiniMax-M3-NVFP4/mlp.gate_proj+mlp.up_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 768, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + }, + "k": 6144, + "n": 24576, + "out": "bf16" + }, + "sources": [ + "MiniMaxAI/MiniMax-M3-MXFP8/mlp.gate_proj+mlp.up_proj", + "nvidia/MiniMax-M3-NVFP4/mlp.gate_proj+mlp.up_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 1024, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + }, + "k": 6144, + "n": 24576, + "out": "bf16" + }, + "sources": [ + "MiniMaxAI/MiniMax-M3-MXFP8/mlp.gate_proj+mlp.up_proj", + "nvidia/MiniMax-M3-NVFP4/mlp.gate_proj+mlp.up_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + }, + "k": 6144, + "n": 24576, + "out": "bf16" + }, + "sources": [ + "MiniMaxAI/MiniMax-M3-MXFP8/mlp.gate_proj+mlp.up_proj", + "nvidia/MiniMax-M3-NVFP4/mlp.gate_proj+mlp.up_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 2048, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + }, + "k": 6144, + "n": 24576, + "out": "bf16" + }, + "sources": [ + "MiniMaxAI/MiniMax-M3-MXFP8/mlp.gate_proj+mlp.up_proj", + "nvidia/MiniMax-M3-NVFP4/mlp.gate_proj+mlp.up_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 3072, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + }, + "k": 6144, + "n": 24576, + "out": "bf16" + }, + "sources": [ + "MiniMaxAI/MiniMax-M3-MXFP8/mlp.gate_proj+mlp.up_proj", + "nvidia/MiniMax-M3-NVFP4/mlp.gate_proj+mlp.up_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 4096, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + }, + "k": 6144, + "n": 24576, + "out": "bf16" + }, + "sources": [ + "MiniMaxAI/MiniMax-M3-MXFP8/mlp.gate_proj+mlp.up_proj", + "nvidia/MiniMax-M3-NVFP4/mlp.gate_proj+mlp.up_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 6144, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + }, + "k": 6144, + "n": 24576, + "out": "bf16" + }, + "sources": [ + "MiniMaxAI/MiniMax-M3-MXFP8/mlp.gate_proj+mlp.up_proj", + "nvidia/MiniMax-M3-NVFP4/mlp.gate_proj+mlp.up_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 8192, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + }, + "k": 6144, + "n": 24576, + "out": "bf16" + }, + "sources": [ + "MiniMaxAI/MiniMax-M3-MXFP8/mlp.gate_proj+mlp.up_proj", + "nvidia/MiniMax-M3-NVFP4/mlp.gate_proj+mlp.up_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 12288, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + }, + "k": 6144, + "n": 24576, + "out": "bf16" + }, + "sources": [ + "MiniMaxAI/MiniMax-M3-MXFP8/mlp.gate_proj+mlp.up_proj", + "nvidia/MiniMax-M3-NVFP4/mlp.gate_proj+mlp.up_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 16384, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + }, + "k": 6144, + "n": 24576, + "out": "bf16" + }, + "sources": [ + "MiniMaxAI/MiniMax-M3-MXFP8/mlp.gate_proj+mlp.up_proj", + "nvidia/MiniMax-M3-NVFP4/mlp.gate_proj+mlp.up_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 24576, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + }, + "k": 6144, + "n": 24576, + "out": "bf16" + }, + "sources": [ + "MiniMaxAI/MiniMax-M3-MXFP8/mlp.gate_proj+mlp.up_proj", + "nvidia/MiniMax-M3-NVFP4/mlp.gate_proj+mlp.up_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 32768, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + }, + "k": 6144, + "n": 24576, + "out": "bf16" + }, + "sources": [ + "MiniMaxAI/MiniMax-M3-MXFP8/mlp.gate_proj+mlp.up_proj", + "nvidia/MiniMax-M3-NVFP4/mlp.gate_proj+mlp.up_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 49152, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + }, + "k": 6144, + "n": 24576, + "out": "bf16" + }, + "sources": [ + "MiniMaxAI/MiniMax-M3-MXFP8/mlp.gate_proj+mlp.up_proj", + "nvidia/MiniMax-M3-NVFP4/mlp.gate_proj+mlp.up_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 65536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + }, + "k": 6144, + "n": 24576, + "out": "bf16" + }, + "sources": [ + "MiniMaxAI/MiniMax-M3-MXFP8/mlp.gate_proj+mlp.up_proj", + "nvidia/MiniMax-M3-NVFP4/mlp.gate_proj+mlp.up_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 1, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + }, + "k": 6144, + "n": 128, + "out": "bf16" + }, + "sources": [ + "MiniMaxAI/MiniMax-M3-MXFP8/self_attn.index_k_proj", + "nvidia/MiniMax-M3-NVFP4/self_attn.index_k_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 2, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + }, + "k": 6144, + "n": 128, + "out": "bf16" + }, + "sources": [ + "MiniMaxAI/MiniMax-M3-MXFP8/self_attn.index_k_proj", + "nvidia/MiniMax-M3-NVFP4/self_attn.index_k_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 3, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + }, + "k": 6144, + "n": 128, + "out": "bf16" + }, + "sources": [ + "MiniMaxAI/MiniMax-M3-MXFP8/self_attn.index_k_proj", + "nvidia/MiniMax-M3-NVFP4/self_attn.index_k_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 4, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + }, + "k": 6144, + "n": 128, + "out": "bf16" + }, + "sources": [ + "MiniMaxAI/MiniMax-M3-MXFP8/self_attn.index_k_proj", + "nvidia/MiniMax-M3-NVFP4/self_attn.index_k_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 6, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + }, + "k": 6144, + "n": 128, + "out": "bf16" + }, + "sources": [ + "MiniMaxAI/MiniMax-M3-MXFP8/self_attn.index_k_proj", + "nvidia/MiniMax-M3-NVFP4/self_attn.index_k_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 8, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + }, + "k": 6144, + "n": 128, + "out": "bf16" + }, + "sources": [ + "MiniMaxAI/MiniMax-M3-MXFP8/self_attn.index_k_proj", + "nvidia/MiniMax-M3-NVFP4/self_attn.index_k_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 12, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + }, + "k": 6144, + "n": 128, + "out": "bf16" + }, + "sources": [ + "MiniMaxAI/MiniMax-M3-MXFP8/self_attn.index_k_proj", + "nvidia/MiniMax-M3-NVFP4/self_attn.index_k_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 16, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + }, + "k": 6144, + "n": 128, + "out": "bf16" + }, + "sources": [ + "MiniMaxAI/MiniMax-M3-MXFP8/self_attn.index_k_proj", + "nvidia/MiniMax-M3-NVFP4/self_attn.index_k_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 24, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + }, + "k": 6144, + "n": 128, + "out": "bf16" + }, + "sources": [ + "MiniMaxAI/MiniMax-M3-MXFP8/self_attn.index_k_proj", + "nvidia/MiniMax-M3-NVFP4/self_attn.index_k_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 32, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + }, + "k": 6144, + "n": 128, + "out": "bf16" + }, + "sources": [ + "MiniMaxAI/MiniMax-M3-MXFP8/self_attn.index_k_proj", + "nvidia/MiniMax-M3-NVFP4/self_attn.index_k_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 48, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + }, + "k": 6144, + "n": 128, + "out": "bf16" + }, + "sources": [ + "MiniMaxAI/MiniMax-M3-MXFP8/self_attn.index_k_proj", + "nvidia/MiniMax-M3-NVFP4/self_attn.index_k_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 64, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + }, + "k": 6144, + "n": 128, + "out": "bf16" + }, + "sources": [ + "MiniMaxAI/MiniMax-M3-MXFP8/self_attn.index_k_proj", + "nvidia/MiniMax-M3-NVFP4/self_attn.index_k_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 96, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + }, + "k": 6144, + "n": 128, + "out": "bf16" + }, + "sources": [ + "MiniMaxAI/MiniMax-M3-MXFP8/self_attn.index_k_proj", + "nvidia/MiniMax-M3-NVFP4/self_attn.index_k_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 128, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + }, + "k": 6144, + "n": 128, + "out": "bf16" + }, + "sources": [ + "MiniMaxAI/MiniMax-M3-MXFP8/self_attn.index_k_proj", + "nvidia/MiniMax-M3-NVFP4/self_attn.index_k_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 192, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + }, + "k": 6144, + "n": 128, + "out": "bf16" + }, + "sources": [ + "MiniMaxAI/MiniMax-M3-MXFP8/self_attn.index_k_proj", + "nvidia/MiniMax-M3-NVFP4/self_attn.index_k_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 256, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + }, + "k": 6144, + "n": 128, + "out": "bf16" + }, + "sources": [ + "MiniMaxAI/MiniMax-M3-MXFP8/self_attn.index_k_proj", + "nvidia/MiniMax-M3-NVFP4/self_attn.index_k_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 384, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + }, + "k": 6144, + "n": 128, + "out": "bf16" + }, + "sources": [ + "MiniMaxAI/MiniMax-M3-MXFP8/self_attn.index_k_proj", + "nvidia/MiniMax-M3-NVFP4/self_attn.index_k_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 512, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + }, + "k": 6144, + "n": 128, + "out": "bf16" + }, + "sources": [ + "MiniMaxAI/MiniMax-M3-MXFP8/self_attn.index_k_proj", + "nvidia/MiniMax-M3-NVFP4/self_attn.index_k_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 768, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + }, + "k": 6144, + "n": 128, + "out": "bf16" + }, + "sources": [ + "MiniMaxAI/MiniMax-M3-MXFP8/self_attn.index_k_proj", + "nvidia/MiniMax-M3-NVFP4/self_attn.index_k_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 1024, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + }, + "k": 6144, + "n": 128, + "out": "bf16" + }, + "sources": [ + "MiniMaxAI/MiniMax-M3-MXFP8/self_attn.index_k_proj", + "nvidia/MiniMax-M3-NVFP4/self_attn.index_k_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + }, + "k": 6144, + "n": 128, + "out": "bf16" + }, + "sources": [ + "MiniMaxAI/MiniMax-M3-MXFP8/self_attn.index_k_proj", + "nvidia/MiniMax-M3-NVFP4/self_attn.index_k_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 2048, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + }, + "k": 6144, + "n": 128, + "out": "bf16" + }, + "sources": [ + "MiniMaxAI/MiniMax-M3-MXFP8/self_attn.index_k_proj", + "nvidia/MiniMax-M3-NVFP4/self_attn.index_k_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 3072, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + }, + "k": 6144, + "n": 128, + "out": "bf16" + }, + "sources": [ + "MiniMaxAI/MiniMax-M3-MXFP8/self_attn.index_k_proj", + "nvidia/MiniMax-M3-NVFP4/self_attn.index_k_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 4096, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + }, + "k": 6144, + "n": 128, + "out": "bf16" + }, + "sources": [ + "MiniMaxAI/MiniMax-M3-MXFP8/self_attn.index_k_proj", + "nvidia/MiniMax-M3-NVFP4/self_attn.index_k_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 6144, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + }, + "k": 6144, + "n": 128, + "out": "bf16" + }, + "sources": [ + "MiniMaxAI/MiniMax-M3-MXFP8/self_attn.index_k_proj", + "nvidia/MiniMax-M3-NVFP4/self_attn.index_k_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 8192, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + }, + "k": 6144, + "n": 128, + "out": "bf16" + }, + "sources": [ + "MiniMaxAI/MiniMax-M3-MXFP8/self_attn.index_k_proj", + "nvidia/MiniMax-M3-NVFP4/self_attn.index_k_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 12288, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + }, + "k": 6144, + "n": 128, + "out": "bf16" + }, + "sources": [ + "MiniMaxAI/MiniMax-M3-MXFP8/self_attn.index_k_proj", + "nvidia/MiniMax-M3-NVFP4/self_attn.index_k_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 16384, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + }, + "k": 6144, + "n": 128, + "out": "bf16" + }, + "sources": [ + "MiniMaxAI/MiniMax-M3-MXFP8/self_attn.index_k_proj", + "nvidia/MiniMax-M3-NVFP4/self_attn.index_k_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 24576, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + }, + "k": 6144, + "n": 128, + "out": "bf16" + }, + "sources": [ + "MiniMaxAI/MiniMax-M3-MXFP8/self_attn.index_k_proj", + "nvidia/MiniMax-M3-NVFP4/self_attn.index_k_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 32768, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + }, + "k": 6144, + "n": 128, + "out": "bf16" + }, + "sources": [ + "MiniMaxAI/MiniMax-M3-MXFP8/self_attn.index_k_proj", + "nvidia/MiniMax-M3-NVFP4/self_attn.index_k_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 49152, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + }, + "k": 6144, + "n": 128, + "out": "bf16" + }, + "sources": [ + "MiniMaxAI/MiniMax-M3-MXFP8/self_attn.index_k_proj", + "nvidia/MiniMax-M3-NVFP4/self_attn.index_k_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 65536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + }, + "k": 6144, + "n": 128, + "out": "bf16" + }, + "sources": [ + "MiniMaxAI/MiniMax-M3-MXFP8/self_attn.index_k_proj", + "nvidia/MiniMax-M3-NVFP4/self_attn.index_k_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 1, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + }, + "k": 6144, + "n": 512, + "out": "bf16" + }, + "sources": [ + "MiniMaxAI/MiniMax-M3-MXFP8/self_attn.index_q_proj", + "nvidia/MiniMax-M3-NVFP4/self_attn.index_q_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 2, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + }, + "k": 6144, + "n": 512, + "out": "bf16" + }, + "sources": [ + "MiniMaxAI/MiniMax-M3-MXFP8/self_attn.index_q_proj", + "nvidia/MiniMax-M3-NVFP4/self_attn.index_q_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 3, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + }, + "k": 6144, + "n": 512, + "out": "bf16" + }, + "sources": [ + "MiniMaxAI/MiniMax-M3-MXFP8/self_attn.index_q_proj", + "nvidia/MiniMax-M3-NVFP4/self_attn.index_q_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 4, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + }, + "k": 6144, + "n": 512, + "out": "bf16" + }, + "sources": [ + "MiniMaxAI/MiniMax-M3-MXFP8/self_attn.index_q_proj", + "nvidia/MiniMax-M3-NVFP4/self_attn.index_q_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 6, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + }, + "k": 6144, + "n": 512, + "out": "bf16" + }, + "sources": [ + "MiniMaxAI/MiniMax-M3-MXFP8/self_attn.index_q_proj", + "nvidia/MiniMax-M3-NVFP4/self_attn.index_q_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 8, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + }, + "k": 6144, + "n": 512, + "out": "bf16" + }, + "sources": [ + "MiniMaxAI/MiniMax-M3-MXFP8/self_attn.index_q_proj", + "nvidia/MiniMax-M3-NVFP4/self_attn.index_q_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 12, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + }, + "k": 6144, + "n": 512, + "out": "bf16" + }, + "sources": [ + "MiniMaxAI/MiniMax-M3-MXFP8/self_attn.index_q_proj", + "nvidia/MiniMax-M3-NVFP4/self_attn.index_q_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 16, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + }, + "k": 6144, + "n": 512, + "out": "bf16" + }, + "sources": [ + "MiniMaxAI/MiniMax-M3-MXFP8/self_attn.index_q_proj", + "nvidia/MiniMax-M3-NVFP4/self_attn.index_q_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 24, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + }, + "k": 6144, + "n": 512, + "out": "bf16" + }, + "sources": [ + "MiniMaxAI/MiniMax-M3-MXFP8/self_attn.index_q_proj", + "nvidia/MiniMax-M3-NVFP4/self_attn.index_q_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 32, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + }, + "k": 6144, + "n": 512, + "out": "bf16" + }, + "sources": [ + "MiniMaxAI/MiniMax-M3-MXFP8/self_attn.index_q_proj", + "nvidia/MiniMax-M3-NVFP4/self_attn.index_q_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 48, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + }, + "k": 6144, + "n": 512, + "out": "bf16" + }, + "sources": [ + "MiniMaxAI/MiniMax-M3-MXFP8/self_attn.index_q_proj", + "nvidia/MiniMax-M3-NVFP4/self_attn.index_q_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 64, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + }, + "k": 6144, + "n": 512, + "out": "bf16" + }, + "sources": [ + "MiniMaxAI/MiniMax-M3-MXFP8/self_attn.index_q_proj", + "nvidia/MiniMax-M3-NVFP4/self_attn.index_q_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 96, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + }, + "k": 6144, + "n": 512, + "out": "bf16" + }, + "sources": [ + "MiniMaxAI/MiniMax-M3-MXFP8/self_attn.index_q_proj", + "nvidia/MiniMax-M3-NVFP4/self_attn.index_q_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 128, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + }, + "k": 6144, + "n": 512, + "out": "bf16" + }, + "sources": [ + "MiniMaxAI/MiniMax-M3-MXFP8/self_attn.index_q_proj", + "nvidia/MiniMax-M3-NVFP4/self_attn.index_q_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 192, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + }, + "k": 6144, + "n": 512, + "out": "bf16" + }, + "sources": [ + "MiniMaxAI/MiniMax-M3-MXFP8/self_attn.index_q_proj", + "nvidia/MiniMax-M3-NVFP4/self_attn.index_q_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 256, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + }, + "k": 6144, + "n": 512, + "out": "bf16" + }, + "sources": [ + "MiniMaxAI/MiniMax-M3-MXFP8/self_attn.index_q_proj", + "nvidia/MiniMax-M3-NVFP4/self_attn.index_q_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 384, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + }, + "k": 6144, + "n": 512, + "out": "bf16" + }, + "sources": [ + "MiniMaxAI/MiniMax-M3-MXFP8/self_attn.index_q_proj", + "nvidia/MiniMax-M3-NVFP4/self_attn.index_q_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 512, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + }, + "k": 6144, + "n": 512, + "out": "bf16" + }, + "sources": [ + "MiniMaxAI/MiniMax-M3-MXFP8/self_attn.index_q_proj", + "nvidia/MiniMax-M3-NVFP4/self_attn.index_q_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 768, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + }, + "k": 6144, + "n": 512, + "out": "bf16" + }, + "sources": [ + "MiniMaxAI/MiniMax-M3-MXFP8/self_attn.index_q_proj", + "nvidia/MiniMax-M3-NVFP4/self_attn.index_q_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 1024, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + }, + "k": 6144, + "n": 512, + "out": "bf16" + }, + "sources": [ + "MiniMaxAI/MiniMax-M3-MXFP8/self_attn.index_q_proj", + "nvidia/MiniMax-M3-NVFP4/self_attn.index_q_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + }, + "k": 6144, + "n": 512, + "out": "bf16" + }, + "sources": [ + "MiniMaxAI/MiniMax-M3-MXFP8/self_attn.index_q_proj", + "nvidia/MiniMax-M3-NVFP4/self_attn.index_q_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 2048, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + }, + "k": 6144, + "n": 512, + "out": "bf16" + }, + "sources": [ + "MiniMaxAI/MiniMax-M3-MXFP8/self_attn.index_q_proj", + "nvidia/MiniMax-M3-NVFP4/self_attn.index_q_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 3072, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + }, + "k": 6144, + "n": 512, + "out": "bf16" + }, + "sources": [ + "MiniMaxAI/MiniMax-M3-MXFP8/self_attn.index_q_proj", + "nvidia/MiniMax-M3-NVFP4/self_attn.index_q_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 4096, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + }, + "k": 6144, + "n": 512, + "out": "bf16" + }, + "sources": [ + "MiniMaxAI/MiniMax-M3-MXFP8/self_attn.index_q_proj", + "nvidia/MiniMax-M3-NVFP4/self_attn.index_q_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 6144, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + }, + "k": 6144, + "n": 512, + "out": "bf16" + }, + "sources": [ + "MiniMaxAI/MiniMax-M3-MXFP8/self_attn.index_q_proj", + "nvidia/MiniMax-M3-NVFP4/self_attn.index_q_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 8192, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + }, + "k": 6144, + "n": 512, + "out": "bf16" + }, + "sources": [ + "MiniMaxAI/MiniMax-M3-MXFP8/self_attn.index_q_proj", + "nvidia/MiniMax-M3-NVFP4/self_attn.index_q_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 12288, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + }, + "k": 6144, + "n": 512, + "out": "bf16" + }, + "sources": [ + "MiniMaxAI/MiniMax-M3-MXFP8/self_attn.index_q_proj", + "nvidia/MiniMax-M3-NVFP4/self_attn.index_q_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 16384, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + }, + "k": 6144, + "n": 512, + "out": "bf16" + }, + "sources": [ + "MiniMaxAI/MiniMax-M3-MXFP8/self_attn.index_q_proj", + "nvidia/MiniMax-M3-NVFP4/self_attn.index_q_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 24576, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + }, + "k": 6144, + "n": 512, + "out": "bf16" + }, + "sources": [ + "MiniMaxAI/MiniMax-M3-MXFP8/self_attn.index_q_proj", + "nvidia/MiniMax-M3-NVFP4/self_attn.index_q_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 32768, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + }, + "k": 6144, + "n": 512, + "out": "bf16" + }, + "sources": [ + "MiniMaxAI/MiniMax-M3-MXFP8/self_attn.index_q_proj", + "nvidia/MiniMax-M3-NVFP4/self_attn.index_q_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 49152, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + }, + "k": 6144, + "n": 512, + "out": "bf16" + }, + "sources": [ + "MiniMaxAI/MiniMax-M3-MXFP8/self_attn.index_q_proj", + "nvidia/MiniMax-M3-NVFP4/self_attn.index_q_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 65536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + }, + "k": 6144, + "n": 512, + "out": "bf16" + }, + "sources": [ + "MiniMaxAI/MiniMax-M3-MXFP8/self_attn.index_q_proj", + "nvidia/MiniMax-M3-NVFP4/self_attn.index_q_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 1, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 6144, + "n": 640, + "out": "bf16" + }, + "sources": [ + "MiniMaxAI/MiniMax-M3-MXFP8/self_attn.index_q_proj+self_attn.index_k_proj", + "amd/MiniMax-M3-MXFP4/self_attn.index_q_proj+self_attn.index_k_proj", + "nvidia/MiniMax-M3-NVFP4/self_attn.index_q_proj+self_attn.index_k_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 2, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 6144, + "n": 640, + "out": "bf16" + }, + "sources": [ + "MiniMaxAI/MiniMax-M3-MXFP8/self_attn.index_q_proj+self_attn.index_k_proj", + "amd/MiniMax-M3-MXFP4/self_attn.index_q_proj+self_attn.index_k_proj", + "nvidia/MiniMax-M3-NVFP4/self_attn.index_q_proj+self_attn.index_k_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 3, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 6144, + "n": 640, + "out": "bf16" + }, + "sources": [ + "MiniMaxAI/MiniMax-M3-MXFP8/self_attn.index_q_proj+self_attn.index_k_proj", + "amd/MiniMax-M3-MXFP4/self_attn.index_q_proj+self_attn.index_k_proj", + "nvidia/MiniMax-M3-NVFP4/self_attn.index_q_proj+self_attn.index_k_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 4, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 6144, + "n": 640, + "out": "bf16" + }, + "sources": [ + "MiniMaxAI/MiniMax-M3-MXFP8/self_attn.index_q_proj+self_attn.index_k_proj", + "amd/MiniMax-M3-MXFP4/self_attn.index_q_proj+self_attn.index_k_proj", + "nvidia/MiniMax-M3-NVFP4/self_attn.index_q_proj+self_attn.index_k_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 6, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 6144, + "n": 640, + "out": "bf16" + }, + "sources": [ + "MiniMaxAI/MiniMax-M3-MXFP8/self_attn.index_q_proj+self_attn.index_k_proj", + "amd/MiniMax-M3-MXFP4/self_attn.index_q_proj+self_attn.index_k_proj", + "nvidia/MiniMax-M3-NVFP4/self_attn.index_q_proj+self_attn.index_k_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 8, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 6144, + "n": 640, + "out": "bf16" + }, + "sources": [ + "MiniMaxAI/MiniMax-M3-MXFP8/self_attn.index_q_proj+self_attn.index_k_proj", + "amd/MiniMax-M3-MXFP4/self_attn.index_q_proj+self_attn.index_k_proj", + "nvidia/MiniMax-M3-NVFP4/self_attn.index_q_proj+self_attn.index_k_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 12, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 6144, + "n": 640, + "out": "bf16" + }, + "sources": [ + "MiniMaxAI/MiniMax-M3-MXFP8/self_attn.index_q_proj+self_attn.index_k_proj", + "amd/MiniMax-M3-MXFP4/self_attn.index_q_proj+self_attn.index_k_proj", + "nvidia/MiniMax-M3-NVFP4/self_attn.index_q_proj+self_attn.index_k_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 16, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 6144, + "n": 640, + "out": "bf16" + }, + "sources": [ + "MiniMaxAI/MiniMax-M3-MXFP8/self_attn.index_q_proj+self_attn.index_k_proj", + "amd/MiniMax-M3-MXFP4/self_attn.index_q_proj+self_attn.index_k_proj", + "nvidia/MiniMax-M3-NVFP4/self_attn.index_q_proj+self_attn.index_k_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 24, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 6144, + "n": 640, + "out": "bf16" + }, + "sources": [ + "MiniMaxAI/MiniMax-M3-MXFP8/self_attn.index_q_proj+self_attn.index_k_proj", + "amd/MiniMax-M3-MXFP4/self_attn.index_q_proj+self_attn.index_k_proj", + "nvidia/MiniMax-M3-NVFP4/self_attn.index_q_proj+self_attn.index_k_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 32, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 6144, + "n": 640, + "out": "bf16" + }, + "sources": [ + "MiniMaxAI/MiniMax-M3-MXFP8/self_attn.index_q_proj+self_attn.index_k_proj", + "amd/MiniMax-M3-MXFP4/self_attn.index_q_proj+self_attn.index_k_proj", + "nvidia/MiniMax-M3-NVFP4/self_attn.index_q_proj+self_attn.index_k_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 48, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 6144, + "n": 640, + "out": "bf16" + }, + "sources": [ + "MiniMaxAI/MiniMax-M3-MXFP8/self_attn.index_q_proj+self_attn.index_k_proj", + "amd/MiniMax-M3-MXFP4/self_attn.index_q_proj+self_attn.index_k_proj", + "nvidia/MiniMax-M3-NVFP4/self_attn.index_q_proj+self_attn.index_k_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 64, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 6144, + "n": 640, + "out": "bf16" + }, + "sources": [ + "MiniMaxAI/MiniMax-M3-MXFP8/self_attn.index_q_proj+self_attn.index_k_proj", + "amd/MiniMax-M3-MXFP4/self_attn.index_q_proj+self_attn.index_k_proj", + "nvidia/MiniMax-M3-NVFP4/self_attn.index_q_proj+self_attn.index_k_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 96, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 6144, + "n": 640, + "out": "bf16" + }, + "sources": [ + "MiniMaxAI/MiniMax-M3-MXFP8/self_attn.index_q_proj+self_attn.index_k_proj", + "amd/MiniMax-M3-MXFP4/self_attn.index_q_proj+self_attn.index_k_proj", + "nvidia/MiniMax-M3-NVFP4/self_attn.index_q_proj+self_attn.index_k_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 128, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 6144, + "n": 640, + "out": "bf16" + }, + "sources": [ + "MiniMaxAI/MiniMax-M3-MXFP8/self_attn.index_q_proj+self_attn.index_k_proj", + "amd/MiniMax-M3-MXFP4/self_attn.index_q_proj+self_attn.index_k_proj", + "nvidia/MiniMax-M3-NVFP4/self_attn.index_q_proj+self_attn.index_k_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 192, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 6144, + "n": 640, + "out": "bf16" + }, + "sources": [ + "MiniMaxAI/MiniMax-M3-MXFP8/self_attn.index_q_proj+self_attn.index_k_proj", + "amd/MiniMax-M3-MXFP4/self_attn.index_q_proj+self_attn.index_k_proj", + "nvidia/MiniMax-M3-NVFP4/self_attn.index_q_proj+self_attn.index_k_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 256, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 6144, + "n": 640, + "out": "bf16" + }, + "sources": [ + "MiniMaxAI/MiniMax-M3-MXFP8/self_attn.index_q_proj+self_attn.index_k_proj", + "amd/MiniMax-M3-MXFP4/self_attn.index_q_proj+self_attn.index_k_proj", + "nvidia/MiniMax-M3-NVFP4/self_attn.index_q_proj+self_attn.index_k_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 384, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 6144, + "n": 640, + "out": "bf16" + }, + "sources": [ + "MiniMaxAI/MiniMax-M3-MXFP8/self_attn.index_q_proj+self_attn.index_k_proj", + "amd/MiniMax-M3-MXFP4/self_attn.index_q_proj+self_attn.index_k_proj", + "nvidia/MiniMax-M3-NVFP4/self_attn.index_q_proj+self_attn.index_k_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 512, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 6144, + "n": 640, + "out": "bf16" + }, + "sources": [ + "MiniMaxAI/MiniMax-M3-MXFP8/self_attn.index_q_proj+self_attn.index_k_proj", + "amd/MiniMax-M3-MXFP4/self_attn.index_q_proj+self_attn.index_k_proj", + "nvidia/MiniMax-M3-NVFP4/self_attn.index_q_proj+self_attn.index_k_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 768, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 6144, + "n": 640, + "out": "bf16" + }, + "sources": [ + "MiniMaxAI/MiniMax-M3-MXFP8/self_attn.index_q_proj+self_attn.index_k_proj", + "amd/MiniMax-M3-MXFP4/self_attn.index_q_proj+self_attn.index_k_proj", + "nvidia/MiniMax-M3-NVFP4/self_attn.index_q_proj+self_attn.index_k_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 1024, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 6144, + "n": 640, + "out": "bf16" + }, + "sources": [ + "MiniMaxAI/MiniMax-M3-MXFP8/self_attn.index_q_proj+self_attn.index_k_proj", + "amd/MiniMax-M3-MXFP4/self_attn.index_q_proj+self_attn.index_k_proj", + "nvidia/MiniMax-M3-NVFP4/self_attn.index_q_proj+self_attn.index_k_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 1536, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 6144, + "n": 640, + "out": "bf16" + }, + "sources": [ + "MiniMaxAI/MiniMax-M3-MXFP8/self_attn.index_q_proj+self_attn.index_k_proj", + "amd/MiniMax-M3-MXFP4/self_attn.index_q_proj+self_attn.index_k_proj", + "nvidia/MiniMax-M3-NVFP4/self_attn.index_q_proj+self_attn.index_k_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 2048, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 6144, + "n": 640, + "out": "bf16" + }, + "sources": [ + "MiniMaxAI/MiniMax-M3-MXFP8/self_attn.index_q_proj+self_attn.index_k_proj", + "amd/MiniMax-M3-MXFP4/self_attn.index_q_proj+self_attn.index_k_proj", + "nvidia/MiniMax-M3-NVFP4/self_attn.index_q_proj+self_attn.index_k_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 3072, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 6144, + "n": 640, + "out": "bf16" + }, + "sources": [ + "MiniMaxAI/MiniMax-M3-MXFP8/self_attn.index_q_proj+self_attn.index_k_proj", + "amd/MiniMax-M3-MXFP4/self_attn.index_q_proj+self_attn.index_k_proj", + "nvidia/MiniMax-M3-NVFP4/self_attn.index_q_proj+self_attn.index_k_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 4096, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 6144, + "n": 640, + "out": "bf16" + }, + "sources": [ + "MiniMaxAI/MiniMax-M3-MXFP8/self_attn.index_q_proj+self_attn.index_k_proj", + "amd/MiniMax-M3-MXFP4/self_attn.index_q_proj+self_attn.index_k_proj", + "nvidia/MiniMax-M3-NVFP4/self_attn.index_q_proj+self_attn.index_k_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 6144, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 6144, + "n": 640, + "out": "bf16" + }, + "sources": [ + "MiniMaxAI/MiniMax-M3-MXFP8/self_attn.index_q_proj+self_attn.index_k_proj", + "amd/MiniMax-M3-MXFP4/self_attn.index_q_proj+self_attn.index_k_proj", + "nvidia/MiniMax-M3-NVFP4/self_attn.index_q_proj+self_attn.index_k_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 8192, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 6144, + "n": 640, + "out": "bf16" + }, + "sources": [ + "MiniMaxAI/MiniMax-M3-MXFP8/self_attn.index_q_proj+self_attn.index_k_proj", + "amd/MiniMax-M3-MXFP4/self_attn.index_q_proj+self_attn.index_k_proj", + "nvidia/MiniMax-M3-NVFP4/self_attn.index_q_proj+self_attn.index_k_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 12288, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 6144, + "n": 640, + "out": "bf16" + }, + "sources": [ + "MiniMaxAI/MiniMax-M3-MXFP8/self_attn.index_q_proj+self_attn.index_k_proj", + "amd/MiniMax-M3-MXFP4/self_attn.index_q_proj+self_attn.index_k_proj", + "nvidia/MiniMax-M3-NVFP4/self_attn.index_q_proj+self_attn.index_k_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 16384, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 6144, + "n": 640, + "out": "bf16" + }, + "sources": [ + "MiniMaxAI/MiniMax-M3-MXFP8/self_attn.index_q_proj+self_attn.index_k_proj", + "amd/MiniMax-M3-MXFP4/self_attn.index_q_proj+self_attn.index_k_proj", + "nvidia/MiniMax-M3-NVFP4/self_attn.index_q_proj+self_attn.index_k_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 24576, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 6144, + "n": 640, + "out": "bf16" + }, + "sources": [ + "MiniMaxAI/MiniMax-M3-MXFP8/self_attn.index_q_proj+self_attn.index_k_proj", + "amd/MiniMax-M3-MXFP4/self_attn.index_q_proj+self_attn.index_k_proj", + "nvidia/MiniMax-M3-NVFP4/self_attn.index_q_proj+self_attn.index_k_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 32768, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 6144, + "n": 640, + "out": "bf16" + }, + "sources": [ + "MiniMaxAI/MiniMax-M3-MXFP8/self_attn.index_q_proj+self_attn.index_k_proj", + "amd/MiniMax-M3-MXFP4/self_attn.index_q_proj+self_attn.index_k_proj", + "nvidia/MiniMax-M3-NVFP4/self_attn.index_q_proj+self_attn.index_k_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 49152, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 6144, + "n": 640, + "out": "bf16" + }, + "sources": [ + "MiniMaxAI/MiniMax-M3-MXFP8/self_attn.index_q_proj+self_attn.index_k_proj", + "amd/MiniMax-M3-MXFP4/self_attn.index_q_proj+self_attn.index_k_proj", + "nvidia/MiniMax-M3-NVFP4/self_attn.index_q_proj+self_attn.index_k_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 65536, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 6144, + "n": 640, + "out": "bf16" + }, + "sources": [ + "MiniMaxAI/MiniMax-M3-MXFP8/self_attn.index_q_proj+self_attn.index_k_proj", + "amd/MiniMax-M3-MXFP4/self_attn.index_q_proj+self_attn.index_k_proj", + "nvidia/MiniMax-M3-NVFP4/self_attn.index_q_proj+self_attn.index_k_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 1, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + }, + "k": 8192, + "n": 6144, + "out": "bf16" + }, + "sources": [ + "MiniMaxAI/MiniMax-M3-MXFP8/self_attn.o_proj", + "nvidia/MiniMax-M3-NVFP4/self_attn.o_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 2, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + }, + "k": 8192, + "n": 6144, + "out": "bf16" + }, + "sources": [ + "MiniMaxAI/MiniMax-M3-MXFP8/self_attn.o_proj", + "nvidia/MiniMax-M3-NVFP4/self_attn.o_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 3, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + }, + "k": 8192, + "n": 6144, + "out": "bf16" + }, + "sources": [ + "MiniMaxAI/MiniMax-M3-MXFP8/self_attn.o_proj", + "nvidia/MiniMax-M3-NVFP4/self_attn.o_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 4, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + }, + "k": 8192, + "n": 6144, + "out": "bf16" + }, + "sources": [ + "MiniMaxAI/MiniMax-M3-MXFP8/self_attn.o_proj", + "nvidia/MiniMax-M3-NVFP4/self_attn.o_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 6, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + }, + "k": 8192, + "n": 6144, + "out": "bf16" + }, + "sources": [ + "MiniMaxAI/MiniMax-M3-MXFP8/self_attn.o_proj", + "nvidia/MiniMax-M3-NVFP4/self_attn.o_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 8, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + }, + "k": 8192, + "n": 6144, + "out": "bf16" + }, + "sources": [ + "MiniMaxAI/MiniMax-M3-MXFP8/self_attn.o_proj", + "nvidia/MiniMax-M3-NVFP4/self_attn.o_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 12, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + }, + "k": 8192, + "n": 6144, + "out": "bf16" + }, + "sources": [ + "MiniMaxAI/MiniMax-M3-MXFP8/self_attn.o_proj", + "nvidia/MiniMax-M3-NVFP4/self_attn.o_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 16, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + }, + "k": 8192, + "n": 6144, + "out": "bf16" + }, + "sources": [ + "MiniMaxAI/MiniMax-M3-MXFP8/self_attn.o_proj", + "nvidia/MiniMax-M3-NVFP4/self_attn.o_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 24, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + }, + "k": 8192, + "n": 6144, + "out": "bf16" + }, + "sources": [ + "MiniMaxAI/MiniMax-M3-MXFP8/self_attn.o_proj", + "nvidia/MiniMax-M3-NVFP4/self_attn.o_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 32, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + }, + "k": 8192, + "n": 6144, + "out": "bf16" + }, + "sources": [ + "MiniMaxAI/MiniMax-M3-MXFP8/self_attn.o_proj", + "nvidia/MiniMax-M3-NVFP4/self_attn.o_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 48, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + }, + "k": 8192, + "n": 6144, + "out": "bf16" + }, + "sources": [ + "MiniMaxAI/MiniMax-M3-MXFP8/self_attn.o_proj", + "nvidia/MiniMax-M3-NVFP4/self_attn.o_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 64, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + }, + "k": 8192, + "n": 6144, + "out": "bf16" + }, + "sources": [ + "MiniMaxAI/MiniMax-M3-MXFP8/self_attn.o_proj", + "nvidia/MiniMax-M3-NVFP4/self_attn.o_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 96, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + }, + "k": 8192, + "n": 6144, + "out": "bf16" + }, + "sources": [ + "MiniMaxAI/MiniMax-M3-MXFP8/self_attn.o_proj", + "nvidia/MiniMax-M3-NVFP4/self_attn.o_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 128, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + }, + "k": 8192, + "n": 6144, + "out": "bf16" + }, + "sources": [ + "MiniMaxAI/MiniMax-M3-MXFP8/self_attn.o_proj", + "nvidia/MiniMax-M3-NVFP4/self_attn.o_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 192, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + }, + "k": 8192, + "n": 6144, + "out": "bf16" + }, + "sources": [ + "MiniMaxAI/MiniMax-M3-MXFP8/self_attn.o_proj", + "nvidia/MiniMax-M3-NVFP4/self_attn.o_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 256, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + }, + "k": 8192, + "n": 6144, + "out": "bf16" + }, + "sources": [ + "MiniMaxAI/MiniMax-M3-MXFP8/self_attn.o_proj", + "nvidia/MiniMax-M3-NVFP4/self_attn.o_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 384, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + }, + "k": 8192, + "n": 6144, + "out": "bf16" + }, + "sources": [ + "MiniMaxAI/MiniMax-M3-MXFP8/self_attn.o_proj", + "nvidia/MiniMax-M3-NVFP4/self_attn.o_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 512, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + }, + "k": 8192, + "n": 6144, + "out": "bf16" + }, + "sources": [ + "MiniMaxAI/MiniMax-M3-MXFP8/self_attn.o_proj", + "nvidia/MiniMax-M3-NVFP4/self_attn.o_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 768, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + }, + "k": 8192, + "n": 6144, + "out": "bf16" + }, + "sources": [ + "MiniMaxAI/MiniMax-M3-MXFP8/self_attn.o_proj", + "nvidia/MiniMax-M3-NVFP4/self_attn.o_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 1024, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + }, + "k": 8192, + "n": 6144, + "out": "bf16" + }, + "sources": [ + "MiniMaxAI/MiniMax-M3-MXFP8/self_attn.o_proj", + "nvidia/MiniMax-M3-NVFP4/self_attn.o_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + }, + "k": 8192, + "n": 6144, + "out": "bf16" + }, + "sources": [ + "MiniMaxAI/MiniMax-M3-MXFP8/self_attn.o_proj", + "nvidia/MiniMax-M3-NVFP4/self_attn.o_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 2048, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + }, + "k": 8192, + "n": 6144, + "out": "bf16" + }, + "sources": [ + "MiniMaxAI/MiniMax-M3-MXFP8/self_attn.o_proj", + "nvidia/MiniMax-M3-NVFP4/self_attn.o_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 3072, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + }, + "k": 8192, + "n": 6144, + "out": "bf16" + }, + "sources": [ + "MiniMaxAI/MiniMax-M3-MXFP8/self_attn.o_proj", + "nvidia/MiniMax-M3-NVFP4/self_attn.o_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 4096, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + }, + "k": 8192, + "n": 6144, + "out": "bf16" + }, + "sources": [ + "MiniMaxAI/MiniMax-M3-MXFP8/self_attn.o_proj", + "nvidia/MiniMax-M3-NVFP4/self_attn.o_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 6144, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + }, + "k": 8192, + "n": 6144, + "out": "bf16" + }, + "sources": [ + "MiniMaxAI/MiniMax-M3-MXFP8/self_attn.o_proj", + "nvidia/MiniMax-M3-NVFP4/self_attn.o_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 8192, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + }, + "k": 8192, + "n": 6144, + "out": "bf16" + }, + "sources": [ + "MiniMaxAI/MiniMax-M3-MXFP8/self_attn.o_proj", + "nvidia/MiniMax-M3-NVFP4/self_attn.o_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 12288, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + }, + "k": 8192, + "n": 6144, + "out": "bf16" + }, + "sources": [ + "MiniMaxAI/MiniMax-M3-MXFP8/self_attn.o_proj", + "nvidia/MiniMax-M3-NVFP4/self_attn.o_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 16384, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + }, + "k": 8192, + "n": 6144, + "out": "bf16" + }, + "sources": [ + "MiniMaxAI/MiniMax-M3-MXFP8/self_attn.o_proj", + "nvidia/MiniMax-M3-NVFP4/self_attn.o_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 24576, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + }, + "k": 8192, + "n": 6144, + "out": "bf16" + }, + "sources": [ + "MiniMaxAI/MiniMax-M3-MXFP8/self_attn.o_proj", + "nvidia/MiniMax-M3-NVFP4/self_attn.o_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 32768, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + }, + "k": 8192, + "n": 6144, + "out": "bf16" + }, + "sources": [ + "MiniMaxAI/MiniMax-M3-MXFP8/self_attn.o_proj", + "nvidia/MiniMax-M3-NVFP4/self_attn.o_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 49152, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + }, + "k": 8192, + "n": 6144, + "out": "bf16" + }, + "sources": [ + "MiniMaxAI/MiniMax-M3-MXFP8/self_attn.o_proj", + "nvidia/MiniMax-M3-NVFP4/self_attn.o_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 65536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + }, + "k": 8192, + "n": 6144, + "out": "bf16" + }, + "sources": [ + "MiniMaxAI/MiniMax-M3-MXFP8/self_attn.o_proj", + "nvidia/MiniMax-M3-NVFP4/self_attn.o_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 1, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + }, + "k": 6144, + "n": 9216, + "out": "bf16" + }, + "sources": [ + "MiniMaxAI/MiniMax-M3-MXFP8/self_attn.q_proj+self_attn.k_proj+self_attn.v_proj", + "nvidia/MiniMax-M3-NVFP4/self_attn.q_proj+self_attn.k_proj+self_attn.v_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 2, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + }, + "k": 6144, + "n": 9216, + "out": "bf16" + }, + "sources": [ + "MiniMaxAI/MiniMax-M3-MXFP8/self_attn.q_proj+self_attn.k_proj+self_attn.v_proj", + "nvidia/MiniMax-M3-NVFP4/self_attn.q_proj+self_attn.k_proj+self_attn.v_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 3, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + }, + "k": 6144, + "n": 9216, + "out": "bf16" + }, + "sources": [ + "MiniMaxAI/MiniMax-M3-MXFP8/self_attn.q_proj+self_attn.k_proj+self_attn.v_proj", + "nvidia/MiniMax-M3-NVFP4/self_attn.q_proj+self_attn.k_proj+self_attn.v_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 4, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + }, + "k": 6144, + "n": 9216, + "out": "bf16" + }, + "sources": [ + "MiniMaxAI/MiniMax-M3-MXFP8/self_attn.q_proj+self_attn.k_proj+self_attn.v_proj", + "nvidia/MiniMax-M3-NVFP4/self_attn.q_proj+self_attn.k_proj+self_attn.v_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 6, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + }, + "k": 6144, + "n": 9216, + "out": "bf16" + }, + "sources": [ + "MiniMaxAI/MiniMax-M3-MXFP8/self_attn.q_proj+self_attn.k_proj+self_attn.v_proj", + "nvidia/MiniMax-M3-NVFP4/self_attn.q_proj+self_attn.k_proj+self_attn.v_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 8, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + }, + "k": 6144, + "n": 9216, + "out": "bf16" + }, + "sources": [ + "MiniMaxAI/MiniMax-M3-MXFP8/self_attn.q_proj+self_attn.k_proj+self_attn.v_proj", + "nvidia/MiniMax-M3-NVFP4/self_attn.q_proj+self_attn.k_proj+self_attn.v_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 12, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + }, + "k": 6144, + "n": 9216, + "out": "bf16" + }, + "sources": [ + "MiniMaxAI/MiniMax-M3-MXFP8/self_attn.q_proj+self_attn.k_proj+self_attn.v_proj", + "nvidia/MiniMax-M3-NVFP4/self_attn.q_proj+self_attn.k_proj+self_attn.v_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 16, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + }, + "k": 6144, + "n": 9216, + "out": "bf16" + }, + "sources": [ + "MiniMaxAI/MiniMax-M3-MXFP8/self_attn.q_proj+self_attn.k_proj+self_attn.v_proj", + "nvidia/MiniMax-M3-NVFP4/self_attn.q_proj+self_attn.k_proj+self_attn.v_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 24, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + }, + "k": 6144, + "n": 9216, + "out": "bf16" + }, + "sources": [ + "MiniMaxAI/MiniMax-M3-MXFP8/self_attn.q_proj+self_attn.k_proj+self_attn.v_proj", + "nvidia/MiniMax-M3-NVFP4/self_attn.q_proj+self_attn.k_proj+self_attn.v_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 32, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + }, + "k": 6144, + "n": 9216, + "out": "bf16" + }, + "sources": [ + "MiniMaxAI/MiniMax-M3-MXFP8/self_attn.q_proj+self_attn.k_proj+self_attn.v_proj", + "nvidia/MiniMax-M3-NVFP4/self_attn.q_proj+self_attn.k_proj+self_attn.v_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 48, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + }, + "k": 6144, + "n": 9216, + "out": "bf16" + }, + "sources": [ + "MiniMaxAI/MiniMax-M3-MXFP8/self_attn.q_proj+self_attn.k_proj+self_attn.v_proj", + "nvidia/MiniMax-M3-NVFP4/self_attn.q_proj+self_attn.k_proj+self_attn.v_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 64, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + }, + "k": 6144, + "n": 9216, + "out": "bf16" + }, + "sources": [ + "MiniMaxAI/MiniMax-M3-MXFP8/self_attn.q_proj+self_attn.k_proj+self_attn.v_proj", + "nvidia/MiniMax-M3-NVFP4/self_attn.q_proj+self_attn.k_proj+self_attn.v_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 96, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + }, + "k": 6144, + "n": 9216, + "out": "bf16" + }, + "sources": [ + "MiniMaxAI/MiniMax-M3-MXFP8/self_attn.q_proj+self_attn.k_proj+self_attn.v_proj", + "nvidia/MiniMax-M3-NVFP4/self_attn.q_proj+self_attn.k_proj+self_attn.v_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 128, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + }, + "k": 6144, + "n": 9216, + "out": "bf16" + }, + "sources": [ + "MiniMaxAI/MiniMax-M3-MXFP8/self_attn.q_proj+self_attn.k_proj+self_attn.v_proj", + "nvidia/MiniMax-M3-NVFP4/self_attn.q_proj+self_attn.k_proj+self_attn.v_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 192, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + }, + "k": 6144, + "n": 9216, + "out": "bf16" + }, + "sources": [ + "MiniMaxAI/MiniMax-M3-MXFP8/self_attn.q_proj+self_attn.k_proj+self_attn.v_proj", + "nvidia/MiniMax-M3-NVFP4/self_attn.q_proj+self_attn.k_proj+self_attn.v_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 256, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + }, + "k": 6144, + "n": 9216, + "out": "bf16" + }, + "sources": [ + "MiniMaxAI/MiniMax-M3-MXFP8/self_attn.q_proj+self_attn.k_proj+self_attn.v_proj", + "nvidia/MiniMax-M3-NVFP4/self_attn.q_proj+self_attn.k_proj+self_attn.v_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 384, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + }, + "k": 6144, + "n": 9216, + "out": "bf16" + }, + "sources": [ + "MiniMaxAI/MiniMax-M3-MXFP8/self_attn.q_proj+self_attn.k_proj+self_attn.v_proj", + "nvidia/MiniMax-M3-NVFP4/self_attn.q_proj+self_attn.k_proj+self_attn.v_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 512, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + }, + "k": 6144, + "n": 9216, + "out": "bf16" + }, + "sources": [ + "MiniMaxAI/MiniMax-M3-MXFP8/self_attn.q_proj+self_attn.k_proj+self_attn.v_proj", + "nvidia/MiniMax-M3-NVFP4/self_attn.q_proj+self_attn.k_proj+self_attn.v_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 768, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + }, + "k": 6144, + "n": 9216, + "out": "bf16" + }, + "sources": [ + "MiniMaxAI/MiniMax-M3-MXFP8/self_attn.q_proj+self_attn.k_proj+self_attn.v_proj", + "nvidia/MiniMax-M3-NVFP4/self_attn.q_proj+self_attn.k_proj+self_attn.v_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 1024, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + }, + "k": 6144, + "n": 9216, + "out": "bf16" + }, + "sources": [ + "MiniMaxAI/MiniMax-M3-MXFP8/self_attn.q_proj+self_attn.k_proj+self_attn.v_proj", + "nvidia/MiniMax-M3-NVFP4/self_attn.q_proj+self_attn.k_proj+self_attn.v_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + }, + "k": 6144, + "n": 9216, + "out": "bf16" + }, + "sources": [ + "MiniMaxAI/MiniMax-M3-MXFP8/self_attn.q_proj+self_attn.k_proj+self_attn.v_proj", + "nvidia/MiniMax-M3-NVFP4/self_attn.q_proj+self_attn.k_proj+self_attn.v_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 2048, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + }, + "k": 6144, + "n": 9216, + "out": "bf16" + }, + "sources": [ + "MiniMaxAI/MiniMax-M3-MXFP8/self_attn.q_proj+self_attn.k_proj+self_attn.v_proj", + "nvidia/MiniMax-M3-NVFP4/self_attn.q_proj+self_attn.k_proj+self_attn.v_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 3072, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + }, + "k": 6144, + "n": 9216, + "out": "bf16" + }, + "sources": [ + "MiniMaxAI/MiniMax-M3-MXFP8/self_attn.q_proj+self_attn.k_proj+self_attn.v_proj", + "nvidia/MiniMax-M3-NVFP4/self_attn.q_proj+self_attn.k_proj+self_attn.v_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 4096, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + }, + "k": 6144, + "n": 9216, + "out": "bf16" + }, + "sources": [ + "MiniMaxAI/MiniMax-M3-MXFP8/self_attn.q_proj+self_attn.k_proj+self_attn.v_proj", + "nvidia/MiniMax-M3-NVFP4/self_attn.q_proj+self_attn.k_proj+self_attn.v_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 6144, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + }, + "k": 6144, + "n": 9216, + "out": "bf16" + }, + "sources": [ + "MiniMaxAI/MiniMax-M3-MXFP8/self_attn.q_proj+self_attn.k_proj+self_attn.v_proj", + "nvidia/MiniMax-M3-NVFP4/self_attn.q_proj+self_attn.k_proj+self_attn.v_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 8192, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + }, + "k": 6144, + "n": 9216, + "out": "bf16" + }, + "sources": [ + "MiniMaxAI/MiniMax-M3-MXFP8/self_attn.q_proj+self_attn.k_proj+self_attn.v_proj", + "nvidia/MiniMax-M3-NVFP4/self_attn.q_proj+self_attn.k_proj+self_attn.v_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 12288, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + }, + "k": 6144, + "n": 9216, + "out": "bf16" + }, + "sources": [ + "MiniMaxAI/MiniMax-M3-MXFP8/self_attn.q_proj+self_attn.k_proj+self_attn.v_proj", + "nvidia/MiniMax-M3-NVFP4/self_attn.q_proj+self_attn.k_proj+self_attn.v_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 16384, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + }, + "k": 6144, + "n": 9216, + "out": "bf16" + }, + "sources": [ + "MiniMaxAI/MiniMax-M3-MXFP8/self_attn.q_proj+self_attn.k_proj+self_attn.v_proj", + "nvidia/MiniMax-M3-NVFP4/self_attn.q_proj+self_attn.k_proj+self_attn.v_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 24576, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + }, + "k": 6144, + "n": 9216, + "out": "bf16" + }, + "sources": [ + "MiniMaxAI/MiniMax-M3-MXFP8/self_attn.q_proj+self_attn.k_proj+self_attn.v_proj", + "nvidia/MiniMax-M3-NVFP4/self_attn.q_proj+self_attn.k_proj+self_attn.v_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 32768, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + }, + "k": 6144, + "n": 9216, + "out": "bf16" + }, + "sources": [ + "MiniMaxAI/MiniMax-M3-MXFP8/self_attn.q_proj+self_attn.k_proj+self_attn.v_proj", + "nvidia/MiniMax-M3-NVFP4/self_attn.q_proj+self_attn.k_proj+self_attn.v_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 49152, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + }, + "k": 6144, + "n": 9216, + "out": "bf16" + }, + "sources": [ + "MiniMaxAI/MiniMax-M3-MXFP8/self_attn.q_proj+self_attn.k_proj+self_attn.v_proj", + "nvidia/MiniMax-M3-NVFP4/self_attn.q_proj+self_attn.k_proj+self_attn.v_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 65536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + }, + "k": 6144, + "n": 9216, + "out": "bf16" + }, + "sources": [ + "MiniMaxAI/MiniMax-M3-MXFP8/self_attn.q_proj+self_attn.k_proj+self_attn.v_proj", + "nvidia/MiniMax-M3-NVFP4/self_attn.q_proj+self_attn.k_proj+self_attn.v_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 1, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + }, + "k": 6144, + "n": 9856, + "out": "bf16" + }, + "sources": [ + "MiniMaxAI/MiniMax-M3-MXFP8/self_attn.q_proj+self_attn.k_proj+self_attn.v_proj+self_attn.index_q_proj+self_attn.index_k_proj", + "nvidia/MiniMax-M3-NVFP4/self_attn.q_proj+self_attn.k_proj+self_attn.v_proj+self_attn.index_q_proj+self_attn.index_k_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 2, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + }, + "k": 6144, + "n": 9856, + "out": "bf16" + }, + "sources": [ + "MiniMaxAI/MiniMax-M3-MXFP8/self_attn.q_proj+self_attn.k_proj+self_attn.v_proj+self_attn.index_q_proj+self_attn.index_k_proj", + "nvidia/MiniMax-M3-NVFP4/self_attn.q_proj+self_attn.k_proj+self_attn.v_proj+self_attn.index_q_proj+self_attn.index_k_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 3, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + }, + "k": 6144, + "n": 9856, + "out": "bf16" + }, + "sources": [ + "MiniMaxAI/MiniMax-M3-MXFP8/self_attn.q_proj+self_attn.k_proj+self_attn.v_proj+self_attn.index_q_proj+self_attn.index_k_proj", + "nvidia/MiniMax-M3-NVFP4/self_attn.q_proj+self_attn.k_proj+self_attn.v_proj+self_attn.index_q_proj+self_attn.index_k_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 4, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + }, + "k": 6144, + "n": 9856, + "out": "bf16" + }, + "sources": [ + "MiniMaxAI/MiniMax-M3-MXFP8/self_attn.q_proj+self_attn.k_proj+self_attn.v_proj+self_attn.index_q_proj+self_attn.index_k_proj", + "nvidia/MiniMax-M3-NVFP4/self_attn.q_proj+self_attn.k_proj+self_attn.v_proj+self_attn.index_q_proj+self_attn.index_k_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 6, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + }, + "k": 6144, + "n": 9856, + "out": "bf16" + }, + "sources": [ + "MiniMaxAI/MiniMax-M3-MXFP8/self_attn.q_proj+self_attn.k_proj+self_attn.v_proj+self_attn.index_q_proj+self_attn.index_k_proj", + "nvidia/MiniMax-M3-NVFP4/self_attn.q_proj+self_attn.k_proj+self_attn.v_proj+self_attn.index_q_proj+self_attn.index_k_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 8, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + }, + "k": 6144, + "n": 9856, + "out": "bf16" + }, + "sources": [ + "MiniMaxAI/MiniMax-M3-MXFP8/self_attn.q_proj+self_attn.k_proj+self_attn.v_proj+self_attn.index_q_proj+self_attn.index_k_proj", + "nvidia/MiniMax-M3-NVFP4/self_attn.q_proj+self_attn.k_proj+self_attn.v_proj+self_attn.index_q_proj+self_attn.index_k_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 12, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + }, + "k": 6144, + "n": 9856, + "out": "bf16" + }, + "sources": [ + "MiniMaxAI/MiniMax-M3-MXFP8/self_attn.q_proj+self_attn.k_proj+self_attn.v_proj+self_attn.index_q_proj+self_attn.index_k_proj", + "nvidia/MiniMax-M3-NVFP4/self_attn.q_proj+self_attn.k_proj+self_attn.v_proj+self_attn.index_q_proj+self_attn.index_k_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 16, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + }, + "k": 6144, + "n": 9856, + "out": "bf16" + }, + "sources": [ + "MiniMaxAI/MiniMax-M3-MXFP8/self_attn.q_proj+self_attn.k_proj+self_attn.v_proj+self_attn.index_q_proj+self_attn.index_k_proj", + "nvidia/MiniMax-M3-NVFP4/self_attn.q_proj+self_attn.k_proj+self_attn.v_proj+self_attn.index_q_proj+self_attn.index_k_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 24, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + }, + "k": 6144, + "n": 9856, + "out": "bf16" + }, + "sources": [ + "MiniMaxAI/MiniMax-M3-MXFP8/self_attn.q_proj+self_attn.k_proj+self_attn.v_proj+self_attn.index_q_proj+self_attn.index_k_proj", + "nvidia/MiniMax-M3-NVFP4/self_attn.q_proj+self_attn.k_proj+self_attn.v_proj+self_attn.index_q_proj+self_attn.index_k_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 32, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + }, + "k": 6144, + "n": 9856, + "out": "bf16" + }, + "sources": [ + "MiniMaxAI/MiniMax-M3-MXFP8/self_attn.q_proj+self_attn.k_proj+self_attn.v_proj+self_attn.index_q_proj+self_attn.index_k_proj", + "nvidia/MiniMax-M3-NVFP4/self_attn.q_proj+self_attn.k_proj+self_attn.v_proj+self_attn.index_q_proj+self_attn.index_k_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 48, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + }, + "k": 6144, + "n": 9856, + "out": "bf16" + }, + "sources": [ + "MiniMaxAI/MiniMax-M3-MXFP8/self_attn.q_proj+self_attn.k_proj+self_attn.v_proj+self_attn.index_q_proj+self_attn.index_k_proj", + "nvidia/MiniMax-M3-NVFP4/self_attn.q_proj+self_attn.k_proj+self_attn.v_proj+self_attn.index_q_proj+self_attn.index_k_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 64, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + }, + "k": 6144, + "n": 9856, + "out": "bf16" + }, + "sources": [ + "MiniMaxAI/MiniMax-M3-MXFP8/self_attn.q_proj+self_attn.k_proj+self_attn.v_proj+self_attn.index_q_proj+self_attn.index_k_proj", + "nvidia/MiniMax-M3-NVFP4/self_attn.q_proj+self_attn.k_proj+self_attn.v_proj+self_attn.index_q_proj+self_attn.index_k_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 96, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + }, + "k": 6144, + "n": 9856, + "out": "bf16" + }, + "sources": [ + "MiniMaxAI/MiniMax-M3-MXFP8/self_attn.q_proj+self_attn.k_proj+self_attn.v_proj+self_attn.index_q_proj+self_attn.index_k_proj", + "nvidia/MiniMax-M3-NVFP4/self_attn.q_proj+self_attn.k_proj+self_attn.v_proj+self_attn.index_q_proj+self_attn.index_k_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 128, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + }, + "k": 6144, + "n": 9856, + "out": "bf16" + }, + "sources": [ + "MiniMaxAI/MiniMax-M3-MXFP8/self_attn.q_proj+self_attn.k_proj+self_attn.v_proj+self_attn.index_q_proj+self_attn.index_k_proj", + "nvidia/MiniMax-M3-NVFP4/self_attn.q_proj+self_attn.k_proj+self_attn.v_proj+self_attn.index_q_proj+self_attn.index_k_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 192, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + }, + "k": 6144, + "n": 9856, + "out": "bf16" + }, + "sources": [ + "MiniMaxAI/MiniMax-M3-MXFP8/self_attn.q_proj+self_attn.k_proj+self_attn.v_proj+self_attn.index_q_proj+self_attn.index_k_proj", + "nvidia/MiniMax-M3-NVFP4/self_attn.q_proj+self_attn.k_proj+self_attn.v_proj+self_attn.index_q_proj+self_attn.index_k_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 256, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + }, + "k": 6144, + "n": 9856, + "out": "bf16" + }, + "sources": [ + "MiniMaxAI/MiniMax-M3-MXFP8/self_attn.q_proj+self_attn.k_proj+self_attn.v_proj+self_attn.index_q_proj+self_attn.index_k_proj", + "nvidia/MiniMax-M3-NVFP4/self_attn.q_proj+self_attn.k_proj+self_attn.v_proj+self_attn.index_q_proj+self_attn.index_k_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 384, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + }, + "k": 6144, + "n": 9856, + "out": "bf16" + }, + "sources": [ + "MiniMaxAI/MiniMax-M3-MXFP8/self_attn.q_proj+self_attn.k_proj+self_attn.v_proj+self_attn.index_q_proj+self_attn.index_k_proj", + "nvidia/MiniMax-M3-NVFP4/self_attn.q_proj+self_attn.k_proj+self_attn.v_proj+self_attn.index_q_proj+self_attn.index_k_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 512, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + }, + "k": 6144, + "n": 9856, + "out": "bf16" + }, + "sources": [ + "MiniMaxAI/MiniMax-M3-MXFP8/self_attn.q_proj+self_attn.k_proj+self_attn.v_proj+self_attn.index_q_proj+self_attn.index_k_proj", + "nvidia/MiniMax-M3-NVFP4/self_attn.q_proj+self_attn.k_proj+self_attn.v_proj+self_attn.index_q_proj+self_attn.index_k_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 768, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + }, + "k": 6144, + "n": 9856, + "out": "bf16" + }, + "sources": [ + "MiniMaxAI/MiniMax-M3-MXFP8/self_attn.q_proj+self_attn.k_proj+self_attn.v_proj+self_attn.index_q_proj+self_attn.index_k_proj", + "nvidia/MiniMax-M3-NVFP4/self_attn.q_proj+self_attn.k_proj+self_attn.v_proj+self_attn.index_q_proj+self_attn.index_k_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 1024, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + }, + "k": 6144, + "n": 9856, + "out": "bf16" + }, + "sources": [ + "MiniMaxAI/MiniMax-M3-MXFP8/self_attn.q_proj+self_attn.k_proj+self_attn.v_proj+self_attn.index_q_proj+self_attn.index_k_proj", + "nvidia/MiniMax-M3-NVFP4/self_attn.q_proj+self_attn.k_proj+self_attn.v_proj+self_attn.index_q_proj+self_attn.index_k_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + }, + "k": 6144, + "n": 9856, + "out": "bf16" + }, + "sources": [ + "MiniMaxAI/MiniMax-M3-MXFP8/self_attn.q_proj+self_attn.k_proj+self_attn.v_proj+self_attn.index_q_proj+self_attn.index_k_proj", + "nvidia/MiniMax-M3-NVFP4/self_attn.q_proj+self_attn.k_proj+self_attn.v_proj+self_attn.index_q_proj+self_attn.index_k_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 2048, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + }, + "k": 6144, + "n": 9856, + "out": "bf16" + }, + "sources": [ + "MiniMaxAI/MiniMax-M3-MXFP8/self_attn.q_proj+self_attn.k_proj+self_attn.v_proj+self_attn.index_q_proj+self_attn.index_k_proj", + "nvidia/MiniMax-M3-NVFP4/self_attn.q_proj+self_attn.k_proj+self_attn.v_proj+self_attn.index_q_proj+self_attn.index_k_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 3072, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + }, + "k": 6144, + "n": 9856, + "out": "bf16" + }, + "sources": [ + "MiniMaxAI/MiniMax-M3-MXFP8/self_attn.q_proj+self_attn.k_proj+self_attn.v_proj+self_attn.index_q_proj+self_attn.index_k_proj", + "nvidia/MiniMax-M3-NVFP4/self_attn.q_proj+self_attn.k_proj+self_attn.v_proj+self_attn.index_q_proj+self_attn.index_k_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 4096, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + }, + "k": 6144, + "n": 9856, + "out": "bf16" + }, + "sources": [ + "MiniMaxAI/MiniMax-M3-MXFP8/self_attn.q_proj+self_attn.k_proj+self_attn.v_proj+self_attn.index_q_proj+self_attn.index_k_proj", + "nvidia/MiniMax-M3-NVFP4/self_attn.q_proj+self_attn.k_proj+self_attn.v_proj+self_attn.index_q_proj+self_attn.index_k_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 6144, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + }, + "k": 6144, + "n": 9856, + "out": "bf16" + }, + "sources": [ + "MiniMaxAI/MiniMax-M3-MXFP8/self_attn.q_proj+self_attn.k_proj+self_attn.v_proj+self_attn.index_q_proj+self_attn.index_k_proj", + "nvidia/MiniMax-M3-NVFP4/self_attn.q_proj+self_attn.k_proj+self_attn.v_proj+self_attn.index_q_proj+self_attn.index_k_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 8192, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + }, + "k": 6144, + "n": 9856, + "out": "bf16" + }, + "sources": [ + "MiniMaxAI/MiniMax-M3-MXFP8/self_attn.q_proj+self_attn.k_proj+self_attn.v_proj+self_attn.index_q_proj+self_attn.index_k_proj", + "nvidia/MiniMax-M3-NVFP4/self_attn.q_proj+self_attn.k_proj+self_attn.v_proj+self_attn.index_q_proj+self_attn.index_k_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 12288, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + }, + "k": 6144, + "n": 9856, + "out": "bf16" + }, + "sources": [ + "MiniMaxAI/MiniMax-M3-MXFP8/self_attn.q_proj+self_attn.k_proj+self_attn.v_proj+self_attn.index_q_proj+self_attn.index_k_proj", + "nvidia/MiniMax-M3-NVFP4/self_attn.q_proj+self_attn.k_proj+self_attn.v_proj+self_attn.index_q_proj+self_attn.index_k_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 16384, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + }, + "k": 6144, + "n": 9856, + "out": "bf16" + }, + "sources": [ + "MiniMaxAI/MiniMax-M3-MXFP8/self_attn.q_proj+self_attn.k_proj+self_attn.v_proj+self_attn.index_q_proj+self_attn.index_k_proj", + "nvidia/MiniMax-M3-NVFP4/self_attn.q_proj+self_attn.k_proj+self_attn.v_proj+self_attn.index_q_proj+self_attn.index_k_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 24576, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + }, + "k": 6144, + "n": 9856, + "out": "bf16" + }, + "sources": [ + "MiniMaxAI/MiniMax-M3-MXFP8/self_attn.q_proj+self_attn.k_proj+self_attn.v_proj+self_attn.index_q_proj+self_attn.index_k_proj", + "nvidia/MiniMax-M3-NVFP4/self_attn.q_proj+self_attn.k_proj+self_attn.v_proj+self_attn.index_q_proj+self_attn.index_k_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 32768, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + }, + "k": 6144, + "n": 9856, + "out": "bf16" + }, + "sources": [ + "MiniMaxAI/MiniMax-M3-MXFP8/self_attn.q_proj+self_attn.k_proj+self_attn.v_proj+self_attn.index_q_proj+self_attn.index_k_proj", + "nvidia/MiniMax-M3-NVFP4/self_attn.q_proj+self_attn.k_proj+self_attn.v_proj+self_attn.index_q_proj+self_attn.index_k_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 49152, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + }, + "k": 6144, + "n": 9856, + "out": "bf16" + }, + "sources": [ + "MiniMaxAI/MiniMax-M3-MXFP8/self_attn.q_proj+self_attn.k_proj+self_attn.v_proj+self_attn.index_q_proj+self_attn.index_k_proj", + "nvidia/MiniMax-M3-NVFP4/self_attn.q_proj+self_attn.k_proj+self_attn.v_proj+self_attn.index_q_proj+self_attn.index_k_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 65536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + }, + "k": 6144, + "n": 9856, + "out": "bf16" + }, + "sources": [ + "MiniMaxAI/MiniMax-M3-MXFP8/self_attn.q_proj+self_attn.k_proj+self_attn.v_proj+self_attn.index_q_proj+self_attn.index_k_proj", + "nvidia/MiniMax-M3-NVFP4/self_attn.q_proj+self_attn.k_proj+self_attn.v_proj+self_attn.index_q_proj+self_attn.index_k_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 1, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + }, + "k": 3072, + "n": 6144, + "out": "bf16" + }, + "sources": [ + "amd/MiniMax-M3-MXFP4/block_sparse_moe.shared_experts.down_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 2, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + }, + "k": 3072, + "n": 6144, + "out": "bf16" + }, + "sources": [ + "amd/MiniMax-M3-MXFP4/block_sparse_moe.shared_experts.down_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 3, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + }, + "k": 3072, + "n": 6144, + "out": "bf16" + }, + "sources": [ + "amd/MiniMax-M3-MXFP4/block_sparse_moe.shared_experts.down_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 4, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + }, + "k": 3072, + "n": 6144, + "out": "bf16" + }, + "sources": [ + "amd/MiniMax-M3-MXFP4/block_sparse_moe.shared_experts.down_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 6, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + }, + "k": 3072, + "n": 6144, + "out": "bf16" + }, + "sources": [ + "amd/MiniMax-M3-MXFP4/block_sparse_moe.shared_experts.down_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 8, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + }, + "k": 3072, + "n": 6144, + "out": "bf16" + }, + "sources": [ + "amd/MiniMax-M3-MXFP4/block_sparse_moe.shared_experts.down_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 12, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + }, + "k": 3072, + "n": 6144, + "out": "bf16" + }, + "sources": [ + "amd/MiniMax-M3-MXFP4/block_sparse_moe.shared_experts.down_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 16, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + }, + "k": 3072, + "n": 6144, + "out": "bf16" + }, + "sources": [ + "amd/MiniMax-M3-MXFP4/block_sparse_moe.shared_experts.down_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 24, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + }, + "k": 3072, + "n": 6144, + "out": "bf16" + }, + "sources": [ + "amd/MiniMax-M3-MXFP4/block_sparse_moe.shared_experts.down_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 32, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + }, + "k": 3072, + "n": 6144, + "out": "bf16" + }, + "sources": [ + "amd/MiniMax-M3-MXFP4/block_sparse_moe.shared_experts.down_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 48, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + }, + "k": 3072, + "n": 6144, + "out": "bf16" + }, + "sources": [ + "amd/MiniMax-M3-MXFP4/block_sparse_moe.shared_experts.down_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 64, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + }, + "k": 3072, + "n": 6144, + "out": "bf16" + }, + "sources": [ + "amd/MiniMax-M3-MXFP4/block_sparse_moe.shared_experts.down_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 96, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + }, + "k": 3072, + "n": 6144, + "out": "bf16" + }, + "sources": [ + "amd/MiniMax-M3-MXFP4/block_sparse_moe.shared_experts.down_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 128, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + }, + "k": 3072, + "n": 6144, + "out": "bf16" + }, + "sources": [ + "amd/MiniMax-M3-MXFP4/block_sparse_moe.shared_experts.down_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 192, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + }, + "k": 3072, + "n": 6144, + "out": "bf16" + }, + "sources": [ + "amd/MiniMax-M3-MXFP4/block_sparse_moe.shared_experts.down_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 256, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + }, + "k": 3072, + "n": 6144, + "out": "bf16" + }, + "sources": [ + "amd/MiniMax-M3-MXFP4/block_sparse_moe.shared_experts.down_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 384, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + }, + "k": 3072, + "n": 6144, + "out": "bf16" + }, + "sources": [ + "amd/MiniMax-M3-MXFP4/block_sparse_moe.shared_experts.down_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 512, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + }, + "k": 3072, + "n": 6144, + "out": "bf16" + }, + "sources": [ + "amd/MiniMax-M3-MXFP4/block_sparse_moe.shared_experts.down_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 768, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + }, + "k": 3072, + "n": 6144, + "out": "bf16" + }, + "sources": [ + "amd/MiniMax-M3-MXFP4/block_sparse_moe.shared_experts.down_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 1024, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + }, + "k": 3072, + "n": 6144, + "out": "bf16" + }, + "sources": [ + "amd/MiniMax-M3-MXFP4/block_sparse_moe.shared_experts.down_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 1536, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + }, + "k": 3072, + "n": 6144, + "out": "bf16" + }, + "sources": [ + "amd/MiniMax-M3-MXFP4/block_sparse_moe.shared_experts.down_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 2048, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + }, + "k": 3072, + "n": 6144, + "out": "bf16" + }, + "sources": [ + "amd/MiniMax-M3-MXFP4/block_sparse_moe.shared_experts.down_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 3072, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + }, + "k": 3072, + "n": 6144, + "out": "bf16" + }, + "sources": [ + "amd/MiniMax-M3-MXFP4/block_sparse_moe.shared_experts.down_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 4096, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + }, + "k": 3072, + "n": 6144, + "out": "bf16" + }, + "sources": [ + "amd/MiniMax-M3-MXFP4/block_sparse_moe.shared_experts.down_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 6144, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + }, + "k": 3072, + "n": 6144, + "out": "bf16" + }, + "sources": [ + "amd/MiniMax-M3-MXFP4/block_sparse_moe.shared_experts.down_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 8192, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + }, + "k": 3072, + "n": 6144, + "out": "bf16" + }, + "sources": [ + "amd/MiniMax-M3-MXFP4/block_sparse_moe.shared_experts.down_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 12288, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + }, + "k": 3072, + "n": 6144, + "out": "bf16" + }, + "sources": [ + "amd/MiniMax-M3-MXFP4/block_sparse_moe.shared_experts.down_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 16384, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + }, + "k": 3072, + "n": 6144, + "out": "bf16" + }, + "sources": [ + "amd/MiniMax-M3-MXFP4/block_sparse_moe.shared_experts.down_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 24576, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + }, + "k": 3072, + "n": 6144, + "out": "bf16" + }, + "sources": [ + "amd/MiniMax-M3-MXFP4/block_sparse_moe.shared_experts.down_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 32768, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + }, + "k": 3072, + "n": 6144, + "out": "bf16" + }, + "sources": [ + "amd/MiniMax-M3-MXFP4/block_sparse_moe.shared_experts.down_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 49152, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + }, + "k": 3072, + "n": 6144, + "out": "bf16" + }, + "sources": [ + "amd/MiniMax-M3-MXFP4/block_sparse_moe.shared_experts.down_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 65536, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + }, + "k": 3072, + "n": 6144, + "out": "bf16" + }, + "sources": [ + "amd/MiniMax-M3-MXFP4/block_sparse_moe.shared_experts.down_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 1, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + }, + "k": 6144, + "n": 6144, + "out": "bf16" + }, + "sources": [ + "amd/MiniMax-M3-MXFP4/block_sparse_moe.shared_experts.gate_proj+block_sparse_moe.shared_experts.up_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 2, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + }, + "k": 6144, + "n": 6144, + "out": "bf16" + }, + "sources": [ + "amd/MiniMax-M3-MXFP4/block_sparse_moe.shared_experts.gate_proj+block_sparse_moe.shared_experts.up_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 3, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + }, + "k": 6144, + "n": 6144, + "out": "bf16" + }, + "sources": [ + "amd/MiniMax-M3-MXFP4/block_sparse_moe.shared_experts.gate_proj+block_sparse_moe.shared_experts.up_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 4, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + }, + "k": 6144, + "n": 6144, + "out": "bf16" + }, + "sources": [ + "amd/MiniMax-M3-MXFP4/block_sparse_moe.shared_experts.gate_proj+block_sparse_moe.shared_experts.up_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 6, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + }, + "k": 6144, + "n": 6144, + "out": "bf16" + }, + "sources": [ + "amd/MiniMax-M3-MXFP4/block_sparse_moe.shared_experts.gate_proj+block_sparse_moe.shared_experts.up_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 8, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + }, + "k": 6144, + "n": 6144, + "out": "bf16" + }, + "sources": [ + "amd/MiniMax-M3-MXFP4/block_sparse_moe.shared_experts.gate_proj+block_sparse_moe.shared_experts.up_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 12, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + }, + "k": 6144, + "n": 6144, + "out": "bf16" + }, + "sources": [ + "amd/MiniMax-M3-MXFP4/block_sparse_moe.shared_experts.gate_proj+block_sparse_moe.shared_experts.up_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 16, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + }, + "k": 6144, + "n": 6144, + "out": "bf16" + }, + "sources": [ + "amd/MiniMax-M3-MXFP4/block_sparse_moe.shared_experts.gate_proj+block_sparse_moe.shared_experts.up_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 24, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + }, + "k": 6144, + "n": 6144, + "out": "bf16" + }, + "sources": [ + "amd/MiniMax-M3-MXFP4/block_sparse_moe.shared_experts.gate_proj+block_sparse_moe.shared_experts.up_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 32, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + }, + "k": 6144, + "n": 6144, + "out": "bf16" + }, + "sources": [ + "amd/MiniMax-M3-MXFP4/block_sparse_moe.shared_experts.gate_proj+block_sparse_moe.shared_experts.up_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 48, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + }, + "k": 6144, + "n": 6144, + "out": "bf16" + }, + "sources": [ + "amd/MiniMax-M3-MXFP4/block_sparse_moe.shared_experts.gate_proj+block_sparse_moe.shared_experts.up_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 64, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + }, + "k": 6144, + "n": 6144, + "out": "bf16" + }, + "sources": [ + "amd/MiniMax-M3-MXFP4/block_sparse_moe.shared_experts.gate_proj+block_sparse_moe.shared_experts.up_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 96, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + }, + "k": 6144, + "n": 6144, + "out": "bf16" + }, + "sources": [ + "amd/MiniMax-M3-MXFP4/block_sparse_moe.shared_experts.gate_proj+block_sparse_moe.shared_experts.up_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 128, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + }, + "k": 6144, + "n": 6144, + "out": "bf16" + }, + "sources": [ + "amd/MiniMax-M3-MXFP4/block_sparse_moe.shared_experts.gate_proj+block_sparse_moe.shared_experts.up_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 192, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + }, + "k": 6144, + "n": 6144, + "out": "bf16" + }, + "sources": [ + "amd/MiniMax-M3-MXFP4/block_sparse_moe.shared_experts.gate_proj+block_sparse_moe.shared_experts.up_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 256, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + }, + "k": 6144, + "n": 6144, + "out": "bf16" + }, + "sources": [ + "amd/MiniMax-M3-MXFP4/block_sparse_moe.shared_experts.gate_proj+block_sparse_moe.shared_experts.up_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 384, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + }, + "k": 6144, + "n": 6144, + "out": "bf16" + }, + "sources": [ + "amd/MiniMax-M3-MXFP4/block_sparse_moe.shared_experts.gate_proj+block_sparse_moe.shared_experts.up_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 512, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + }, + "k": 6144, + "n": 6144, + "out": "bf16" + }, + "sources": [ + "amd/MiniMax-M3-MXFP4/block_sparse_moe.shared_experts.gate_proj+block_sparse_moe.shared_experts.up_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 768, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + }, + "k": 6144, + "n": 6144, + "out": "bf16" + }, + "sources": [ + "amd/MiniMax-M3-MXFP4/block_sparse_moe.shared_experts.gate_proj+block_sparse_moe.shared_experts.up_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 1024, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + }, + "k": 6144, + "n": 6144, + "out": "bf16" + }, + "sources": [ + "amd/MiniMax-M3-MXFP4/block_sparse_moe.shared_experts.gate_proj+block_sparse_moe.shared_experts.up_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 1536, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + }, + "k": 6144, + "n": 6144, + "out": "bf16" + }, + "sources": [ + "amd/MiniMax-M3-MXFP4/block_sparse_moe.shared_experts.gate_proj+block_sparse_moe.shared_experts.up_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 2048, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + }, + "k": 6144, + "n": 6144, + "out": "bf16" + }, + "sources": [ + "amd/MiniMax-M3-MXFP4/block_sparse_moe.shared_experts.gate_proj+block_sparse_moe.shared_experts.up_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 3072, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + }, + "k": 6144, + "n": 6144, + "out": "bf16" + }, + "sources": [ + "amd/MiniMax-M3-MXFP4/block_sparse_moe.shared_experts.gate_proj+block_sparse_moe.shared_experts.up_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 4096, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + }, + "k": 6144, + "n": 6144, + "out": "bf16" + }, + "sources": [ + "amd/MiniMax-M3-MXFP4/block_sparse_moe.shared_experts.gate_proj+block_sparse_moe.shared_experts.up_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 6144, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + }, + "k": 6144, + "n": 6144, + "out": "bf16" + }, + "sources": [ + "amd/MiniMax-M3-MXFP4/block_sparse_moe.shared_experts.gate_proj+block_sparse_moe.shared_experts.up_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 8192, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + }, + "k": 6144, + "n": 6144, + "out": "bf16" + }, + "sources": [ + "amd/MiniMax-M3-MXFP4/block_sparse_moe.shared_experts.gate_proj+block_sparse_moe.shared_experts.up_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 12288, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + }, + "k": 6144, + "n": 6144, + "out": "bf16" + }, + "sources": [ + "amd/MiniMax-M3-MXFP4/block_sparse_moe.shared_experts.gate_proj+block_sparse_moe.shared_experts.up_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 16384, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + }, + "k": 6144, + "n": 6144, + "out": "bf16" + }, + "sources": [ + "amd/MiniMax-M3-MXFP4/block_sparse_moe.shared_experts.gate_proj+block_sparse_moe.shared_experts.up_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 24576, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + }, + "k": 6144, + "n": 6144, + "out": "bf16" + }, + "sources": [ + "amd/MiniMax-M3-MXFP4/block_sparse_moe.shared_experts.gate_proj+block_sparse_moe.shared_experts.up_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 32768, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + }, + "k": 6144, + "n": 6144, + "out": "bf16" + }, + "sources": [ + "amd/MiniMax-M3-MXFP4/block_sparse_moe.shared_experts.gate_proj+block_sparse_moe.shared_experts.up_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 49152, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + }, + "k": 6144, + "n": 6144, + "out": "bf16" + }, + "sources": [ + "amd/MiniMax-M3-MXFP4/block_sparse_moe.shared_experts.gate_proj+block_sparse_moe.shared_experts.up_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 65536, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + }, + "k": 6144, + "n": 6144, + "out": "bf16" + }, + "sources": [ + "amd/MiniMax-M3-MXFP4/block_sparse_moe.shared_experts.gate_proj+block_sparse_moe.shared_experts.up_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 1, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 12288, + "n": 6144, + "out": "bf16" + }, + "sources": [ + "amd/MiniMax-M3-MXFP4/mlp.down_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 2, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 12288, + "n": 6144, + "out": "bf16" + }, + "sources": [ + "amd/MiniMax-M3-MXFP4/mlp.down_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 3, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 12288, + "n": 6144, + "out": "bf16" + }, + "sources": [ + "amd/MiniMax-M3-MXFP4/mlp.down_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 4, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 12288, + "n": 6144, + "out": "bf16" + }, + "sources": [ + "amd/MiniMax-M3-MXFP4/mlp.down_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 6, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 12288, + "n": 6144, + "out": "bf16" + }, + "sources": [ + "amd/MiniMax-M3-MXFP4/mlp.down_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 8, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 12288, + "n": 6144, + "out": "bf16" + }, + "sources": [ + "amd/MiniMax-M3-MXFP4/mlp.down_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 12, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 12288, + "n": 6144, + "out": "bf16" + }, + "sources": [ + "amd/MiniMax-M3-MXFP4/mlp.down_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 16, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 12288, + "n": 6144, + "out": "bf16" + }, + "sources": [ + "amd/MiniMax-M3-MXFP4/mlp.down_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 24, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 12288, + "n": 6144, + "out": "bf16" + }, + "sources": [ + "amd/MiniMax-M3-MXFP4/mlp.down_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 32, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 12288, + "n": 6144, + "out": "bf16" + }, + "sources": [ + "amd/MiniMax-M3-MXFP4/mlp.down_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 48, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 12288, + "n": 6144, + "out": "bf16" + }, + "sources": [ + "amd/MiniMax-M3-MXFP4/mlp.down_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 64, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 12288, + "n": 6144, + "out": "bf16" + }, + "sources": [ + "amd/MiniMax-M3-MXFP4/mlp.down_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 96, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 12288, + "n": 6144, + "out": "bf16" + }, + "sources": [ + "amd/MiniMax-M3-MXFP4/mlp.down_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 128, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 12288, + "n": 6144, + "out": "bf16" + }, + "sources": [ + "amd/MiniMax-M3-MXFP4/mlp.down_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 192, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 12288, + "n": 6144, + "out": "bf16" + }, + "sources": [ + "amd/MiniMax-M3-MXFP4/mlp.down_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 256, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 12288, + "n": 6144, + "out": "bf16" + }, + "sources": [ + "amd/MiniMax-M3-MXFP4/mlp.down_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 384, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 12288, + "n": 6144, + "out": "bf16" + }, + "sources": [ + "amd/MiniMax-M3-MXFP4/mlp.down_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 512, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 12288, + "n": 6144, + "out": "bf16" + }, + "sources": [ + "amd/MiniMax-M3-MXFP4/mlp.down_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 768, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 12288, + "n": 6144, + "out": "bf16" + }, + "sources": [ + "amd/MiniMax-M3-MXFP4/mlp.down_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 1024, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 12288, + "n": 6144, + "out": "bf16" + }, + "sources": [ + "amd/MiniMax-M3-MXFP4/mlp.down_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 1536, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 12288, + "n": 6144, + "out": "bf16" + }, + "sources": [ + "amd/MiniMax-M3-MXFP4/mlp.down_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 2048, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 12288, + "n": 6144, + "out": "bf16" + }, + "sources": [ + "amd/MiniMax-M3-MXFP4/mlp.down_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 3072, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 12288, + "n": 6144, + "out": "bf16" + }, + "sources": [ + "amd/MiniMax-M3-MXFP4/mlp.down_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 4096, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 12288, + "n": 6144, + "out": "bf16" + }, + "sources": [ + "amd/MiniMax-M3-MXFP4/mlp.down_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 6144, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 12288, + "n": 6144, + "out": "bf16" + }, + "sources": [ + "amd/MiniMax-M3-MXFP4/mlp.down_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 8192, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 12288, + "n": 6144, + "out": "bf16" + }, + "sources": [ + "amd/MiniMax-M3-MXFP4/mlp.down_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 12288, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 12288, + "n": 6144, + "out": "bf16" + }, + "sources": [ + "amd/MiniMax-M3-MXFP4/mlp.down_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 16384, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 12288, + "n": 6144, + "out": "bf16" + }, + "sources": [ + "amd/MiniMax-M3-MXFP4/mlp.down_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 24576, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 12288, + "n": 6144, + "out": "bf16" + }, + "sources": [ + "amd/MiniMax-M3-MXFP4/mlp.down_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 32768, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 12288, + "n": 6144, + "out": "bf16" + }, + "sources": [ + "amd/MiniMax-M3-MXFP4/mlp.down_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 49152, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 12288, + "n": 6144, + "out": "bf16" + }, + "sources": [ + "amd/MiniMax-M3-MXFP4/mlp.down_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 65536, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 12288, + "n": 6144, + "out": "bf16" + }, + "sources": [ + "amd/MiniMax-M3-MXFP4/mlp.down_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 1, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 6144, + "n": 24576, + "out": "bf16" + }, + "sources": [ + "amd/MiniMax-M3-MXFP4/mlp.gate_proj+mlp.up_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 2, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 6144, + "n": 24576, + "out": "bf16" + }, + "sources": [ + "amd/MiniMax-M3-MXFP4/mlp.gate_proj+mlp.up_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 3, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 6144, + "n": 24576, + "out": "bf16" + }, + "sources": [ + "amd/MiniMax-M3-MXFP4/mlp.gate_proj+mlp.up_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 4, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 6144, + "n": 24576, + "out": "bf16" + }, + "sources": [ + "amd/MiniMax-M3-MXFP4/mlp.gate_proj+mlp.up_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 6, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 6144, + "n": 24576, + "out": "bf16" + }, + "sources": [ + "amd/MiniMax-M3-MXFP4/mlp.gate_proj+mlp.up_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 8, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 6144, + "n": 24576, + "out": "bf16" + }, + "sources": [ + "amd/MiniMax-M3-MXFP4/mlp.gate_proj+mlp.up_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 12, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 6144, + "n": 24576, + "out": "bf16" + }, + "sources": [ + "amd/MiniMax-M3-MXFP4/mlp.gate_proj+mlp.up_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 16, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 6144, + "n": 24576, + "out": "bf16" + }, + "sources": [ + "amd/MiniMax-M3-MXFP4/mlp.gate_proj+mlp.up_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 24, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 6144, + "n": 24576, + "out": "bf16" + }, + "sources": [ + "amd/MiniMax-M3-MXFP4/mlp.gate_proj+mlp.up_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 32, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 6144, + "n": 24576, + "out": "bf16" + }, + "sources": [ + "amd/MiniMax-M3-MXFP4/mlp.gate_proj+mlp.up_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 48, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 6144, + "n": 24576, + "out": "bf16" + }, + "sources": [ + "amd/MiniMax-M3-MXFP4/mlp.gate_proj+mlp.up_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 64, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 6144, + "n": 24576, + "out": "bf16" + }, + "sources": [ + "amd/MiniMax-M3-MXFP4/mlp.gate_proj+mlp.up_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 96, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 6144, + "n": 24576, + "out": "bf16" + }, + "sources": [ + "amd/MiniMax-M3-MXFP4/mlp.gate_proj+mlp.up_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 128, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 6144, + "n": 24576, + "out": "bf16" + }, + "sources": [ + "amd/MiniMax-M3-MXFP4/mlp.gate_proj+mlp.up_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 192, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 6144, + "n": 24576, + "out": "bf16" + }, + "sources": [ + "amd/MiniMax-M3-MXFP4/mlp.gate_proj+mlp.up_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 256, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 6144, + "n": 24576, + "out": "bf16" + }, + "sources": [ + "amd/MiniMax-M3-MXFP4/mlp.gate_proj+mlp.up_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 384, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 6144, + "n": 24576, + "out": "bf16" + }, + "sources": [ + "amd/MiniMax-M3-MXFP4/mlp.gate_proj+mlp.up_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 512, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 6144, + "n": 24576, + "out": "bf16" + }, + "sources": [ + "amd/MiniMax-M3-MXFP4/mlp.gate_proj+mlp.up_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 768, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 6144, + "n": 24576, + "out": "bf16" + }, + "sources": [ + "amd/MiniMax-M3-MXFP4/mlp.gate_proj+mlp.up_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 1024, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 6144, + "n": 24576, + "out": "bf16" + }, + "sources": [ + "amd/MiniMax-M3-MXFP4/mlp.gate_proj+mlp.up_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 1536, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 6144, + "n": 24576, + "out": "bf16" + }, + "sources": [ + "amd/MiniMax-M3-MXFP4/mlp.gate_proj+mlp.up_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 2048, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 6144, + "n": 24576, + "out": "bf16" + }, + "sources": [ + "amd/MiniMax-M3-MXFP4/mlp.gate_proj+mlp.up_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 3072, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 6144, + "n": 24576, + "out": "bf16" + }, + "sources": [ + "amd/MiniMax-M3-MXFP4/mlp.gate_proj+mlp.up_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 4096, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 6144, + "n": 24576, + "out": "bf16" + }, + "sources": [ + "amd/MiniMax-M3-MXFP4/mlp.gate_proj+mlp.up_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 6144, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 6144, + "n": 24576, + "out": "bf16" + }, + "sources": [ + "amd/MiniMax-M3-MXFP4/mlp.gate_proj+mlp.up_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 8192, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 6144, + "n": 24576, + "out": "bf16" + }, + "sources": [ + "amd/MiniMax-M3-MXFP4/mlp.gate_proj+mlp.up_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 12288, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 6144, + "n": 24576, + "out": "bf16" + }, + "sources": [ + "amd/MiniMax-M3-MXFP4/mlp.gate_proj+mlp.up_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 16384, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 6144, + "n": 24576, + "out": "bf16" + }, + "sources": [ + "amd/MiniMax-M3-MXFP4/mlp.gate_proj+mlp.up_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 24576, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 6144, + "n": 24576, + "out": "bf16" + }, + "sources": [ + "amd/MiniMax-M3-MXFP4/mlp.gate_proj+mlp.up_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 32768, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 6144, + "n": 24576, + "out": "bf16" + }, + "sources": [ + "amd/MiniMax-M3-MXFP4/mlp.gate_proj+mlp.up_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 49152, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 6144, + "n": 24576, + "out": "bf16" + }, + "sources": [ + "amd/MiniMax-M3-MXFP4/mlp.gate_proj+mlp.up_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 65536, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 6144, + "n": 24576, + "out": "bf16" + }, + "sources": [ + "amd/MiniMax-M3-MXFP4/mlp.gate_proj+mlp.up_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 1, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 6144, + "n": 512, + "out": "bf16" + }, + "sources": [ + "amd/MiniMax-M3-MXFP4/self_attn.index_q_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 2, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 6144, + "n": 512, + "out": "bf16" + }, + "sources": [ + "amd/MiniMax-M3-MXFP4/self_attn.index_q_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 3, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 6144, + "n": 512, + "out": "bf16" + }, + "sources": [ + "amd/MiniMax-M3-MXFP4/self_attn.index_q_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 4, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 6144, + "n": 512, + "out": "bf16" + }, + "sources": [ + "amd/MiniMax-M3-MXFP4/self_attn.index_q_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 6, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 6144, + "n": 512, + "out": "bf16" + }, + "sources": [ + "amd/MiniMax-M3-MXFP4/self_attn.index_q_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 8, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 6144, + "n": 512, + "out": "bf16" + }, + "sources": [ + "amd/MiniMax-M3-MXFP4/self_attn.index_q_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 12, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 6144, + "n": 512, + "out": "bf16" + }, + "sources": [ + "amd/MiniMax-M3-MXFP4/self_attn.index_q_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 16, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 6144, + "n": 512, + "out": "bf16" + }, + "sources": [ + "amd/MiniMax-M3-MXFP4/self_attn.index_q_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 24, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 6144, + "n": 512, + "out": "bf16" + }, + "sources": [ + "amd/MiniMax-M3-MXFP4/self_attn.index_q_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 32, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 6144, + "n": 512, + "out": "bf16" + }, + "sources": [ + "amd/MiniMax-M3-MXFP4/self_attn.index_q_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 48, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 6144, + "n": 512, + "out": "bf16" + }, + "sources": [ + "amd/MiniMax-M3-MXFP4/self_attn.index_q_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 64, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 6144, + "n": 512, + "out": "bf16" + }, + "sources": [ + "amd/MiniMax-M3-MXFP4/self_attn.index_q_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 96, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 6144, + "n": 512, + "out": "bf16" + }, + "sources": [ + "amd/MiniMax-M3-MXFP4/self_attn.index_q_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 128, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 6144, + "n": 512, + "out": "bf16" + }, + "sources": [ + "amd/MiniMax-M3-MXFP4/self_attn.index_q_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 192, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 6144, + "n": 512, + "out": "bf16" + }, + "sources": [ + "amd/MiniMax-M3-MXFP4/self_attn.index_q_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 256, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 6144, + "n": 512, + "out": "bf16" + }, + "sources": [ + "amd/MiniMax-M3-MXFP4/self_attn.index_q_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 384, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 6144, + "n": 512, + "out": "bf16" + }, + "sources": [ + "amd/MiniMax-M3-MXFP4/self_attn.index_q_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 512, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 6144, + "n": 512, + "out": "bf16" + }, + "sources": [ + "amd/MiniMax-M3-MXFP4/self_attn.index_q_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 768, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 6144, + "n": 512, + "out": "bf16" + }, + "sources": [ + "amd/MiniMax-M3-MXFP4/self_attn.index_q_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 1024, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 6144, + "n": 512, + "out": "bf16" + }, + "sources": [ + "amd/MiniMax-M3-MXFP4/self_attn.index_q_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 1536, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 6144, + "n": 512, + "out": "bf16" + }, + "sources": [ + "amd/MiniMax-M3-MXFP4/self_attn.index_q_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 2048, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 6144, + "n": 512, + "out": "bf16" + }, + "sources": [ + "amd/MiniMax-M3-MXFP4/self_attn.index_q_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 3072, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 6144, + "n": 512, + "out": "bf16" + }, + "sources": [ + "amd/MiniMax-M3-MXFP4/self_attn.index_q_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 4096, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 6144, + "n": 512, + "out": "bf16" + }, + "sources": [ + "amd/MiniMax-M3-MXFP4/self_attn.index_q_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 6144, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 6144, + "n": 512, + "out": "bf16" + }, + "sources": [ + "amd/MiniMax-M3-MXFP4/self_attn.index_q_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 8192, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 6144, + "n": 512, + "out": "bf16" + }, + "sources": [ + "amd/MiniMax-M3-MXFP4/self_attn.index_q_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 12288, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 6144, + "n": 512, + "out": "bf16" + }, + "sources": [ + "amd/MiniMax-M3-MXFP4/self_attn.index_q_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 16384, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 6144, + "n": 512, + "out": "bf16" + }, + "sources": [ + "amd/MiniMax-M3-MXFP4/self_attn.index_q_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 24576, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 6144, + "n": 512, + "out": "bf16" + }, + "sources": [ + "amd/MiniMax-M3-MXFP4/self_attn.index_q_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 32768, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 6144, + "n": 512, + "out": "bf16" + }, + "sources": [ + "amd/MiniMax-M3-MXFP4/self_attn.index_q_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 49152, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 6144, + "n": 512, + "out": "bf16" + }, + "sources": [ + "amd/MiniMax-M3-MXFP4/self_attn.index_q_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 65536, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 6144, + "n": 512, + "out": "bf16" + }, + "sources": [ + "amd/MiniMax-M3-MXFP4/self_attn.index_q_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 1, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 8192, + "n": 6144, + "out": "bf16" + }, + "sources": [ + "amd/MiniMax-M3-MXFP4/self_attn.o_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 2, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 8192, + "n": 6144, + "out": "bf16" + }, + "sources": [ + "amd/MiniMax-M3-MXFP4/self_attn.o_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 3, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 8192, + "n": 6144, + "out": "bf16" + }, + "sources": [ + "amd/MiniMax-M3-MXFP4/self_attn.o_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 4, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 8192, + "n": 6144, + "out": "bf16" + }, + "sources": [ + "amd/MiniMax-M3-MXFP4/self_attn.o_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 6, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 8192, + "n": 6144, + "out": "bf16" + }, + "sources": [ + "amd/MiniMax-M3-MXFP4/self_attn.o_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 8, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 8192, + "n": 6144, + "out": "bf16" + }, + "sources": [ + "amd/MiniMax-M3-MXFP4/self_attn.o_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 12, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 8192, + "n": 6144, + "out": "bf16" + }, + "sources": [ + "amd/MiniMax-M3-MXFP4/self_attn.o_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 16, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 8192, + "n": 6144, + "out": "bf16" + }, + "sources": [ + "amd/MiniMax-M3-MXFP4/self_attn.o_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 24, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 8192, + "n": 6144, + "out": "bf16" + }, + "sources": [ + "amd/MiniMax-M3-MXFP4/self_attn.o_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 32, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 8192, + "n": 6144, + "out": "bf16" + }, + "sources": [ + "amd/MiniMax-M3-MXFP4/self_attn.o_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 48, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 8192, + "n": 6144, + "out": "bf16" + }, + "sources": [ + "amd/MiniMax-M3-MXFP4/self_attn.o_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 64, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 8192, + "n": 6144, + "out": "bf16" + }, + "sources": [ + "amd/MiniMax-M3-MXFP4/self_attn.o_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 96, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 8192, + "n": 6144, + "out": "bf16" + }, + "sources": [ + "amd/MiniMax-M3-MXFP4/self_attn.o_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 128, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 8192, + "n": 6144, + "out": "bf16" + }, + "sources": [ + "amd/MiniMax-M3-MXFP4/self_attn.o_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 192, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 8192, + "n": 6144, + "out": "bf16" + }, + "sources": [ + "amd/MiniMax-M3-MXFP4/self_attn.o_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 256, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 8192, + "n": 6144, + "out": "bf16" + }, + "sources": [ + "amd/MiniMax-M3-MXFP4/self_attn.o_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 384, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 8192, + "n": 6144, + "out": "bf16" + }, + "sources": [ + "amd/MiniMax-M3-MXFP4/self_attn.o_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 512, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 8192, + "n": 6144, + "out": "bf16" + }, + "sources": [ + "amd/MiniMax-M3-MXFP4/self_attn.o_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 768, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 8192, + "n": 6144, + "out": "bf16" + }, + "sources": [ + "amd/MiniMax-M3-MXFP4/self_attn.o_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 1024, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 8192, + "n": 6144, + "out": "bf16" + }, + "sources": [ + "amd/MiniMax-M3-MXFP4/self_attn.o_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 1536, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 8192, + "n": 6144, + "out": "bf16" + }, + "sources": [ + "amd/MiniMax-M3-MXFP4/self_attn.o_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 2048, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 8192, + "n": 6144, + "out": "bf16" + }, + "sources": [ + "amd/MiniMax-M3-MXFP4/self_attn.o_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 3072, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 8192, + "n": 6144, + "out": "bf16" + }, + "sources": [ + "amd/MiniMax-M3-MXFP4/self_attn.o_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 4096, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 8192, + "n": 6144, + "out": "bf16" + }, + "sources": [ + "amd/MiniMax-M3-MXFP4/self_attn.o_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 6144, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 8192, + "n": 6144, + "out": "bf16" + }, + "sources": [ + "amd/MiniMax-M3-MXFP4/self_attn.o_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 8192, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 8192, + "n": 6144, + "out": "bf16" + }, + "sources": [ + "amd/MiniMax-M3-MXFP4/self_attn.o_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 12288, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 8192, + "n": 6144, + "out": "bf16" + }, + "sources": [ + "amd/MiniMax-M3-MXFP4/self_attn.o_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 16384, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 8192, + "n": 6144, + "out": "bf16" + }, + "sources": [ + "amd/MiniMax-M3-MXFP4/self_attn.o_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 24576, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 8192, + "n": 6144, + "out": "bf16" + }, + "sources": [ + "amd/MiniMax-M3-MXFP4/self_attn.o_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 32768, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 8192, + "n": 6144, + "out": "bf16" + }, + "sources": [ + "amd/MiniMax-M3-MXFP4/self_attn.o_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 49152, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 8192, + "n": 6144, + "out": "bf16" + }, + "sources": [ + "amd/MiniMax-M3-MXFP4/self_attn.o_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 65536, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 8192, + "n": 6144, + "out": "bf16" + }, + "sources": [ + "amd/MiniMax-M3-MXFP4/self_attn.o_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 1, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 6144, + "n": 9216, + "out": "bf16" + }, + "sources": [ + "amd/MiniMax-M3-MXFP4/self_attn.q_proj+self_attn.k_proj+self_attn.v_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 2, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 6144, + "n": 9216, + "out": "bf16" + }, + "sources": [ + "amd/MiniMax-M3-MXFP4/self_attn.q_proj+self_attn.k_proj+self_attn.v_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 3, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 6144, + "n": 9216, + "out": "bf16" + }, + "sources": [ + "amd/MiniMax-M3-MXFP4/self_attn.q_proj+self_attn.k_proj+self_attn.v_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 4, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 6144, + "n": 9216, + "out": "bf16" + }, + "sources": [ + "amd/MiniMax-M3-MXFP4/self_attn.q_proj+self_attn.k_proj+self_attn.v_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 6, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 6144, + "n": 9216, + "out": "bf16" + }, + "sources": [ + "amd/MiniMax-M3-MXFP4/self_attn.q_proj+self_attn.k_proj+self_attn.v_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 8, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 6144, + "n": 9216, + "out": "bf16" + }, + "sources": [ + "amd/MiniMax-M3-MXFP4/self_attn.q_proj+self_attn.k_proj+self_attn.v_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 12, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 6144, + "n": 9216, + "out": "bf16" + }, + "sources": [ + "amd/MiniMax-M3-MXFP4/self_attn.q_proj+self_attn.k_proj+self_attn.v_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 16, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 6144, + "n": 9216, + "out": "bf16" + }, + "sources": [ + "amd/MiniMax-M3-MXFP4/self_attn.q_proj+self_attn.k_proj+self_attn.v_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 24, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 6144, + "n": 9216, + "out": "bf16" + }, + "sources": [ + "amd/MiniMax-M3-MXFP4/self_attn.q_proj+self_attn.k_proj+self_attn.v_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 32, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 6144, + "n": 9216, + "out": "bf16" + }, + "sources": [ + "amd/MiniMax-M3-MXFP4/self_attn.q_proj+self_attn.k_proj+self_attn.v_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 48, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 6144, + "n": 9216, + "out": "bf16" + }, + "sources": [ + "amd/MiniMax-M3-MXFP4/self_attn.q_proj+self_attn.k_proj+self_attn.v_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 64, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 6144, + "n": 9216, + "out": "bf16" + }, + "sources": [ + "amd/MiniMax-M3-MXFP4/self_attn.q_proj+self_attn.k_proj+self_attn.v_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 96, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 6144, + "n": 9216, + "out": "bf16" + }, + "sources": [ + "amd/MiniMax-M3-MXFP4/self_attn.q_proj+self_attn.k_proj+self_attn.v_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 128, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 6144, + "n": 9216, + "out": "bf16" + }, + "sources": [ + "amd/MiniMax-M3-MXFP4/self_attn.q_proj+self_attn.k_proj+self_attn.v_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 192, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 6144, + "n": 9216, + "out": "bf16" + }, + "sources": [ + "amd/MiniMax-M3-MXFP4/self_attn.q_proj+self_attn.k_proj+self_attn.v_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 256, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 6144, + "n": 9216, + "out": "bf16" + }, + "sources": [ + "amd/MiniMax-M3-MXFP4/self_attn.q_proj+self_attn.k_proj+self_attn.v_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 384, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 6144, + "n": 9216, + "out": "bf16" + }, + "sources": [ + "amd/MiniMax-M3-MXFP4/self_attn.q_proj+self_attn.k_proj+self_attn.v_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 512, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 6144, + "n": 9216, + "out": "bf16" + }, + "sources": [ + "amd/MiniMax-M3-MXFP4/self_attn.q_proj+self_attn.k_proj+self_attn.v_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 768, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 6144, + "n": 9216, + "out": "bf16" + }, + "sources": [ + "amd/MiniMax-M3-MXFP4/self_attn.q_proj+self_attn.k_proj+self_attn.v_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 1024, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 6144, + "n": 9216, + "out": "bf16" + }, + "sources": [ + "amd/MiniMax-M3-MXFP4/self_attn.q_proj+self_attn.k_proj+self_attn.v_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 1536, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 6144, + "n": 9216, + "out": "bf16" + }, + "sources": [ + "amd/MiniMax-M3-MXFP4/self_attn.q_proj+self_attn.k_proj+self_attn.v_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 2048, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 6144, + "n": 9216, + "out": "bf16" + }, + "sources": [ + "amd/MiniMax-M3-MXFP4/self_attn.q_proj+self_attn.k_proj+self_attn.v_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 3072, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 6144, + "n": 9216, + "out": "bf16" + }, + "sources": [ + "amd/MiniMax-M3-MXFP4/self_attn.q_proj+self_attn.k_proj+self_attn.v_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 4096, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 6144, + "n": 9216, + "out": "bf16" + }, + "sources": [ + "amd/MiniMax-M3-MXFP4/self_attn.q_proj+self_attn.k_proj+self_attn.v_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 6144, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 6144, + "n": 9216, + "out": "bf16" + }, + "sources": [ + "amd/MiniMax-M3-MXFP4/self_attn.q_proj+self_attn.k_proj+self_attn.v_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 8192, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 6144, + "n": 9216, + "out": "bf16" + }, + "sources": [ + "amd/MiniMax-M3-MXFP4/self_attn.q_proj+self_attn.k_proj+self_attn.v_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 12288, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 6144, + "n": 9216, + "out": "bf16" + }, + "sources": [ + "amd/MiniMax-M3-MXFP4/self_attn.q_proj+self_attn.k_proj+self_attn.v_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 16384, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 6144, + "n": 9216, + "out": "bf16" + }, + "sources": [ + "amd/MiniMax-M3-MXFP4/self_attn.q_proj+self_attn.k_proj+self_attn.v_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 24576, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 6144, + "n": 9216, + "out": "bf16" + }, + "sources": [ + "amd/MiniMax-M3-MXFP4/self_attn.q_proj+self_attn.k_proj+self_attn.v_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 32768, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 6144, + "n": 9216, + "out": "bf16" + }, + "sources": [ + "amd/MiniMax-M3-MXFP4/self_attn.q_proj+self_attn.k_proj+self_attn.v_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 49152, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 6144, + "n": 9216, + "out": "bf16" + }, + "sources": [ + "amd/MiniMax-M3-MXFP4/self_attn.q_proj+self_attn.k_proj+self_attn.v_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 65536, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 6144, + "n": 9216, + "out": "bf16" + }, + "sources": [ + "amd/MiniMax-M3-MXFP4/self_attn.q_proj+self_attn.k_proj+self_attn.v_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 1, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 6144, + "n": 9856, + "out": "bf16" + }, + "sources": [ + "amd/MiniMax-M3-MXFP4/self_attn.q_proj+self_attn.k_proj+self_attn.v_proj+self_attn.index_q_proj+self_attn.index_k_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 2, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 6144, + "n": 9856, + "out": "bf16" + }, + "sources": [ + "amd/MiniMax-M3-MXFP4/self_attn.q_proj+self_attn.k_proj+self_attn.v_proj+self_attn.index_q_proj+self_attn.index_k_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 3, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 6144, + "n": 9856, + "out": "bf16" + }, + "sources": [ + "amd/MiniMax-M3-MXFP4/self_attn.q_proj+self_attn.k_proj+self_attn.v_proj+self_attn.index_q_proj+self_attn.index_k_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 4, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 6144, + "n": 9856, + "out": "bf16" + }, + "sources": [ + "amd/MiniMax-M3-MXFP4/self_attn.q_proj+self_attn.k_proj+self_attn.v_proj+self_attn.index_q_proj+self_attn.index_k_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 6, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 6144, + "n": 9856, + "out": "bf16" + }, + "sources": [ + "amd/MiniMax-M3-MXFP4/self_attn.q_proj+self_attn.k_proj+self_attn.v_proj+self_attn.index_q_proj+self_attn.index_k_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 8, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 6144, + "n": 9856, + "out": "bf16" + }, + "sources": [ + "amd/MiniMax-M3-MXFP4/self_attn.q_proj+self_attn.k_proj+self_attn.v_proj+self_attn.index_q_proj+self_attn.index_k_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 12, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 6144, + "n": 9856, + "out": "bf16" + }, + "sources": [ + "amd/MiniMax-M3-MXFP4/self_attn.q_proj+self_attn.k_proj+self_attn.v_proj+self_attn.index_q_proj+self_attn.index_k_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 16, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 6144, + "n": 9856, + "out": "bf16" + }, + "sources": [ + "amd/MiniMax-M3-MXFP4/self_attn.q_proj+self_attn.k_proj+self_attn.v_proj+self_attn.index_q_proj+self_attn.index_k_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 24, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 6144, + "n": 9856, + "out": "bf16" + }, + "sources": [ + "amd/MiniMax-M3-MXFP4/self_attn.q_proj+self_attn.k_proj+self_attn.v_proj+self_attn.index_q_proj+self_attn.index_k_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 32, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 6144, + "n": 9856, + "out": "bf16" + }, + "sources": [ + "amd/MiniMax-M3-MXFP4/self_attn.q_proj+self_attn.k_proj+self_attn.v_proj+self_attn.index_q_proj+self_attn.index_k_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 48, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 6144, + "n": 9856, + "out": "bf16" + }, + "sources": [ + "amd/MiniMax-M3-MXFP4/self_attn.q_proj+self_attn.k_proj+self_attn.v_proj+self_attn.index_q_proj+self_attn.index_k_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 64, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 6144, + "n": 9856, + "out": "bf16" + }, + "sources": [ + "amd/MiniMax-M3-MXFP4/self_attn.q_proj+self_attn.k_proj+self_attn.v_proj+self_attn.index_q_proj+self_attn.index_k_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 96, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 6144, + "n": 9856, + "out": "bf16" + }, + "sources": [ + "amd/MiniMax-M3-MXFP4/self_attn.q_proj+self_attn.k_proj+self_attn.v_proj+self_attn.index_q_proj+self_attn.index_k_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 128, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 6144, + "n": 9856, + "out": "bf16" + }, + "sources": [ + "amd/MiniMax-M3-MXFP4/self_attn.q_proj+self_attn.k_proj+self_attn.v_proj+self_attn.index_q_proj+self_attn.index_k_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 192, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 6144, + "n": 9856, + "out": "bf16" + }, + "sources": [ + "amd/MiniMax-M3-MXFP4/self_attn.q_proj+self_attn.k_proj+self_attn.v_proj+self_attn.index_q_proj+self_attn.index_k_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 256, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 6144, + "n": 9856, + "out": "bf16" + }, + "sources": [ + "amd/MiniMax-M3-MXFP4/self_attn.q_proj+self_attn.k_proj+self_attn.v_proj+self_attn.index_q_proj+self_attn.index_k_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 384, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 6144, + "n": 9856, + "out": "bf16" + }, + "sources": [ + "amd/MiniMax-M3-MXFP4/self_attn.q_proj+self_attn.k_proj+self_attn.v_proj+self_attn.index_q_proj+self_attn.index_k_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 512, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 6144, + "n": 9856, + "out": "bf16" + }, + "sources": [ + "amd/MiniMax-M3-MXFP4/self_attn.q_proj+self_attn.k_proj+self_attn.v_proj+self_attn.index_q_proj+self_attn.index_k_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 768, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 6144, + "n": 9856, + "out": "bf16" + }, + "sources": [ + "amd/MiniMax-M3-MXFP4/self_attn.q_proj+self_attn.k_proj+self_attn.v_proj+self_attn.index_q_proj+self_attn.index_k_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 1024, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 6144, + "n": 9856, + "out": "bf16" + }, + "sources": [ + "amd/MiniMax-M3-MXFP4/self_attn.q_proj+self_attn.k_proj+self_attn.v_proj+self_attn.index_q_proj+self_attn.index_k_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 1536, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 6144, + "n": 9856, + "out": "bf16" + }, + "sources": [ + "amd/MiniMax-M3-MXFP4/self_attn.q_proj+self_attn.k_proj+self_attn.v_proj+self_attn.index_q_proj+self_attn.index_k_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 2048, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 6144, + "n": 9856, + "out": "bf16" + }, + "sources": [ + "amd/MiniMax-M3-MXFP4/self_attn.q_proj+self_attn.k_proj+self_attn.v_proj+self_attn.index_q_proj+self_attn.index_k_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 3072, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 6144, + "n": 9856, + "out": "bf16" + }, + "sources": [ + "amd/MiniMax-M3-MXFP4/self_attn.q_proj+self_attn.k_proj+self_attn.v_proj+self_attn.index_q_proj+self_attn.index_k_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 4096, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 6144, + "n": 9856, + "out": "bf16" + }, + "sources": [ + "amd/MiniMax-M3-MXFP4/self_attn.q_proj+self_attn.k_proj+self_attn.v_proj+self_attn.index_q_proj+self_attn.index_k_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 6144, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 6144, + "n": 9856, + "out": "bf16" + }, + "sources": [ + "amd/MiniMax-M3-MXFP4/self_attn.q_proj+self_attn.k_proj+self_attn.v_proj+self_attn.index_q_proj+self_attn.index_k_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 8192, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 6144, + "n": 9856, + "out": "bf16" + }, + "sources": [ + "amd/MiniMax-M3-MXFP4/self_attn.q_proj+self_attn.k_proj+self_attn.v_proj+self_attn.index_q_proj+self_attn.index_k_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 12288, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 6144, + "n": 9856, + "out": "bf16" + }, + "sources": [ + "amd/MiniMax-M3-MXFP4/self_attn.q_proj+self_attn.k_proj+self_attn.v_proj+self_attn.index_q_proj+self_attn.index_k_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 16384, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 6144, + "n": 9856, + "out": "bf16" + }, + "sources": [ + "amd/MiniMax-M3-MXFP4/self_attn.q_proj+self_attn.k_proj+self_attn.v_proj+self_attn.index_q_proj+self_attn.index_k_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 24576, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 6144, + "n": 9856, + "out": "bf16" + }, + "sources": [ + "amd/MiniMax-M3-MXFP4/self_attn.q_proj+self_attn.k_proj+self_attn.v_proj+self_attn.index_q_proj+self_attn.index_k_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 32768, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 6144, + "n": 9856, + "out": "bf16" + }, + "sources": [ + "amd/MiniMax-M3-MXFP4/self_attn.q_proj+self_attn.k_proj+self_attn.v_proj+self_attn.index_q_proj+self_attn.index_k_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 49152, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 6144, + "n": 9856, + "out": "bf16" + }, + "sources": [ + "amd/MiniMax-M3-MXFP4/self_attn.q_proj+self_attn.k_proj+self_attn.v_proj+self_attn.index_q_proj+self_attn.index_k_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 65536, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 6144, + "n": 9856, + "out": "bf16" + }, + "sources": [ + "amd/MiniMax-M3-MXFP4/self_attn.q_proj+self_attn.k_proj+self_attn.v_proj+self_attn.index_q_proj+self_attn.index_k_proj" + ] + } +] diff --git a/operatorx/testlists/gemm_parallel.json b/operatorx/testlists/gemm_parallel.json new file mode 100644 index 0000000000..21bf5797e0 --- /dev/null +++ b/operatorx/testlists/gemm_parallel.json @@ -0,0 +1,1692 @@ +[ + { + "type": "gemm", + "args": { + "m": 8, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + }, + "k": 3072, + "n": 6144, + "out": "bf16", + "parallel": { + "tp": 8 + } + }, + "sources": [ + "MiniMaxAI/MiniMax-M3-MXFP8/block_sparse_moe.shared_experts.down_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 8, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + }, + "k": 3072, + "n": 6144, + "out": "bf16", + "parallel": { + "tp": 2 + } + }, + "sources": [ + "nvidia/MiniMax-M3-NVFP4/block_sparse_moe.shared_experts.down_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 8, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + }, + "k": 3072, + "n": 6144, + "out": "bf16", + "parallel": { + "tp": 4 + } + }, + "sources": [ + "nvidia/MiniMax-M3-NVFP4/block_sparse_moe.shared_experts.down_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 256, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + }, + "k": 3072, + "n": 6144, + "out": "bf16", + "parallel": { + "tp": 8 + } + }, + "sources": [ + "MiniMaxAI/MiniMax-M3-MXFP8/block_sparse_moe.shared_experts.down_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 256, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + }, + "k": 3072, + "n": 6144, + "out": "bf16", + "parallel": { + "tp": 2 + } + }, + "sources": [ + "nvidia/MiniMax-M3-NVFP4/block_sparse_moe.shared_experts.down_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 256, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + }, + "k": 3072, + "n": 6144, + "out": "bf16", + "parallel": { + "tp": 4 + } + }, + "sources": [ + "nvidia/MiniMax-M3-NVFP4/block_sparse_moe.shared_experts.down_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 8, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + }, + "k": 12288, + "n": 6144, + "out": "bf16", + "parallel": { + "tp": 8 + } + }, + "sources": [ + "MiniMaxAI/MiniMax-M3-MXFP8/mlp.down_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 8, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + }, + "k": 12288, + "n": 6144, + "out": "bf16", + "parallel": { + "tp": 2 + } + }, + "sources": [ + "nvidia/MiniMax-M3-NVFP4/mlp.down_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 8, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + }, + "k": 12288, + "n": 6144, + "out": "bf16", + "parallel": { + "tp": 4 + } + }, + "sources": [ + "nvidia/MiniMax-M3-NVFP4/mlp.down_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 256, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + }, + "k": 12288, + "n": 6144, + "out": "bf16", + "parallel": { + "tp": 8 + } + }, + "sources": [ + "MiniMaxAI/MiniMax-M3-MXFP8/mlp.down_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 256, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + }, + "k": 12288, + "n": 6144, + "out": "bf16", + "parallel": { + "tp": 2 + } + }, + "sources": [ + "nvidia/MiniMax-M3-NVFP4/mlp.down_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 256, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + }, + "k": 12288, + "n": 6144, + "out": "bf16", + "parallel": { + "tp": 4 + } + }, + "sources": [ + "nvidia/MiniMax-M3-NVFP4/mlp.down_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 8, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + }, + "k": 8192, + "n": 6144, + "out": "bf16", + "parallel": { + "tp": 8 + } + }, + "sources": [ + "MiniMaxAI/MiniMax-M3-MXFP8/self_attn.o_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 8, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + }, + "k": 8192, + "n": 6144, + "out": "bf16", + "parallel": { + "tp": 2 + } + }, + "sources": [ + "nvidia/MiniMax-M3-NVFP4/self_attn.o_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 8, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + }, + "k": 8192, + "n": 6144, + "out": "bf16", + "parallel": { + "tp": 4 + } + }, + "sources": [ + "nvidia/MiniMax-M3-NVFP4/self_attn.o_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 256, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + }, + "k": 8192, + "n": 6144, + "out": "bf16", + "parallel": { + "tp": 8 + } + }, + "sources": [ + "MiniMaxAI/MiniMax-M3-MXFP8/self_attn.o_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 256, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + }, + "k": 8192, + "n": 6144, + "out": "bf16", + "parallel": { + "tp": 2 + } + }, + "sources": [ + "nvidia/MiniMax-M3-NVFP4/self_attn.o_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 256, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + }, + "k": 8192, + "n": 6144, + "out": "bf16", + "parallel": { + "tp": 4 + } + }, + "sources": [ + "nvidia/MiniMax-M3-NVFP4/self_attn.o_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 8, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + }, + "k": 3072, + "n": 6144, + "out": "bf16", + "parallel": { + "tp": 8 + } + }, + "sources": [ + "amd/MiniMax-M3-MXFP4/block_sparse_moe.shared_experts.down_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 256, + "a": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + }, + "k": 3072, + "n": 6144, + "out": "bf16", + "parallel": { + "tp": 8 + } + }, + "sources": [ + "amd/MiniMax-M3-MXFP4/block_sparse_moe.shared_experts.down_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 8, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 12288, + "n": 6144, + "out": "bf16", + "parallel": { + "tp": 8 + } + }, + "sources": [ + "amd/MiniMax-M3-MXFP4/mlp.down_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 256, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 12288, + "n": 6144, + "out": "bf16", + "parallel": { + "tp": 8 + } + }, + "sources": [ + "amd/MiniMax-M3-MXFP4/mlp.down_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 8, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 8192, + "n": 6144, + "out": "bf16", + "parallel": { + "tp": 8 + } + }, + "sources": [ + "amd/MiniMax-M3-MXFP4/self_attn.o_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 256, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 8192, + "n": 6144, + "out": "bf16", + "parallel": { + "tp": 8 + } + }, + "sources": [ + "amd/MiniMax-M3-MXFP4/self_attn.o_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 8, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 128 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + }, + "k": 16384, + "n": 7168, + "out": "bf16", + "parallel": { + "tp": 8 + } + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 8, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 128 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + }, + "k": 16384, + "n": 7168, + "out": "bf16", + "parallel": { + "tp": 4 + } + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 256, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 128 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + }, + "k": 16384, + "n": 7168, + "out": "bf16", + "parallel": { + "tp": 8 + } + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 256, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 128 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + }, + "k": 16384, + "n": 7168, + "out": "bf16", + "parallel": { + "tp": 4 + } + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 8, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 128 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + }, + "k": 3072, + "n": 7168, + "out": "bf16", + "parallel": { + "tp": 8 + } + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w2" + ] + }, + { + "type": "gemm", + "args": { + "m": 8, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 128 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + }, + "k": 3072, + "n": 7168, + "out": "bf16", + "parallel": { + "tp": 4 + } + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w2" + ] + }, + { + "type": "gemm", + "args": { + "m": 256, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 128 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + }, + "k": 3072, + "n": 7168, + "out": "bf16", + "parallel": { + "tp": 8 + } + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w2" + ] + }, + { + "type": "gemm", + "args": { + "m": 256, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 128 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + }, + "k": 3072, + "n": 7168, + "out": "bf16", + "parallel": { + "tp": 4 + } + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w2" + ] + }, + { + "type": "gemm", + "args": { + "m": 8, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + }, + "k": 8192, + "n": 5120, + "out": "bf16", + "parallel": { + "tp": 2 + } + }, + "sources": [ + "deepseek-ai/DeepSeek-V4.1-Flash/attn.wo_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 8, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + }, + "k": 8192, + "n": 5120, + "out": "bf16", + "parallel": { + "tp": 4 + } + }, + "sources": [ + "deepseek-ai/DeepSeek-V4.1-Flash/attn.wo_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 8, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + }, + "k": 8192, + "n": 5120, + "out": "bf16", + "parallel": { + "tp": 8 + } + }, + "sources": [ + "deepseek-ai/DeepSeek-V4.1-Flash/attn.wo_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 256, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + }, + "k": 8192, + "n": 5120, + "out": "bf16", + "parallel": { + "tp": 2 + } + }, + "sources": [ + "deepseek-ai/DeepSeek-V4.1-Flash/attn.wo_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 256, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + }, + "k": 8192, + "n": 5120, + "out": "bf16", + "parallel": { + "tp": 4 + } + }, + "sources": [ + "deepseek-ai/DeepSeek-V4.1-Flash/attn.wo_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 256, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + }, + "k": 8192, + "n": 5120, + "out": "bf16", + "parallel": { + "tp": 8 + } + }, + "sources": [ + "deepseek-ai/DeepSeek-V4.1-Flash/attn.wo_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 8, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + }, + "k": 2304, + "n": 5120, + "out": "bf16", + "parallel": { + "tp": 2 + } + }, + "sources": [ + "deepseek-ai/DeepSeek-V4.1-Flash/ffn.shared_experts.w2" + ] + }, + { + "type": "gemm", + "args": { + "m": 8, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + }, + "k": 2304, + "n": 5120, + "out": "bf16", + "parallel": { + "tp": 4 + } + }, + "sources": [ + "deepseek-ai/DeepSeek-V4.1-Flash/ffn.shared_experts.w2" + ] + }, + { + "type": "gemm", + "args": { + "m": 8, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + }, + "k": 2304, + "n": 5120, + "out": "bf16", + "parallel": { + "tp": 8 + } + }, + "sources": [ + "deepseek-ai/DeepSeek-V4.1-Flash/ffn.shared_experts.w2" + ] + }, + { + "type": "gemm", + "args": { + "m": 256, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + }, + "k": 2304, + "n": 5120, + "out": "bf16", + "parallel": { + "tp": 2 + } + }, + "sources": [ + "deepseek-ai/DeepSeek-V4.1-Flash/ffn.shared_experts.w2" + ] + }, + { + "type": "gemm", + "args": { + "m": 256, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + }, + "k": 2304, + "n": 5120, + "out": "bf16", + "parallel": { + "tp": 4 + } + }, + "sources": [ + "deepseek-ai/DeepSeek-V4.1-Flash/ffn.shared_experts.w2" + ] + }, + { + "type": "gemm", + "args": { + "m": 256, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + }, + "k": 2304, + "n": 5120, + "out": "bf16", + "parallel": { + "tp": 8 + } + }, + "sources": [ + "deepseek-ai/DeepSeek-V4.1-Flash/ffn.shared_experts.w2" + ] + }, + { + "type": "gemm", + "args": { + "m": 8, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 6144, + "n": 7168, + "out": "bf16", + "parallel": { + "tp": 8 + } + }, + "sources": [ + "moonshotai/Kimi-K3/block_sparse_moe.shared_experts.down_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 256, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 6144, + "n": 7168, + "out": "bf16", + "parallel": { + "tp": 8 + } + }, + "sources": [ + "moonshotai/Kimi-K3/block_sparse_moe.shared_experts.down_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 8, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 33792, + "n": 7168, + "out": "bf16", + "parallel": { + "tp": 8 + } + }, + "sources": [ + "moonshotai/Kimi-K3/mlp.down_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 256, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 33792, + "n": 7168, + "out": "bf16", + "parallel": { + "tp": 8 + } + }, + "sources": [ + "moonshotai/Kimi-K3/mlp.down_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 8, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 12288, + "n": 7168, + "out": "bf16", + "parallel": { + "tp": 8 + } + }, + "sources": [ + "moonshotai/Kimi-K3/self_attn.o_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 256, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "k": 12288, + "n": 7168, + "out": "bf16", + "parallel": { + "tp": 8 + } + }, + "sources": [ + "moonshotai/Kimi-K3/self_attn.o_proj" + ] + } +] \ No newline at end of file diff --git a/operatorx/testlists/gemm_perf.json b/operatorx/testlists/gemm_perf.json index 4abab80949..cf7a9731d9 100644 --- a/operatorx/testlists/gemm_perf.json +++ b/operatorx/testlists/gemm_perf.json @@ -5,11 +5,20 @@ "m": 1, "n": 7168, "k": 2048, - "dtype_a": "bf16", - "dtype_b": "bf16", - "dtype_out": "bf16" + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" }, - "name": "gemmperf" + "sources": [ + "amd/Kimi-K2.5-MXFP4/o_proj", + "moonshotai/Kimi-K2.5/o_proj", + "nvidia/Kimi-K2.5-NVFP4/o_proj", + "nvidia/Kimi-K2.6-NVFP4/o_proj" + ] }, { "type": "gemm", @@ -17,11 +26,20 @@ "m": 64, "n": 7168, "k": 2048, - "dtype_a": "bf16", - "dtype_b": "bf16", - "dtype_out": "bf16" + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" }, - "name": "gemmperf" + "sources": [ + "amd/Kimi-K2.5-MXFP4/o_proj", + "moonshotai/Kimi-K2.5/o_proj", + "nvidia/Kimi-K2.5-NVFP4/o_proj", + "nvidia/Kimi-K2.6-NVFP4/o_proj" + ] }, { "type": "gemm", @@ -29,11 +47,22 @@ "m": 256, "n": 4096, "k": 4096, - "dtype_a": "bf16", - "dtype_b": "bf16", - "dtype_out": "bf16" + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" }, - "name": "gemmperf" + "sources": [ + "Qwen/Qwen3.5-397B-A17B/o_proj", + "Qwen/Qwen3.5-397B-A17B/q_proj", + "amd/Qwen3.5-397B-A17B-MXFP4/o_proj", + "amd/Qwen3.5-397B-A17B-MXFP4/q_proj", + "nvidia/Qwen3.5-397B-A17B-NVFP4/o_proj", + "nvidia/Qwen3.5-397B-A17B-NVFP4/q_proj" + ] }, { "type": "gemm", @@ -41,11 +70,22 @@ "m": 1024, "n": 1536, "k": 7168, - "dtype_a": "bf16", - "dtype_b": "bf16", - "dtype_out": "bf16" + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" }, - "name": "gemmperf" + "sources": [ + "amd/Kimi-K2.5-MXFP4/q_a_proj", + "moonshotai/Kimi-K2.5/q_a_proj", + "nvidia/DeepSeek-R1-0528-FP4-V2/q_a_proj", + "nvidia/DeepSeek-R1-0528-NVFP4-v2/q_a_proj", + "nvidia/Kimi-K2.5-NVFP4/q_a_proj", + "nvidia/Kimi-K2.6-NVFP4/q_a_proj" + ] }, { "type": "gemm", @@ -53,11 +93,22 @@ "m": 2048, "n": 4096, "k": 4096, - "dtype_a": "bf16", - "dtype_b": "bf16", - "dtype_out": "bf16" + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" }, - "name": "gemmperf" + "sources": [ + "Qwen/Qwen3.5-397B-A17B/o_proj", + "Qwen/Qwen3.5-397B-A17B/q_proj", + "amd/Qwen3.5-397B-A17B-MXFP4/o_proj", + "amd/Qwen3.5-397B-A17B-MXFP4/q_proj", + "nvidia/Qwen3.5-397B-A17B-NVFP4/o_proj", + "nvidia/Qwen3.5-397B-A17B-NVFP4/q_proj" + ] }, { "type": "gemm", @@ -65,11 +116,22 @@ "m": 4096, "n": 4096, "k": 4096, - "dtype_a": "bf16", - "dtype_b": "bf16", - "dtype_out": "bf16" + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" }, - "name": "gemmperf" + "sources": [ + "Qwen/Qwen3.5-397B-A17B/o_proj", + "Qwen/Qwen3.5-397B-A17B/q_proj", + "amd/Qwen3.5-397B-A17B-MXFP4/o_proj", + "amd/Qwen3.5-397B-A17B-MXFP4/q_proj", + "nvidia/Qwen3.5-397B-A17B-NVFP4/o_proj", + "nvidia/Qwen3.5-397B-A17B-NVFP4/q_proj" + ] }, { "type": "gemm", @@ -77,11 +139,15 @@ "m": 8192, "n": 8192, "k": 8192, - "dtype_a": "bf16", - "dtype_b": "bf16", - "dtype_out": "bf16" + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" }, - "name": "gemmperf" + "sources": [] }, { "type": "gemm", @@ -89,11 +155,15 @@ "m": 16384, "n": 8192, "k": 8192, - "dtype_a": "bf16", - "dtype_b": "bf16", - "dtype_out": "bf16" + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" }, - "name": "gemmperf" + "sources": [] }, { "type": "gemm", @@ -101,11 +171,20 @@ "m": 8192, "n": 7168, "k": 2048, - "dtype_a": "bf16", - "dtype_b": "bf16", - "dtype_out": "bf16" + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" }, - "name": "gemmperf" + "sources": [ + "amd/Kimi-K2.5-MXFP4/o_proj", + "moonshotai/Kimi-K2.5/o_proj", + "nvidia/Kimi-K2.5-NVFP4/o_proj", + "nvidia/Kimi-K2.6-NVFP4/o_proj" + ] }, { "type": "gemm", @@ -113,11 +192,19 @@ "m": 32768, "n": 2048, "k": 7168, - "dtype_a": "bf16", - "dtype_b": "bf16", - "dtype_out": "bf16" + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" }, - "name": "gemmperf" + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.compressor.wkv+wgate", + "deepseek-ai/DeepSeek-V4-Pro/attn.compressor.wkv+wgate", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.compressor.wkv+wgate" + ] }, { "type": "gemm", @@ -125,10 +212,21 @@ "m": 512, "n": 4096, "k": 4096, - "dtype_a": "bf16", - "dtype_b": "bf16", - "dtype_out": "bf16" + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" }, - "name": "gemmperf" + "sources": [ + "Qwen/Qwen3.5-397B-A17B/o_proj", + "Qwen/Qwen3.5-397B-A17B/q_proj", + "amd/Qwen3.5-397B-A17B-MXFP4/o_proj", + "amd/Qwen3.5-397B-A17B-MXFP4/q_proj", + "nvidia/Qwen3.5-397B-A17B-NVFP4/o_proj", + "nvidia/Qwen3.5-397B-A17B-NVFP4/q_proj" + ] } -] \ No newline at end of file +] diff --git a/operatorx/testlists/gemm_serving_8k1k_min.json b/operatorx/testlists/gemm_serving_8k1k_min.json new file mode 100644 index 0000000000..ef50eaffb4 --- /dev/null +++ b/operatorx/testlists/gemm_serving_8k1k_min.json @@ -0,0 +1,44782 @@ +[ + { + "type": "gemm", + "args": { + "m": 1, + "n": 24, + "k": 28672, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/hc_attn", + "deepseek-ai/DeepSeek-V4-Pro-0813/hc_ffn", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.hc_attn", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.hc_ffn", + "deepseek-ai/DeepSeek-V4-Pro/hc_attn", + "deepseek-ai/DeepSeek-V4-Pro/hc_ffn", + "deepseek-ai/DeepSeek-V4-Pro/mtp.hc_attn", + "deepseek-ai/DeepSeek-V4-Pro/mtp.hc_ffn", + "sgl-project/DeepSeek-V4-Pro-FP8/hc_attn", + "sgl-project/DeepSeek-V4-Pro-FP8/hc_ffn", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.hc_attn", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.hc_ffn" + ] + }, + { + "type": "gemm", + "args": { + "m": 1, + "n": 24, + "k": 40960, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 1, + "n": 24, + "k": 49152, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 1, + "n": 64, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.weights_proj", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.weights_proj", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.weights_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 1, + "n": 64, + "k": 10240, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 1, + "n": 64, + "k": 12288, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 1, + "n": 120, + "k": 12288, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 1, + "n": 192, + "k": 10240, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 1, + "n": 256, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.compressor.wkv", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.compressor.wkv", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.compressor.wgate", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.compressor.wkv" + ] + }, + { + "type": "gemm", + "args": { + "m": 1, + "n": 256, + "k": 10240, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 1, + "n": 256, + "k": 12288, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 1, + "n": 320, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 1, + "n": 384, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.gate", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.gate", + "deepseek-ai/DeepSeek-V4-Pro/ffn.gate", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.gate", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.gate", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.gate" + ] + }, + { + "type": "gemm", + "args": { + "m": 1, + "n": 384, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w3" + ] + }, + { + "type": "gemm", + "args": { + "m": 1, + "n": 512, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wkv", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wkv", + "deepseek-ai/DeepSeek-V4-Pro/attn.wkv", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wkv", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wkv", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wkv" + ] + }, + { + "type": "gemm", + "args": { + "m": 1, + "n": 512, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.compressor.wkv+wgate", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.compressor.wkv+wgate", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.compressor.wkv+wgate" + ] + }, + { + "type": "gemm", + "args": { + "m": 1, + "n": 512, + "k": 10240, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 1, + "n": 512, + "k": 10240, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 1, + "n": 512, + "k": 12288, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 1, + "n": 512, + "k": 12288, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 1, + "n": 768, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w3" + ] + }, + { + "type": "gemm", + "args": { + "m": 1, + "n": 896, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 1, + "n": 1024, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.compressor.wkv", + "deepseek-ai/DeepSeek-V4-Pro/attn.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro/attn.compressor.wkv", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.compressor.wgate", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.compressor.wkv" + ] + }, + { + "type": "gemm", + "args": { + "m": 1, + "n": 1024, + "k": 10240, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 1, + "n": 1024, + "k": 12288, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 1, + "n": 1536, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w3" + ] + }, + { + "type": "gemm", + "args": { + "m": 1, + "n": 1536, + "k": 10240, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 1, + "n": 1536, + "k": 12288, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 1, + "n": 1792, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 1, + "n": 4096, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 1, + "n": 5120, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 1, + "n": 6144, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3" + ] + }, + { + "type": "gemm", + "args": { + "m": 1, + "n": 7168, + "k": 192, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w2" + ] + }, + { + "type": "gemm", + "args": { + "m": 1, + "n": 7168, + "k": 384, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w2" + ] + }, + { + "type": "gemm", + "args": { + "m": 1, + "n": 7168, + "k": 448, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 1, + "n": 7168, + "k": 768, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w2" + ] + }, + { + "type": "gemm", + "args": { + "m": 1, + "n": 7168, + "k": 896, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 1, + "n": 7168, + "k": 1024, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 1, + "n": 7168, + "k": 2048, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 1, + "n": 7168, + "k": 3072, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w2" + ] + }, + { + "type": "gemm", + "args": { + "m": 1, + "n": 7168, + "k": 4096, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 1, + "n": 7168, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.e_proj", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.h_proj", + "deepseek-ai/DeepSeek-V4-Pro/mtp.e_proj", + "deepseek-ai/DeepSeek-V4-Pro/mtp.h_proj", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.e_proj", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.h_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 1, + "n": 7168, + "k": 16384, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 1, + "n": 8080, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/head", + "deepseek-ai/DeepSeek-V4-Pro/head", + "sgl-project/DeepSeek-V4-Pro-FP8/head" + ] + }, + { + "type": "gemm", + "args": { + "m": 1, + "n": 8080, + "k": 10240, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 1, + "n": 8080, + "k": 12288, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 1, + "n": 8192, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 1, + "n": 8192, + "k": 10240, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 1, + "n": 10240, + "k": 1024, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 1, + "n": 10240, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 1, + "n": 10240, + "k": 2048, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 1, + "n": 10240, + "k": 4096, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 1, + "n": 10240, + "k": 16384, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 1, + "n": 12288, + "k": 1024, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 1, + "n": 12288, + "k": 2048, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 1, + "n": 12288, + "k": 8192, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 1, + "n": 12288, + "k": 16384, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 1, + "n": 14336, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 1, + "n": 16160, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/head", + "deepseek-ai/DeepSeek-V4-Pro/head", + "sgl-project/DeepSeek-V4-Pro-FP8/head" + ] + }, + { + "type": "gemm", + "args": { + "m": 1, + "n": 16384, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 1, + "n": 16384, + "k": 12288, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 1, + "n": 65536, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 1, + "n": 81920, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 1, + "n": 129280, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/head", + "deepseek-ai/DeepSeek-V4-Pro/head", + "sgl-project/DeepSeek-V4-Pro-FP8/head" + ] + }, + { + "type": "gemm", + "args": { + "m": 1, + "n": 129280, + "k": 10240, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 1, + "n": 129280, + "k": 12288, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 2, + "n": 24, + "k": 28672, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/hc_attn", + "deepseek-ai/DeepSeek-V4-Pro-0813/hc_ffn", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.hc_attn", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.hc_ffn", + "deepseek-ai/DeepSeek-V4-Pro/hc_attn", + "deepseek-ai/DeepSeek-V4-Pro/hc_ffn", + "deepseek-ai/DeepSeek-V4-Pro/mtp.hc_attn", + "deepseek-ai/DeepSeek-V4-Pro/mtp.hc_ffn", + "sgl-project/DeepSeek-V4-Pro-FP8/hc_attn", + "sgl-project/DeepSeek-V4-Pro-FP8/hc_ffn", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.hc_attn", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.hc_ffn" + ] + }, + { + "type": "gemm", + "args": { + "m": 2, + "n": 24, + "k": 40960, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 2, + "n": 24, + "k": 49152, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 2, + "n": 64, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.weights_proj", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.weights_proj", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.weights_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 2, + "n": 64, + "k": 10240, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 2, + "n": 64, + "k": 12288, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 2, + "n": 120, + "k": 12288, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 2, + "n": 192, + "k": 10240, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 2, + "n": 256, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.compressor.wkv", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.compressor.wkv", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.compressor.wgate", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.compressor.wkv" + ] + }, + { + "type": "gemm", + "args": { + "m": 2, + "n": 256, + "k": 10240, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 2, + "n": 256, + "k": 12288, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 2, + "n": 320, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 2, + "n": 384, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.gate", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.gate", + "deepseek-ai/DeepSeek-V4-Pro/ffn.gate", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.gate", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.gate", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.gate" + ] + }, + { + "type": "gemm", + "args": { + "m": 2, + "n": 384, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w3" + ] + }, + { + "type": "gemm", + "args": { + "m": 2, + "n": 512, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wkv", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wkv", + "deepseek-ai/DeepSeek-V4-Pro/attn.wkv", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wkv", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wkv", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wkv" + ] + }, + { + "type": "gemm", + "args": { + "m": 2, + "n": 512, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.compressor.wkv+wgate", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.compressor.wkv+wgate", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.compressor.wkv+wgate" + ] + }, + { + "type": "gemm", + "args": { + "m": 2, + "n": 512, + "k": 10240, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 2, + "n": 512, + "k": 10240, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 2, + "n": 512, + "k": 12288, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 2, + "n": 512, + "k": 12288, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 2, + "n": 768, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w3" + ] + }, + { + "type": "gemm", + "args": { + "m": 2, + "n": 896, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 2, + "n": 1024, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.compressor.wkv", + "deepseek-ai/DeepSeek-V4-Pro/attn.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro/attn.compressor.wkv", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.compressor.wgate", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.compressor.wkv" + ] + }, + { + "type": "gemm", + "args": { + "m": 2, + "n": 1024, + "k": 10240, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 2, + "n": 1024, + "k": 12288, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 2, + "n": 1536, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w3" + ] + }, + { + "type": "gemm", + "args": { + "m": 2, + "n": 1536, + "k": 10240, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 2, + "n": 1536, + "k": 12288, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 2, + "n": 1792, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 2, + "n": 4096, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 2, + "n": 5120, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 2, + "n": 6144, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3" + ] + }, + { + "type": "gemm", + "args": { + "m": 2, + "n": 7168, + "k": 192, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w2" + ] + }, + { + "type": "gemm", + "args": { + "m": 2, + "n": 7168, + "k": 384, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w2" + ] + }, + { + "type": "gemm", + "args": { + "m": 2, + "n": 7168, + "k": 448, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 2, + "n": 7168, + "k": 896, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 2, + "n": 7168, + "k": 1024, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 2, + "n": 7168, + "k": 2048, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 2, + "n": 7168, + "k": 3072, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w2" + ] + }, + { + "type": "gemm", + "args": { + "m": 2, + "n": 7168, + "k": 4096, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 2, + "n": 7168, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.e_proj", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.h_proj", + "deepseek-ai/DeepSeek-V4-Pro/mtp.e_proj", + "deepseek-ai/DeepSeek-V4-Pro/mtp.h_proj", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.e_proj", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.h_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 2, + "n": 7168, + "k": 16384, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 2, + "n": 8080, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/head", + "deepseek-ai/DeepSeek-V4-Pro/head", + "sgl-project/DeepSeek-V4-Pro-FP8/head" + ] + }, + { + "type": "gemm", + "args": { + "m": 2, + "n": 8192, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 2, + "n": 8192, + "k": 10240, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 2, + "n": 10240, + "k": 1024, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 2, + "n": 10240, + "k": 2048, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 2, + "n": 10240, + "k": 4096, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 2, + "n": 10240, + "k": 16384, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 2, + "n": 12288, + "k": 1024, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 2, + "n": 12288, + "k": 8192, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 2, + "n": 12288, + "k": 16384, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 2, + "n": 14336, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 2, + "n": 16160, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/head", + "deepseek-ai/DeepSeek-V4-Pro/head", + "sgl-project/DeepSeek-V4-Pro-FP8/head" + ] + }, + { + "type": "gemm", + "args": { + "m": 2, + "n": 16384, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 2, + "n": 16384, + "k": 12288, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 2, + "n": 65536, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 2, + "n": 81920, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 2, + "n": 129280, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/head", + "deepseek-ai/DeepSeek-V4-Pro/head", + "sgl-project/DeepSeek-V4-Pro-FP8/head" + ] + }, + { + "type": "gemm", + "args": { + "m": 2, + "n": 129280, + "k": 10240, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 2, + "n": 129280, + "k": 12288, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 3, + "n": 24, + "k": 28672, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/hc_attn", + "deepseek-ai/DeepSeek-V4-Pro-0813/hc_ffn", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.hc_attn", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.hc_ffn", + "deepseek-ai/DeepSeek-V4-Pro/hc_attn", + "deepseek-ai/DeepSeek-V4-Pro/hc_ffn", + "deepseek-ai/DeepSeek-V4-Pro/mtp.hc_attn", + "deepseek-ai/DeepSeek-V4-Pro/mtp.hc_ffn", + "sgl-project/DeepSeek-V4-Pro-FP8/hc_attn", + "sgl-project/DeepSeek-V4-Pro-FP8/hc_ffn", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.hc_attn", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.hc_ffn" + ] + }, + { + "type": "gemm", + "args": { + "m": 3, + "n": 24, + "k": 40960, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 3, + "n": 24, + "k": 49152, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 3, + "n": 64, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.weights_proj", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.weights_proj", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.weights_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 3, + "n": 64, + "k": 10240, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 3, + "n": 64, + "k": 12288, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 3, + "n": 120, + "k": 12288, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 3, + "n": 192, + "k": 10240, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 3, + "n": 256, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.compressor.wkv", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.compressor.wkv", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.compressor.wgate", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.compressor.wkv" + ] + }, + { + "type": "gemm", + "args": { + "m": 3, + "n": 256, + "k": 10240, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 3, + "n": 256, + "k": 12288, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 3, + "n": 320, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 3, + "n": 384, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.gate", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.gate", + "deepseek-ai/DeepSeek-V4-Pro/ffn.gate", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.gate", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.gate", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.gate" + ] + }, + { + "type": "gemm", + "args": { + "m": 3, + "n": 384, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w3" + ] + }, + { + "type": "gemm", + "args": { + "m": 3, + "n": 512, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wkv", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wkv", + "deepseek-ai/DeepSeek-V4-Pro/attn.wkv", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wkv", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wkv", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wkv" + ] + }, + { + "type": "gemm", + "args": { + "m": 3, + "n": 512, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.compressor.wkv+wgate", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.compressor.wkv+wgate", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.compressor.wkv+wgate" + ] + }, + { + "type": "gemm", + "args": { + "m": 3, + "n": 512, + "k": 10240, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 3, + "n": 512, + "k": 10240, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 3, + "n": 512, + "k": 12288, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 3, + "n": 512, + "k": 12288, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 3, + "n": 768, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w3" + ] + }, + { + "type": "gemm", + "args": { + "m": 3, + "n": 896, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 3, + "n": 1024, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.compressor.wkv", + "deepseek-ai/DeepSeek-V4-Pro/attn.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro/attn.compressor.wkv", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.compressor.wgate", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.compressor.wkv" + ] + }, + { + "type": "gemm", + "args": { + "m": 3, + "n": 1024, + "k": 10240, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 3, + "n": 1024, + "k": 12288, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 3, + "n": 1536, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w3" + ] + }, + { + "type": "gemm", + "args": { + "m": 3, + "n": 1536, + "k": 10240, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 3, + "n": 1536, + "k": 12288, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 3, + "n": 1792, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 3, + "n": 4096, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 3, + "n": 5120, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 3, + "n": 6144, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3" + ] + }, + { + "type": "gemm", + "args": { + "m": 3, + "n": 7168, + "k": 192, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w2" + ] + }, + { + "type": "gemm", + "args": { + "m": 3, + "n": 7168, + "k": 384, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w2" + ] + }, + { + "type": "gemm", + "args": { + "m": 3, + "n": 7168, + "k": 448, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 3, + "n": 7168, + "k": 896, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 3, + "n": 7168, + "k": 1024, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 3, + "n": 7168, + "k": 2048, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 3, + "n": 7168, + "k": 3072, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w2" + ] + }, + { + "type": "gemm", + "args": { + "m": 3, + "n": 7168, + "k": 16384, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 3, + "n": 8080, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/head", + "deepseek-ai/DeepSeek-V4-Pro/head", + "sgl-project/DeepSeek-V4-Pro-FP8/head" + ] + }, + { + "type": "gemm", + "args": { + "m": 3, + "n": 8192, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 3, + "n": 8192, + "k": 10240, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 3, + "n": 10240, + "k": 1024, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 3, + "n": 10240, + "k": 4096, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 3, + "n": 10240, + "k": 16384, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 3, + "n": 12288, + "k": 1024, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 3, + "n": 12288, + "k": 8192, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 3, + "n": 12288, + "k": 16384, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 3, + "n": 16160, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/head", + "deepseek-ai/DeepSeek-V4-Pro/head", + "sgl-project/DeepSeek-V4-Pro-FP8/head" + ] + }, + { + "type": "gemm", + "args": { + "m": 3, + "n": 16384, + "k": 12288, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 3, + "n": 65536, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 3, + "n": 81920, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 3, + "n": 129280, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/head", + "deepseek-ai/DeepSeek-V4-Pro/head", + "sgl-project/DeepSeek-V4-Pro-FP8/head" + ] + }, + { + "type": "gemm", + "args": { + "m": 3, + "n": 129280, + "k": 10240, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 3, + "n": 129280, + "k": 12288, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 4, + "n": 24, + "k": 28672, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/hc_attn", + "deepseek-ai/DeepSeek-V4-Pro-0813/hc_ffn", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.hc_attn", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.hc_ffn", + "deepseek-ai/DeepSeek-V4-Pro/hc_attn", + "deepseek-ai/DeepSeek-V4-Pro/hc_ffn", + "deepseek-ai/DeepSeek-V4-Pro/mtp.hc_attn", + "deepseek-ai/DeepSeek-V4-Pro/mtp.hc_ffn", + "sgl-project/DeepSeek-V4-Pro-FP8/hc_attn", + "sgl-project/DeepSeek-V4-Pro-FP8/hc_ffn", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.hc_attn", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.hc_ffn" + ] + }, + { + "type": "gemm", + "args": { + "m": 4, + "n": 24, + "k": 40960, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 4, + "n": 24, + "k": 49152, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 4, + "n": 64, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.weights_proj", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.weights_proj", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.weights_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 4, + "n": 64, + "k": 10240, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 4, + "n": 64, + "k": 12288, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 4, + "n": 120, + "k": 12288, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 4, + "n": 192, + "k": 10240, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 4, + "n": 256, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.compressor.wkv", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.compressor.wkv", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.compressor.wgate", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.compressor.wkv" + ] + }, + { + "type": "gemm", + "args": { + "m": 4, + "n": 256, + "k": 10240, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 4, + "n": 256, + "k": 12288, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 4, + "n": 320, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 4, + "n": 384, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.gate", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.gate", + "deepseek-ai/DeepSeek-V4-Pro/ffn.gate", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.gate", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.gate", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.gate" + ] + }, + { + "type": "gemm", + "args": { + "m": 4, + "n": 384, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w3" + ] + }, + { + "type": "gemm", + "args": { + "m": 4, + "n": 512, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wkv", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wkv", + "deepseek-ai/DeepSeek-V4-Pro/attn.wkv", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wkv", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wkv", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wkv" + ] + }, + { + "type": "gemm", + "args": { + "m": 4, + "n": 512, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.compressor.wkv+wgate", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.compressor.wkv+wgate", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.compressor.wkv+wgate" + ] + }, + { + "type": "gemm", + "args": { + "m": 4, + "n": 512, + "k": 10240, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 4, + "n": 512, + "k": 10240, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 4, + "n": 512, + "k": 12288, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 4, + "n": 512, + "k": 12288, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 4, + "n": 768, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w3" + ] + }, + { + "type": "gemm", + "args": { + "m": 4, + "n": 896, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 4, + "n": 1024, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.compressor.wkv", + "deepseek-ai/DeepSeek-V4-Pro/attn.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro/attn.compressor.wkv", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.compressor.wgate", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.compressor.wkv" + ] + }, + { + "type": "gemm", + "args": { + "m": 4, + "n": 1024, + "k": 10240, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 4, + "n": 1024, + "k": 12288, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 4, + "n": 1536, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w3" + ] + }, + { + "type": "gemm", + "args": { + "m": 4, + "n": 1536, + "k": 10240, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 4, + "n": 1536, + "k": 12288, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 4, + "n": 1792, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 4, + "n": 4096, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 4, + "n": 5120, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 4, + "n": 6144, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3" + ] + }, + { + "type": "gemm", + "args": { + "m": 4, + "n": 7168, + "k": 192, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w2" + ] + }, + { + "type": "gemm", + "args": { + "m": 4, + "n": 7168, + "k": 384, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w2" + ] + }, + { + "type": "gemm", + "args": { + "m": 4, + "n": 7168, + "k": 448, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 4, + "n": 7168, + "k": 768, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w2" + ] + }, + { + "type": "gemm", + "args": { + "m": 4, + "n": 7168, + "k": 896, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 4, + "n": 7168, + "k": 1024, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 4, + "n": 7168, + "k": 2048, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 4, + "n": 7168, + "k": 3072, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w2" + ] + }, + { + "type": "gemm", + "args": { + "m": 4, + "n": 7168, + "k": 4096, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 4, + "n": 7168, + "k": 16384, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 4, + "n": 8080, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/head", + "deepseek-ai/DeepSeek-V4-Pro/head", + "sgl-project/DeepSeek-V4-Pro-FP8/head" + ] + }, + { + "type": "gemm", + "args": { + "m": 4, + "n": 8192, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 4, + "n": 8192, + "k": 10240, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 4, + "n": 10240, + "k": 1024, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 4, + "n": 10240, + "k": 2048, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 4, + "n": 10240, + "k": 4096, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 4, + "n": 10240, + "k": 16384, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 4, + "n": 12288, + "k": 1024, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 4, + "n": 12288, + "k": 8192, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 4, + "n": 12288, + "k": 16384, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 4, + "n": 16384, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 4, + "n": 16384, + "k": 12288, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 4, + "n": 65536, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 4, + "n": 81920, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 4, + "n": 129280, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/head", + "deepseek-ai/DeepSeek-V4-Pro/head", + "sgl-project/DeepSeek-V4-Pro-FP8/head" + ] + }, + { + "type": "gemm", + "args": { + "m": 4, + "n": 129280, + "k": 10240, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 4, + "n": 129280, + "k": 12288, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 6, + "n": 24, + "k": 28672, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/hc_attn", + "deepseek-ai/DeepSeek-V4-Pro-0813/hc_ffn", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.hc_attn", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.hc_ffn", + "deepseek-ai/DeepSeek-V4-Pro/hc_attn", + "deepseek-ai/DeepSeek-V4-Pro/hc_ffn", + "deepseek-ai/DeepSeek-V4-Pro/mtp.hc_attn", + "deepseek-ai/DeepSeek-V4-Pro/mtp.hc_ffn", + "sgl-project/DeepSeek-V4-Pro-FP8/hc_attn", + "sgl-project/DeepSeek-V4-Pro-FP8/hc_ffn", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.hc_attn", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.hc_ffn" + ] + }, + { + "type": "gemm", + "args": { + "m": 6, + "n": 24, + "k": 40960, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 6, + "n": 24, + "k": 49152, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 6, + "n": 64, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.weights_proj", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.weights_proj", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.weights_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 6, + "n": 64, + "k": 10240, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 6, + "n": 64, + "k": 12288, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 6, + "n": 120, + "k": 12288, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 6, + "n": 192, + "k": 10240, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 6, + "n": 256, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.compressor.wkv", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.compressor.wkv", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.compressor.wgate", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.compressor.wkv" + ] + }, + { + "type": "gemm", + "args": { + "m": 6, + "n": 256, + "k": 10240, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 6, + "n": 256, + "k": 12288, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 6, + "n": 320, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 6, + "n": 384, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.gate", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.gate", + "deepseek-ai/DeepSeek-V4-Pro/ffn.gate", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.gate", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.gate", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.gate" + ] + }, + { + "type": "gemm", + "args": { + "m": 6, + "n": 384, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w3" + ] + }, + { + "type": "gemm", + "args": { + "m": 6, + "n": 512, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wkv", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wkv", + "deepseek-ai/DeepSeek-V4-Pro/attn.wkv", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wkv", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wkv", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wkv" + ] + }, + { + "type": "gemm", + "args": { + "m": 6, + "n": 512, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.compressor.wkv+wgate", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.compressor.wkv+wgate", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.compressor.wkv+wgate" + ] + }, + { + "type": "gemm", + "args": { + "m": 6, + "n": 512, + "k": 10240, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 6, + "n": 512, + "k": 10240, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 6, + "n": 512, + "k": 12288, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 6, + "n": 512, + "k": 12288, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 6, + "n": 768, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w3" + ] + }, + { + "type": "gemm", + "args": { + "m": 6, + "n": 896, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 6, + "n": 1024, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.compressor.wkv", + "deepseek-ai/DeepSeek-V4-Pro/attn.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro/attn.compressor.wkv", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.compressor.wgate", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.compressor.wkv" + ] + }, + { + "type": "gemm", + "args": { + "m": 6, + "n": 1024, + "k": 10240, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 6, + "n": 1024, + "k": 12288, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 6, + "n": 1536, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w3" + ] + }, + { + "type": "gemm", + "args": { + "m": 6, + "n": 1536, + "k": 10240, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 6, + "n": 1536, + "k": 12288, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 6, + "n": 1792, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 6, + "n": 4096, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 6, + "n": 5120, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 6, + "n": 6144, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3" + ] + }, + { + "type": "gemm", + "args": { + "m": 6, + "n": 7168, + "k": 192, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w2" + ] + }, + { + "type": "gemm", + "args": { + "m": 6, + "n": 7168, + "k": 384, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w2" + ] + }, + { + "type": "gemm", + "args": { + "m": 6, + "n": 7168, + "k": 448, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 6, + "n": 7168, + "k": 768, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w2" + ] + }, + { + "type": "gemm", + "args": { + "m": 6, + "n": 7168, + "k": 896, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 6, + "n": 7168, + "k": 1024, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 6, + "n": 7168, + "k": 2048, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 6, + "n": 7168, + "k": 3072, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w2" + ] + }, + { + "type": "gemm", + "args": { + "m": 6, + "n": 7168, + "k": 4096, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 6, + "n": 7168, + "k": 16384, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 6, + "n": 8192, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 6, + "n": 10240, + "k": 1024, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 6, + "n": 12288, + "k": 1024, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 6, + "n": 16384, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 6, + "n": 65536, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 8, + "n": 24, + "k": 28672, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/hc_attn", + "deepseek-ai/DeepSeek-V4-Pro-0813/hc_ffn", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.hc_attn", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.hc_ffn", + "deepseek-ai/DeepSeek-V4-Pro/hc_attn", + "deepseek-ai/DeepSeek-V4-Pro/hc_ffn", + "deepseek-ai/DeepSeek-V4-Pro/mtp.hc_attn", + "deepseek-ai/DeepSeek-V4-Pro/mtp.hc_ffn", + "sgl-project/DeepSeek-V4-Pro-FP8/hc_attn", + "sgl-project/DeepSeek-V4-Pro-FP8/hc_ffn", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.hc_attn", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.hc_ffn" + ] + }, + { + "type": "gemm", + "args": { + "m": 8, + "n": 24, + "k": 40960, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 8, + "n": 24, + "k": 49152, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 8, + "n": 64, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.weights_proj", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.weights_proj", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.weights_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 8, + "n": 64, + "k": 10240, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 8, + "n": 64, + "k": 12288, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 8, + "n": 120, + "k": 12288, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 8, + "n": 192, + "k": 10240, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 8, + "n": 256, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.compressor.wkv", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.compressor.wkv", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.compressor.wgate", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.compressor.wkv" + ] + }, + { + "type": "gemm", + "args": { + "m": 8, + "n": 256, + "k": 10240, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 8, + "n": 256, + "k": 12288, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 8, + "n": 320, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 8, + "n": 384, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.gate", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.gate", + "deepseek-ai/DeepSeek-V4-Pro/ffn.gate", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.gate", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.gate", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.gate" + ] + }, + { + "type": "gemm", + "args": { + "m": 8, + "n": 384, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w3" + ] + }, + { + "type": "gemm", + "args": { + "m": 8, + "n": 512, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wkv", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wkv", + "deepseek-ai/DeepSeek-V4-Pro/attn.wkv", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wkv", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wkv", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wkv" + ] + }, + { + "type": "gemm", + "args": { + "m": 8, + "n": 512, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.compressor.wkv+wgate", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.compressor.wkv+wgate", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.compressor.wkv+wgate" + ] + }, + { + "type": "gemm", + "args": { + "m": 8, + "n": 512, + "k": 10240, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 8, + "n": 512, + "k": 10240, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 8, + "n": 512, + "k": 12288, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 8, + "n": 512, + "k": 12288, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 8, + "n": 768, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w3" + ] + }, + { + "type": "gemm", + "args": { + "m": 8, + "n": 896, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 8, + "n": 1024, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.compressor.wkv", + "deepseek-ai/DeepSeek-V4-Pro/attn.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro/attn.compressor.wkv", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.compressor.wgate", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.compressor.wkv" + ] + }, + { + "type": "gemm", + "args": { + "m": 8, + "n": 1024, + "k": 10240, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 8, + "n": 1024, + "k": 12288, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 8, + "n": 1536, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w3" + ] + }, + { + "type": "gemm", + "args": { + "m": 8, + "n": 1536, + "k": 10240, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 8, + "n": 1536, + "k": 12288, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 8, + "n": 1792, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 8, + "n": 4096, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 8, + "n": 5120, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 8, + "n": 6144, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3" + ] + }, + { + "type": "gemm", + "args": { + "m": 8, + "n": 7168, + "k": 192, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w2" + ] + }, + { + "type": "gemm", + "args": { + "m": 8, + "n": 7168, + "k": 384, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w2" + ] + }, + { + "type": "gemm", + "args": { + "m": 8, + "n": 7168, + "k": 448, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 8, + "n": 7168, + "k": 768, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w2" + ] + }, + { + "type": "gemm", + "args": { + "m": 8, + "n": 7168, + "k": 896, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 8, + "n": 7168, + "k": 1024, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 8, + "n": 7168, + "k": 2048, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 8, + "n": 7168, + "k": 3072, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w2" + ] + }, + { + "type": "gemm", + "args": { + "m": 8, + "n": 7168, + "k": 4096, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 8, + "n": 7168, + "k": 16384, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 8, + "n": 8192, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 8, + "n": 10240, + "k": 1024, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 8, + "n": 12288, + "k": 1024, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 8, + "n": 16384, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 8, + "n": 65536, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 9, + "n": 24, + "k": 28672, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/hc_attn", + "deepseek-ai/DeepSeek-V4-Pro-0813/hc_ffn", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.hc_attn", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.hc_ffn", + "deepseek-ai/DeepSeek-V4-Pro/hc_attn", + "deepseek-ai/DeepSeek-V4-Pro/hc_ffn", + "deepseek-ai/DeepSeek-V4-Pro/mtp.hc_attn", + "deepseek-ai/DeepSeek-V4-Pro/mtp.hc_ffn", + "sgl-project/DeepSeek-V4-Pro-FP8/hc_attn", + "sgl-project/DeepSeek-V4-Pro-FP8/hc_ffn", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.hc_attn", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.hc_ffn" + ] + }, + { + "type": "gemm", + "args": { + "m": 9, + "n": 64, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.weights_proj", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.weights_proj", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.weights_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 9, + "n": 256, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.compressor.wkv", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.compressor.wkv", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.compressor.wgate", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.compressor.wkv" + ] + }, + { + "type": "gemm", + "args": { + "m": 9, + "n": 320, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 9, + "n": 384, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.gate", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.gate", + "deepseek-ai/DeepSeek-V4-Pro/ffn.gate", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.gate", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.gate", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.gate" + ] + }, + { + "type": "gemm", + "args": { + "m": 9, + "n": 384, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w3" + ] + }, + { + "type": "gemm", + "args": { + "m": 9, + "n": 512, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wkv", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wkv", + "deepseek-ai/DeepSeek-V4-Pro/attn.wkv", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wkv", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wkv", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wkv" + ] + }, + { + "type": "gemm", + "args": { + "m": 9, + "n": 512, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.compressor.wkv+wgate", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.compressor.wkv+wgate", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.compressor.wkv+wgate" + ] + }, + { + "type": "gemm", + "args": { + "m": 9, + "n": 896, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 9, + "n": 1024, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.compressor.wkv", + "deepseek-ai/DeepSeek-V4-Pro/attn.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro/attn.compressor.wkv", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.compressor.wgate", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.compressor.wkv" + ] + }, + { + "type": "gemm", + "args": { + "m": 9, + "n": 1536, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w3" + ] + }, + { + "type": "gemm", + "args": { + "m": 9, + "n": 4096, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 9, + "n": 6144, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3" + ] + }, + { + "type": "gemm", + "args": { + "m": 9, + "n": 7168, + "k": 192, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w2" + ] + }, + { + "type": "gemm", + "args": { + "m": 9, + "n": 7168, + "k": 448, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 9, + "n": 7168, + "k": 1024, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 9, + "n": 7168, + "k": 3072, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w2" + ] + }, + { + "type": "gemm", + "args": { + "m": 9, + "n": 7168, + "k": 16384, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 9, + "n": 8192, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 9, + "n": 65536, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 10, + "n": 768, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w3" + ] + }, + { + "type": "gemm", + "args": { + "m": 10, + "n": 7168, + "k": 384, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w2" + ] + }, + { + "type": "gemm", + "args": { + "m": 10, + "n": 7168, + "k": 2048, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 11, + "n": 24, + "k": 28672, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/hc_attn", + "deepseek-ai/DeepSeek-V4-Pro-0813/hc_ffn", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.hc_attn", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.hc_ffn", + "deepseek-ai/DeepSeek-V4-Pro/hc_attn", + "deepseek-ai/DeepSeek-V4-Pro/hc_ffn", + "deepseek-ai/DeepSeek-V4-Pro/mtp.hc_attn", + "deepseek-ai/DeepSeek-V4-Pro/mtp.hc_ffn", + "sgl-project/DeepSeek-V4-Pro-FP8/hc_attn", + "sgl-project/DeepSeek-V4-Pro-FP8/hc_ffn", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.hc_attn", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.hc_ffn" + ] + }, + { + "type": "gemm", + "args": { + "m": 11, + "n": 64, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.weights_proj", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.weights_proj", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.weights_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 11, + "n": 256, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.compressor.wkv", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.compressor.wkv", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.compressor.wgate", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.compressor.wkv" + ] + }, + { + "type": "gemm", + "args": { + "m": 11, + "n": 320, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 11, + "n": 384, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.gate", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.gate", + "deepseek-ai/DeepSeek-V4-Pro/ffn.gate", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.gate", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.gate", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.gate" + ] + }, + { + "type": "gemm", + "args": { + "m": 11, + "n": 384, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w3" + ] + }, + { + "type": "gemm", + "args": { + "m": 11, + "n": 512, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wkv", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wkv", + "deepseek-ai/DeepSeek-V4-Pro/attn.wkv", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wkv", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wkv", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wkv" + ] + }, + { + "type": "gemm", + "args": { + "m": 11, + "n": 512, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.compressor.wkv+wgate", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.compressor.wkv+wgate", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.compressor.wkv+wgate" + ] + }, + { + "type": "gemm", + "args": { + "m": 11, + "n": 768, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w3" + ] + }, + { + "type": "gemm", + "args": { + "m": 11, + "n": 896, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 11, + "n": 1024, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.compressor.wkv", + "deepseek-ai/DeepSeek-V4-Pro/attn.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro/attn.compressor.wkv", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.compressor.wgate", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.compressor.wkv" + ] + }, + { + "type": "gemm", + "args": { + "m": 11, + "n": 1536, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w3" + ] + }, + { + "type": "gemm", + "args": { + "m": 11, + "n": 4096, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 11, + "n": 7168, + "k": 192, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w2" + ] + }, + { + "type": "gemm", + "args": { + "m": 11, + "n": 7168, + "k": 384, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w2" + ] + }, + { + "type": "gemm", + "args": { + "m": 11, + "n": 7168, + "k": 448, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 11, + "n": 7168, + "k": 1024, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 11, + "n": 7168, + "k": 2048, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 11, + "n": 7168, + "k": 16384, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 11, + "n": 8192, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 11, + "n": 65536, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 12, + "n": 6144, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3" + ] + }, + { + "type": "gemm", + "args": { + "m": 12, + "n": 7168, + "k": 3072, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w2" + ] + }, + { + "type": "gemm", + "args": { + "m": 14, + "n": 24, + "k": 28672, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/hc_attn", + "deepseek-ai/DeepSeek-V4-Pro-0813/hc_ffn", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.hc_attn", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.hc_ffn", + "deepseek-ai/DeepSeek-V4-Pro/hc_attn", + "deepseek-ai/DeepSeek-V4-Pro/hc_ffn", + "deepseek-ai/DeepSeek-V4-Pro/mtp.hc_attn", + "deepseek-ai/DeepSeek-V4-Pro/mtp.hc_ffn", + "sgl-project/DeepSeek-V4-Pro-FP8/hc_attn", + "sgl-project/DeepSeek-V4-Pro-FP8/hc_ffn", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.hc_attn", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.hc_ffn" + ] + }, + { + "type": "gemm", + "args": { + "m": 14, + "n": 64, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.weights_proj", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.weights_proj", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.weights_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 14, + "n": 256, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.compressor.wkv", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.compressor.wkv", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.compressor.wgate", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.compressor.wkv" + ] + }, + { + "type": "gemm", + "args": { + "m": 14, + "n": 320, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 14, + "n": 384, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.gate", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.gate", + "deepseek-ai/DeepSeek-V4-Pro/ffn.gate", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.gate", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.gate", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.gate" + ] + }, + { + "type": "gemm", + "args": { + "m": 14, + "n": 384, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w3" + ] + }, + { + "type": "gemm", + "args": { + "m": 14, + "n": 512, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wkv", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wkv", + "deepseek-ai/DeepSeek-V4-Pro/attn.wkv", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wkv", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wkv", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wkv" + ] + }, + { + "type": "gemm", + "args": { + "m": 14, + "n": 512, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.compressor.wkv+wgate", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.compressor.wkv+wgate", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.compressor.wkv+wgate" + ] + }, + { + "type": "gemm", + "args": { + "m": 14, + "n": 768, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w3" + ] + }, + { + "type": "gemm", + "args": { + "m": 14, + "n": 896, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 14, + "n": 1024, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.compressor.wkv", + "deepseek-ai/DeepSeek-V4-Pro/attn.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro/attn.compressor.wkv", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.compressor.wgate", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.compressor.wkv" + ] + }, + { + "type": "gemm", + "args": { + "m": 14, + "n": 1536, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w3" + ] + }, + { + "type": "gemm", + "args": { + "m": 14, + "n": 4096, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 14, + "n": 7168, + "k": 192, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w2" + ] + }, + { + "type": "gemm", + "args": { + "m": 14, + "n": 7168, + "k": 384, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w2" + ] + }, + { + "type": "gemm", + "args": { + "m": 14, + "n": 7168, + "k": 448, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 14, + "n": 7168, + "k": 1024, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 14, + "n": 7168, + "k": 2048, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 14, + "n": 7168, + "k": 16384, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 14, + "n": 8192, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 14, + "n": 65536, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 15, + "n": 6144, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3" + ] + }, + { + "type": "gemm", + "args": { + "m": 15, + "n": 7168, + "k": 3072, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w2" + ] + }, + { + "type": "gemm", + "args": { + "m": 16, + "n": 24, + "k": 28672, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/hc_attn", + "deepseek-ai/DeepSeek-V4-Pro-0813/hc_ffn", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.hc_attn", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.hc_ffn", + "deepseek-ai/DeepSeek-V4-Pro/hc_attn", + "deepseek-ai/DeepSeek-V4-Pro/hc_ffn", + "deepseek-ai/DeepSeek-V4-Pro/mtp.hc_attn", + "deepseek-ai/DeepSeek-V4-Pro/mtp.hc_ffn", + "sgl-project/DeepSeek-V4-Pro-FP8/hc_attn", + "sgl-project/DeepSeek-V4-Pro-FP8/hc_ffn", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.hc_attn", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.hc_ffn" + ] + }, + { + "type": "gemm", + "args": { + "m": 16, + "n": 64, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.weights_proj", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.weights_proj", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.weights_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 16, + "n": 256, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.compressor.wkv", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.compressor.wkv", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.compressor.wgate", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.compressor.wkv" + ] + }, + { + "type": "gemm", + "args": { + "m": 16, + "n": 384, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.gate", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.gate", + "deepseek-ai/DeepSeek-V4-Pro/ffn.gate", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.gate", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.gate", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.gate" + ] + }, + { + "type": "gemm", + "args": { + "m": 16, + "n": 384, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w3" + ] + }, + { + "type": "gemm", + "args": { + "m": 16, + "n": 512, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wkv", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wkv", + "deepseek-ai/DeepSeek-V4-Pro/attn.wkv", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wkv", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wkv", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wkv" + ] + }, + { + "type": "gemm", + "args": { + "m": 16, + "n": 512, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.compressor.wkv+wgate", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.compressor.wkv+wgate", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.compressor.wkv+wgate" + ] + }, + { + "type": "gemm", + "args": { + "m": 16, + "n": 1024, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.compressor.wkv", + "deepseek-ai/DeepSeek-V4-Pro/attn.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro/attn.compressor.wkv", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.compressor.wgate", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.compressor.wkv" + ] + }, + { + "type": "gemm", + "args": { + "m": 16, + "n": 1536, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w3" + ] + }, + { + "type": "gemm", + "args": { + "m": 16, + "n": 4096, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 16, + "n": 6144, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3" + ] + }, + { + "type": "gemm", + "args": { + "m": 16, + "n": 7168, + "k": 192, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w2" + ] + }, + { + "type": "gemm", + "args": { + "m": 16, + "n": 7168, + "k": 1024, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 16, + "n": 7168, + "k": 3072, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w2" + ] + }, + { + "type": "gemm", + "args": { + "m": 16, + "n": 7168, + "k": 16384, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 16, + "n": 8192, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 16, + "n": 65536, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 17, + "n": 6144, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3" + ] + }, + { + "type": "gemm", + "args": { + "m": 17, + "n": 7168, + "k": 3072, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w2" + ] + }, + { + "type": "gemm", + "args": { + "m": 18, + "n": 24, + "k": 28672, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/hc_attn", + "deepseek-ai/DeepSeek-V4-Pro-0813/hc_ffn", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.hc_attn", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.hc_ffn", + "deepseek-ai/DeepSeek-V4-Pro/hc_attn", + "deepseek-ai/DeepSeek-V4-Pro/hc_ffn", + "deepseek-ai/DeepSeek-V4-Pro/mtp.hc_attn", + "deepseek-ai/DeepSeek-V4-Pro/mtp.hc_ffn", + "sgl-project/DeepSeek-V4-Pro-FP8/hc_attn", + "sgl-project/DeepSeek-V4-Pro-FP8/hc_ffn", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.hc_attn", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.hc_ffn" + ] + }, + { + "type": "gemm", + "args": { + "m": 18, + "n": 64, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.weights_proj", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.weights_proj", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.weights_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 18, + "n": 256, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.compressor.wkv", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.compressor.wkv", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.compressor.wgate", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.compressor.wkv" + ] + }, + { + "type": "gemm", + "args": { + "m": 18, + "n": 384, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.gate", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.gate", + "deepseek-ai/DeepSeek-V4-Pro/ffn.gate", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.gate", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.gate", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.gate" + ] + }, + { + "type": "gemm", + "args": { + "m": 18, + "n": 512, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wkv", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wkv", + "deepseek-ai/DeepSeek-V4-Pro/attn.wkv", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wkv", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wkv", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wkv" + ] + }, + { + "type": "gemm", + "args": { + "m": 18, + "n": 512, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.compressor.wkv+wgate", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.compressor.wkv+wgate", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.compressor.wkv+wgate" + ] + }, + { + "type": "gemm", + "args": { + "m": 18, + "n": 1024, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.compressor.wkv", + "deepseek-ai/DeepSeek-V4-Pro/attn.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro/attn.compressor.wkv", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.compressor.wgate", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.compressor.wkv" + ] + }, + { + "type": "gemm", + "args": { + "m": 18, + "n": 1536, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w3" + ] + }, + { + "type": "gemm", + "args": { + "m": 18, + "n": 7168, + "k": 16384, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 18, + "n": 8192, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 18, + "n": 65536, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 22, + "n": 6144, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3" + ] + }, + { + "type": "gemm", + "args": { + "m": 22, + "n": 7168, + "k": 3072, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w2" + ] + }, + { + "type": "gemm", + "args": { + "m": 23, + "n": 24, + "k": 28672, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/hc_attn", + "deepseek-ai/DeepSeek-V4-Pro-0813/hc_ffn", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.hc_attn", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.hc_ffn", + "deepseek-ai/DeepSeek-V4-Pro/hc_attn", + "deepseek-ai/DeepSeek-V4-Pro/hc_ffn", + "deepseek-ai/DeepSeek-V4-Pro/mtp.hc_attn", + "deepseek-ai/DeepSeek-V4-Pro/mtp.hc_ffn", + "sgl-project/DeepSeek-V4-Pro-FP8/hc_attn", + "sgl-project/DeepSeek-V4-Pro-FP8/hc_ffn", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.hc_attn", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.hc_ffn" + ] + }, + { + "type": "gemm", + "args": { + "m": 23, + "n": 64, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.weights_proj", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.weights_proj", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.weights_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 23, + "n": 256, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.compressor.wkv", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.compressor.wkv", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.compressor.wgate", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.compressor.wkv" + ] + }, + { + "type": "gemm", + "args": { + "m": 23, + "n": 384, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.gate", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.gate", + "deepseek-ai/DeepSeek-V4-Pro/ffn.gate", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.gate", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.gate", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.gate" + ] + }, + { + "type": "gemm", + "args": { + "m": 23, + "n": 512, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wkv", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wkv", + "deepseek-ai/DeepSeek-V4-Pro/attn.wkv", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wkv", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wkv", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wkv" + ] + }, + { + "type": "gemm", + "args": { + "m": 23, + "n": 512, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.compressor.wkv+wgate", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.compressor.wkv+wgate", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.compressor.wkv+wgate" + ] + }, + { + "type": "gemm", + "args": { + "m": 23, + "n": 1024, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.compressor.wkv", + "deepseek-ai/DeepSeek-V4-Pro/attn.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro/attn.compressor.wkv", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.compressor.wgate", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.compressor.wkv" + ] + }, + { + "type": "gemm", + "args": { + "m": 23, + "n": 1536, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w3" + ] + }, + { + "type": "gemm", + "args": { + "m": 23, + "n": 7168, + "k": 16384, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 23, + "n": 8192, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 23, + "n": 65536, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 24, + "n": 24, + "k": 28672, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/hc_attn", + "deepseek-ai/DeepSeek-V4-Pro-0813/hc_ffn", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.hc_attn", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.hc_ffn", + "deepseek-ai/DeepSeek-V4-Pro/hc_attn", + "deepseek-ai/DeepSeek-V4-Pro/hc_ffn", + "deepseek-ai/DeepSeek-V4-Pro/mtp.hc_attn", + "deepseek-ai/DeepSeek-V4-Pro/mtp.hc_ffn", + "sgl-project/DeepSeek-V4-Pro-FP8/hc_attn", + "sgl-project/DeepSeek-V4-Pro-FP8/hc_ffn", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.hc_attn", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.hc_ffn" + ] + }, + { + "type": "gemm", + "args": { + "m": 24, + "n": 64, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.weights_proj", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.weights_proj", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.weights_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 24, + "n": 256, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.compressor.wkv", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.compressor.wkv", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.compressor.wgate", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.compressor.wkv" + ] + }, + { + "type": "gemm", + "args": { + "m": 24, + "n": 384, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.gate", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.gate", + "deepseek-ai/DeepSeek-V4-Pro/ffn.gate", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.gate", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.gate", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.gate" + ] + }, + { + "type": "gemm", + "args": { + "m": 24, + "n": 512, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wkv", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wkv", + "deepseek-ai/DeepSeek-V4-Pro/attn.wkv", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wkv", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wkv", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wkv" + ] + }, + { + "type": "gemm", + "args": { + "m": 24, + "n": 512, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.compressor.wkv+wgate", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.compressor.wkv+wgate", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.compressor.wkv+wgate" + ] + }, + { + "type": "gemm", + "args": { + "m": 24, + "n": 1024, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.compressor.wkv", + "deepseek-ai/DeepSeek-V4-Pro/attn.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro/attn.compressor.wkv", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.compressor.wgate", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.compressor.wkv" + ] + }, + { + "type": "gemm", + "args": { + "m": 24, + "n": 1536, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w3" + ] + }, + { + "type": "gemm", + "args": { + "m": 24, + "n": 6144, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3" + ] + }, + { + "type": "gemm", + "args": { + "m": 24, + "n": 7168, + "k": 3072, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w2" + ] + }, + { + "type": "gemm", + "args": { + "m": 24, + "n": 7168, + "k": 16384, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 24, + "n": 8192, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 24, + "n": 65536, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 29, + "n": 24, + "k": 28672, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/hc_attn", + "deepseek-ai/DeepSeek-V4-Pro-0813/hc_ffn", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.hc_attn", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.hc_ffn", + "deepseek-ai/DeepSeek-V4-Pro/hc_attn", + "deepseek-ai/DeepSeek-V4-Pro/hc_ffn", + "deepseek-ai/DeepSeek-V4-Pro/mtp.hc_attn", + "deepseek-ai/DeepSeek-V4-Pro/mtp.hc_ffn", + "sgl-project/DeepSeek-V4-Pro-FP8/hc_attn", + "sgl-project/DeepSeek-V4-Pro-FP8/hc_ffn", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.hc_attn", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.hc_ffn" + ] + }, + { + "type": "gemm", + "args": { + "m": 29, + "n": 64, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.weights_proj", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.weights_proj", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.weights_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 29, + "n": 256, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.compressor.wkv", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.compressor.wkv", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.compressor.wgate", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.compressor.wkv" + ] + }, + { + "type": "gemm", + "args": { + "m": 29, + "n": 384, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.gate", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.gate", + "deepseek-ai/DeepSeek-V4-Pro/ffn.gate", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.gate", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.gate", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.gate" + ] + }, + { + "type": "gemm", + "args": { + "m": 29, + "n": 512, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wkv", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wkv", + "deepseek-ai/DeepSeek-V4-Pro/attn.wkv", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wkv", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wkv", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wkv" + ] + }, + { + "type": "gemm", + "args": { + "m": 29, + "n": 512, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.compressor.wkv+wgate", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.compressor.wkv+wgate", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.compressor.wkv+wgate" + ] + }, + { + "type": "gemm", + "args": { + "m": 29, + "n": 1024, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.compressor.wkv", + "deepseek-ai/DeepSeek-V4-Pro/attn.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro/attn.compressor.wkv", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.compressor.wgate", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.compressor.wkv" + ] + }, + { + "type": "gemm", + "args": { + "m": 29, + "n": 1536, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w3" + ] + }, + { + "type": "gemm", + "args": { + "m": 29, + "n": 6144, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3" + ] + }, + { + "type": "gemm", + "args": { + "m": 29, + "n": 7168, + "k": 3072, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w2" + ] + }, + { + "type": "gemm", + "args": { + "m": 29, + "n": 7168, + "k": 16384, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 29, + "n": 8192, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 29, + "n": 65536, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 30, + "n": 6144, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3" + ] + }, + { + "type": "gemm", + "args": { + "m": 30, + "n": 7168, + "k": 3072, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w2" + ] + }, + { + "type": "gemm", + "args": { + "m": 32, + "n": 24, + "k": 28672, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/hc_attn", + "deepseek-ai/DeepSeek-V4-Pro-0813/hc_ffn", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.hc_attn", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.hc_ffn", + "deepseek-ai/DeepSeek-V4-Pro/hc_attn", + "deepseek-ai/DeepSeek-V4-Pro/hc_ffn", + "deepseek-ai/DeepSeek-V4-Pro/mtp.hc_attn", + "deepseek-ai/DeepSeek-V4-Pro/mtp.hc_ffn", + "sgl-project/DeepSeek-V4-Pro-FP8/hc_attn", + "sgl-project/DeepSeek-V4-Pro-FP8/hc_ffn", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.hc_attn", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.hc_ffn" + ] + }, + { + "type": "gemm", + "args": { + "m": 32, + "n": 64, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.weights_proj", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.weights_proj", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.weights_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 32, + "n": 256, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.compressor.wkv", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.compressor.wkv", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.compressor.wgate", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.compressor.wkv" + ] + }, + { + "type": "gemm", + "args": { + "m": 32, + "n": 384, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.gate", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.gate", + "deepseek-ai/DeepSeek-V4-Pro/ffn.gate", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.gate", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.gate", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.gate" + ] + }, + { + "type": "gemm", + "args": { + "m": 32, + "n": 512, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wkv", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wkv", + "deepseek-ai/DeepSeek-V4-Pro/attn.wkv", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wkv", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wkv", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wkv" + ] + }, + { + "type": "gemm", + "args": { + "m": 32, + "n": 512, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.compressor.wkv+wgate", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.compressor.wkv+wgate", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.compressor.wkv+wgate" + ] + }, + { + "type": "gemm", + "args": { + "m": 32, + "n": 1024, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.compressor.wkv", + "deepseek-ai/DeepSeek-V4-Pro/attn.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro/attn.compressor.wkv", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.compressor.wgate", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.compressor.wkv" + ] + }, + { + "type": "gemm", + "args": { + "m": 32, + "n": 1536, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w3" + ] + }, + { + "type": "gemm", + "args": { + "m": 32, + "n": 6144, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3" + ] + }, + { + "type": "gemm", + "args": { + "m": 32, + "n": 7168, + "k": 3072, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w2" + ] + }, + { + "type": "gemm", + "args": { + "m": 32, + "n": 7168, + "k": 16384, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 32, + "n": 8192, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 32, + "n": 65536, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 38, + "n": 24, + "k": 28672, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/hc_attn", + "deepseek-ai/DeepSeek-V4-Pro-0813/hc_ffn", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.hc_attn", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.hc_ffn", + "deepseek-ai/DeepSeek-V4-Pro/hc_attn", + "deepseek-ai/DeepSeek-V4-Pro/hc_ffn", + "deepseek-ai/DeepSeek-V4-Pro/mtp.hc_attn", + "deepseek-ai/DeepSeek-V4-Pro/mtp.hc_ffn", + "sgl-project/DeepSeek-V4-Pro-FP8/hc_attn", + "sgl-project/DeepSeek-V4-Pro-FP8/hc_ffn", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.hc_attn", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.hc_ffn" + ] + }, + { + "type": "gemm", + "args": { + "m": 38, + "n": 64, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.weights_proj", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.weights_proj", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.weights_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 38, + "n": 256, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.compressor.wkv", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.compressor.wkv", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.compressor.wgate", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.compressor.wkv" + ] + }, + { + "type": "gemm", + "args": { + "m": 38, + "n": 384, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.gate", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.gate", + "deepseek-ai/DeepSeek-V4-Pro/ffn.gate", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.gate", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.gate", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.gate" + ] + }, + { + "type": "gemm", + "args": { + "m": 38, + "n": 512, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wkv", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wkv", + "deepseek-ai/DeepSeek-V4-Pro/attn.wkv", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wkv", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wkv", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wkv" + ] + }, + { + "type": "gemm", + "args": { + "m": 38, + "n": 512, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.compressor.wkv+wgate", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.compressor.wkv+wgate", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.compressor.wkv+wgate" + ] + }, + { + "type": "gemm", + "args": { + "m": 38, + "n": 1024, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.compressor.wkv", + "deepseek-ai/DeepSeek-V4-Pro/attn.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro/attn.compressor.wkv", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.compressor.wgate", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.compressor.wkv" + ] + }, + { + "type": "gemm", + "args": { + "m": 38, + "n": 1536, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w3" + ] + }, + { + "type": "gemm", + "args": { + "m": 38, + "n": 6144, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3" + ] + }, + { + "type": "gemm", + "args": { + "m": 38, + "n": 7168, + "k": 3072, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w2" + ] + }, + { + "type": "gemm", + "args": { + "m": 38, + "n": 7168, + "k": 16384, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 38, + "n": 8192, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 38, + "n": 65536, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 40, + "n": 24, + "k": 28672, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/hc_attn", + "deepseek-ai/DeepSeek-V4-Pro-0813/hc_ffn", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.hc_attn", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.hc_ffn", + "deepseek-ai/DeepSeek-V4-Pro/hc_attn", + "deepseek-ai/DeepSeek-V4-Pro/hc_ffn", + "deepseek-ai/DeepSeek-V4-Pro/mtp.hc_attn", + "deepseek-ai/DeepSeek-V4-Pro/mtp.hc_ffn", + "sgl-project/DeepSeek-V4-Pro-FP8/hc_attn", + "sgl-project/DeepSeek-V4-Pro-FP8/hc_ffn", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.hc_attn", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.hc_ffn" + ] + }, + { + "type": "gemm", + "args": { + "m": 40, + "n": 64, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.weights_proj", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.weights_proj", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.weights_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 40, + "n": 256, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.compressor.wkv", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.compressor.wkv", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.compressor.wgate", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.compressor.wkv" + ] + }, + { + "type": "gemm", + "args": { + "m": 40, + "n": 384, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.gate", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.gate", + "deepseek-ai/DeepSeek-V4-Pro/ffn.gate", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.gate", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.gate", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.gate" + ] + }, + { + "type": "gemm", + "args": { + "m": 40, + "n": 512, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wkv", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wkv", + "deepseek-ai/DeepSeek-V4-Pro/attn.wkv", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wkv", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wkv", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wkv" + ] + }, + { + "type": "gemm", + "args": { + "m": 40, + "n": 512, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.compressor.wkv+wgate", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.compressor.wkv+wgate", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.compressor.wkv+wgate" + ] + }, + { + "type": "gemm", + "args": { + "m": 40, + "n": 1024, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.compressor.wkv", + "deepseek-ai/DeepSeek-V4-Pro/attn.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro/attn.compressor.wkv", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.compressor.wgate", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.compressor.wkv" + ] + }, + { + "type": "gemm", + "args": { + "m": 40, + "n": 1536, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w3" + ] + }, + { + "type": "gemm", + "args": { + "m": 40, + "n": 6144, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3" + ] + }, + { + "type": "gemm", + "args": { + "m": 40, + "n": 7168, + "k": 3072, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w2" + ] + }, + { + "type": "gemm", + "args": { + "m": 40, + "n": 7168, + "k": 16384, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 40, + "n": 8192, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 40, + "n": 65536, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 41, + "n": 6144, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3" + ] + }, + { + "type": "gemm", + "args": { + "m": 41, + "n": 7168, + "k": 3072, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w2" + ] + }, + { + "type": "gemm", + "args": { + "m": 48, + "n": 24, + "k": 28672, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/hc_attn", + "deepseek-ai/DeepSeek-V4-Pro-0813/hc_ffn", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.hc_attn", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.hc_ffn", + "deepseek-ai/DeepSeek-V4-Pro/hc_attn", + "deepseek-ai/DeepSeek-V4-Pro/hc_ffn", + "deepseek-ai/DeepSeek-V4-Pro/mtp.hc_attn", + "deepseek-ai/DeepSeek-V4-Pro/mtp.hc_ffn", + "sgl-project/DeepSeek-V4-Pro-FP8/hc_attn", + "sgl-project/DeepSeek-V4-Pro-FP8/hc_ffn", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.hc_attn", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.hc_ffn" + ] + }, + { + "type": "gemm", + "args": { + "m": 48, + "n": 64, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.weights_proj", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.weights_proj", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.weights_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 48, + "n": 256, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.compressor.wkv", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.compressor.wkv", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.compressor.wgate", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.compressor.wkv" + ] + }, + { + "type": "gemm", + "args": { + "m": 48, + "n": 384, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.gate", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.gate", + "deepseek-ai/DeepSeek-V4-Pro/ffn.gate", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.gate", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.gate", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.gate" + ] + }, + { + "type": "gemm", + "args": { + "m": 48, + "n": 512, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wkv", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wkv", + "deepseek-ai/DeepSeek-V4-Pro/attn.wkv", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wkv", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wkv", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wkv" + ] + }, + { + "type": "gemm", + "args": { + "m": 48, + "n": 512, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.compressor.wkv+wgate", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.compressor.wkv+wgate", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.compressor.wkv+wgate" + ] + }, + { + "type": "gemm", + "args": { + "m": 48, + "n": 1024, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.compressor.wkv", + "deepseek-ai/DeepSeek-V4-Pro/attn.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro/attn.compressor.wkv", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.compressor.wgate", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.compressor.wkv" + ] + }, + { + "type": "gemm", + "args": { + "m": 48, + "n": 1536, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w3" + ] + }, + { + "type": "gemm", + "args": { + "m": 48, + "n": 6144, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3" + ] + }, + { + "type": "gemm", + "args": { + "m": 48, + "n": 7168, + "k": 3072, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w2" + ] + }, + { + "type": "gemm", + "args": { + "m": 48, + "n": 7168, + "k": 16384, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 48, + "n": 8192, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 48, + "n": 65536, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 50, + "n": 24, + "k": 28672, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/hc_attn", + "deepseek-ai/DeepSeek-V4-Pro-0813/hc_ffn", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.hc_attn", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.hc_ffn", + "deepseek-ai/DeepSeek-V4-Pro/hc_attn", + "deepseek-ai/DeepSeek-V4-Pro/hc_ffn", + "deepseek-ai/DeepSeek-V4-Pro/mtp.hc_attn", + "deepseek-ai/DeepSeek-V4-Pro/mtp.hc_ffn", + "sgl-project/DeepSeek-V4-Pro-FP8/hc_attn", + "sgl-project/DeepSeek-V4-Pro-FP8/hc_ffn", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.hc_attn", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.hc_ffn" + ] + }, + { + "type": "gemm", + "args": { + "m": 50, + "n": 64, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.weights_proj", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.weights_proj", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.weights_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 50, + "n": 256, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.compressor.wkv", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.compressor.wkv", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.compressor.wgate", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.compressor.wkv" + ] + }, + { + "type": "gemm", + "args": { + "m": 50, + "n": 384, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.gate", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.gate", + "deepseek-ai/DeepSeek-V4-Pro/ffn.gate", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.gate", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.gate", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.gate" + ] + }, + { + "type": "gemm", + "args": { + "m": 50, + "n": 512, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wkv", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wkv", + "deepseek-ai/DeepSeek-V4-Pro/attn.wkv", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wkv", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wkv", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wkv" + ] + }, + { + "type": "gemm", + "args": { + "m": 50, + "n": 512, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.compressor.wkv+wgate", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.compressor.wkv+wgate", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.compressor.wkv+wgate" + ] + }, + { + "type": "gemm", + "args": { + "m": 50, + "n": 1024, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.compressor.wkv", + "deepseek-ai/DeepSeek-V4-Pro/attn.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro/attn.compressor.wkv", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.compressor.wgate", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.compressor.wkv" + ] + }, + { + "type": "gemm", + "args": { + "m": 50, + "n": 1536, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w3" + ] + }, + { + "type": "gemm", + "args": { + "m": 50, + "n": 6144, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3" + ] + }, + { + "type": "gemm", + "args": { + "m": 50, + "n": 7168, + "k": 3072, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w2" + ] + }, + { + "type": "gemm", + "args": { + "m": 50, + "n": 7168, + "k": 16384, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 50, + "n": 8192, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 50, + "n": 65536, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 53, + "n": 6144, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3" + ] + }, + { + "type": "gemm", + "args": { + "m": 53, + "n": 7168, + "k": 3072, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w2" + ] + }, + { + "type": "gemm", + "args": { + "m": 61, + "n": 24, + "k": 28672, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/hc_attn", + "deepseek-ai/DeepSeek-V4-Pro-0813/hc_ffn", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.hc_attn", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.hc_ffn", + "deepseek-ai/DeepSeek-V4-Pro/hc_attn", + "deepseek-ai/DeepSeek-V4-Pro/hc_ffn", + "deepseek-ai/DeepSeek-V4-Pro/mtp.hc_attn", + "deepseek-ai/DeepSeek-V4-Pro/mtp.hc_ffn", + "sgl-project/DeepSeek-V4-Pro-FP8/hc_attn", + "sgl-project/DeepSeek-V4-Pro-FP8/hc_ffn", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.hc_attn", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.hc_ffn" + ] + }, + { + "type": "gemm", + "args": { + "m": 61, + "n": 64, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.weights_proj", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.weights_proj", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.weights_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 61, + "n": 256, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.compressor.wkv", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.compressor.wkv", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.compressor.wgate", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.compressor.wkv" + ] + }, + { + "type": "gemm", + "args": { + "m": 61, + "n": 384, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.gate", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.gate", + "deepseek-ai/DeepSeek-V4-Pro/ffn.gate", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.gate", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.gate", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.gate" + ] + }, + { + "type": "gemm", + "args": { + "m": 61, + "n": 512, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wkv", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wkv", + "deepseek-ai/DeepSeek-V4-Pro/attn.wkv", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wkv", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wkv", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wkv" + ] + }, + { + "type": "gemm", + "args": { + "m": 61, + "n": 512, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.compressor.wkv+wgate", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.compressor.wkv+wgate", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.compressor.wkv+wgate" + ] + }, + { + "type": "gemm", + "args": { + "m": 61, + "n": 1024, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.compressor.wkv", + "deepseek-ai/DeepSeek-V4-Pro/attn.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro/attn.compressor.wkv", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.compressor.wgate", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.compressor.wkv" + ] + }, + { + "type": "gemm", + "args": { + "m": 61, + "n": 1536, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w3" + ] + }, + { + "type": "gemm", + "args": { + "m": 61, + "n": 6144, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3" + ] + }, + { + "type": "gemm", + "args": { + "m": 61, + "n": 7168, + "k": 3072, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w2" + ] + }, + { + "type": "gemm", + "args": { + "m": 61, + "n": 7168, + "k": 16384, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 61, + "n": 8192, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 61, + "n": 65536, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 725, + "n": 24, + "k": 28672, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/hc_attn", + "deepseek-ai/DeepSeek-V4-Pro-0813/hc_ffn", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.hc_attn", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.hc_ffn", + "deepseek-ai/DeepSeek-V4-Pro/hc_attn", + "deepseek-ai/DeepSeek-V4-Pro/hc_ffn", + "deepseek-ai/DeepSeek-V4-Pro/mtp.hc_attn", + "deepseek-ai/DeepSeek-V4-Pro/mtp.hc_ffn", + "sgl-project/DeepSeek-V4-Pro-FP8/hc_attn", + "sgl-project/DeepSeek-V4-Pro-FP8/hc_ffn", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.hc_attn", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.hc_ffn" + ] + }, + { + "type": "gemm", + "args": { + "m": 725, + "n": 24, + "k": 40960, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 725, + "n": 24, + "k": 49152, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 725, + "n": 64, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.weights_proj", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.weights_proj", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.weights_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 725, + "n": 64, + "k": 10240, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 725, + "n": 64, + "k": 12288, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 725, + "n": 120, + "k": 12288, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 725, + "n": 192, + "k": 10240, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 725, + "n": 256, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.compressor.wkv", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.compressor.wkv", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.compressor.wgate", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.compressor.wkv" + ] + }, + { + "type": "gemm", + "args": { + "m": 725, + "n": 256, + "k": 10240, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 725, + "n": 256, + "k": 12288, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 725, + "n": 320, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 725, + "n": 512, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wkv", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wkv", + "deepseek-ai/DeepSeek-V4-Pro/attn.wkv", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wkv", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wkv", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wkv" + ] + }, + { + "type": "gemm", + "args": { + "m": 725, + "n": 512, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.compressor.wkv+wgate", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.compressor.wkv+wgate", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.compressor.wkv+wgate" + ] + }, + { + "type": "gemm", + "args": { + "m": 725, + "n": 512, + "k": 10240, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 725, + "n": 512, + "k": 10240, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 725, + "n": 512, + "k": 12288, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 725, + "n": 512, + "k": 12288, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 725, + "n": 1024, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.compressor.wkv", + "deepseek-ai/DeepSeek-V4-Pro/attn.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro/attn.compressor.wkv", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.compressor.wgate", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.compressor.wkv" + ] + }, + { + "type": "gemm", + "args": { + "m": 725, + "n": 1024, + "k": 10240, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 725, + "n": 1024, + "k": 12288, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 725, + "n": 1536, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w3" + ] + }, + { + "type": "gemm", + "args": { + "m": 725, + "n": 1536, + "k": 10240, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 725, + "n": 1536, + "k": 12288, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 725, + "n": 4096, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 725, + "n": 5120, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 725, + "n": 7168, + "k": 1024, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 725, + "n": 8192, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 725, + "n": 10240, + "k": 1024, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 725, + "n": 12288, + "k": 1024, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 853, + "n": 384, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.gate", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.gate", + "deepseek-ai/DeepSeek-V4-Pro/ffn.gate", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.gate", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.gate", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.gate" + ] + }, + { + "type": "gemm", + "args": { + "m": 853, + "n": 384, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w3" + ] + }, + { + "type": "gemm", + "args": { + "m": 853, + "n": 896, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 853, + "n": 7168, + "k": 192, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w2" + ] + }, + { + "type": "gemm", + "args": { + "m": 853, + "n": 7168, + "k": 448, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 978, + "n": 24, + "k": 28672, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/hc_attn", + "deepseek-ai/DeepSeek-V4-Pro-0813/hc_ffn", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.hc_attn", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.hc_ffn", + "deepseek-ai/DeepSeek-V4-Pro/hc_attn", + "deepseek-ai/DeepSeek-V4-Pro/hc_ffn", + "deepseek-ai/DeepSeek-V4-Pro/mtp.hc_attn", + "deepseek-ai/DeepSeek-V4-Pro/mtp.hc_ffn", + "sgl-project/DeepSeek-V4-Pro-FP8/hc_attn", + "sgl-project/DeepSeek-V4-Pro-FP8/hc_ffn", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.hc_attn", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.hc_ffn" + ] + }, + { + "type": "gemm", + "args": { + "m": 978, + "n": 64, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.weights_proj", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.weights_proj", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.weights_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 978, + "n": 256, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.compressor.wkv", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.compressor.wkv", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.compressor.wgate", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.compressor.wkv" + ] + }, + { + "type": "gemm", + "args": { + "m": 978, + "n": 320, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 978, + "n": 512, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wkv", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wkv", + "deepseek-ai/DeepSeek-V4-Pro/attn.wkv", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wkv", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wkv", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wkv" + ] + }, + { + "type": "gemm", + "args": { + "m": 978, + "n": 512, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.compressor.wkv+wgate", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.compressor.wkv+wgate", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.compressor.wkv+wgate" + ] + }, + { + "type": "gemm", + "args": { + "m": 978, + "n": 1024, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.compressor.wkv", + "deepseek-ai/DeepSeek-V4-Pro/attn.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro/attn.compressor.wkv", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.compressor.wgate", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.compressor.wkv" + ] + }, + { + "type": "gemm", + "args": { + "m": 978, + "n": 1536, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w3" + ] + }, + { + "type": "gemm", + "args": { + "m": 978, + "n": 1792, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 978, + "n": 7168, + "k": 896, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 978, + "n": 7168, + "k": 2048, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 978, + "n": 8192, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 1684, + "n": 24, + "k": 28672, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/hc_attn", + "deepseek-ai/DeepSeek-V4-Pro-0813/hc_ffn", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.hc_attn", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.hc_ffn", + "deepseek-ai/DeepSeek-V4-Pro/hc_attn", + "deepseek-ai/DeepSeek-V4-Pro/hc_ffn", + "deepseek-ai/DeepSeek-V4-Pro/mtp.hc_attn", + "deepseek-ai/DeepSeek-V4-Pro/mtp.hc_ffn", + "sgl-project/DeepSeek-V4-Pro-FP8/hc_attn", + "sgl-project/DeepSeek-V4-Pro-FP8/hc_ffn", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.hc_attn", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.hc_ffn" + ] + }, + { + "type": "gemm", + "args": { + "m": 1684, + "n": 24, + "k": 40960, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 1684, + "n": 24, + "k": 49152, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 1684, + "n": 64, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.weights_proj", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.weights_proj", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.weights_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 1684, + "n": 64, + "k": 10240, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 1684, + "n": 64, + "k": 12288, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 1684, + "n": 120, + "k": 12288, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 1684, + "n": 192, + "k": 10240, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 1684, + "n": 256, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.compressor.wkv", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.compressor.wkv", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.compressor.wgate", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.compressor.wkv" + ] + }, + { + "type": "gemm", + "args": { + "m": 1684, + "n": 256, + "k": 10240, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 1684, + "n": 256, + "k": 12288, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 1684, + "n": 320, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 1684, + "n": 512, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wkv", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wkv", + "deepseek-ai/DeepSeek-V4-Pro/attn.wkv", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wkv", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wkv", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wkv" + ] + }, + { + "type": "gemm", + "args": { + "m": 1684, + "n": 512, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.compressor.wkv+wgate", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.compressor.wkv+wgate", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.compressor.wkv+wgate" + ] + }, + { + "type": "gemm", + "args": { + "m": 1684, + "n": 512, + "k": 10240, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 1684, + "n": 512, + "k": 10240, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 1684, + "n": 512, + "k": 12288, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 1684, + "n": 512, + "k": 12288, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 1684, + "n": 1024, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.compressor.wkv", + "deepseek-ai/DeepSeek-V4-Pro/attn.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro/attn.compressor.wkv", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.compressor.wgate", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.compressor.wkv" + ] + }, + { + "type": "gemm", + "args": { + "m": 1684, + "n": 1024, + "k": 10240, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 1684, + "n": 1024, + "k": 12288, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 1684, + "n": 1536, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w3" + ] + }, + { + "type": "gemm", + "args": { + "m": 1684, + "n": 1536, + "k": 10240, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 1684, + "n": 1536, + "k": 12288, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 1684, + "n": 4096, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 1684, + "n": 5120, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 1684, + "n": 7168, + "k": 1024, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 1684, + "n": 8192, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 1684, + "n": 10240, + "k": 1024, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 1684, + "n": 12288, + "k": 1024, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 1710, + "n": 384, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.gate", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.gate", + "deepseek-ai/DeepSeek-V4-Pro/ffn.gate", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.gate", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.gate", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.gate" + ] + }, + { + "type": "gemm", + "args": { + "m": 1710, + "n": 6144, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3" + ] + }, + { + "type": "gemm", + "args": { + "m": 1710, + "n": 7168, + "k": 3072, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w2" + ] + }, + { + "type": "gemm", + "args": { + "m": 1710, + "n": 7168, + "k": 16384, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 1710, + "n": 65536, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 1845, + "n": 1792, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 1845, + "n": 7168, + "k": 896, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 1845, + "n": 7168, + "k": 2048, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 1979, + "n": 1024, + "k": 12288, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 1979, + "n": 12288, + "k": 512, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 1994, + "n": 7168, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.e_proj", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.h_proj", + "deepseek-ai/DeepSeek-V4-Pro/mtp.e_proj", + "deepseek-ai/DeepSeek-V4-Pro/mtp.h_proj", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.e_proj", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.h_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 1994, + "n": 8192, + "k": 10240, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 1994, + "n": 10240, + "k": 4096, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 1994, + "n": 10240, + "k": 16384, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 1994, + "n": 14336, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 2335, + "n": 24, + "k": 40960, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 2335, + "n": 64, + "k": 10240, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 2335, + "n": 192, + "k": 10240, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 2335, + "n": 256, + "k": 10240, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 2335, + "n": 512, + "k": 10240, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 2335, + "n": 512, + "k": 10240, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 2335, + "n": 1024, + "k": 10240, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 2335, + "n": 1536, + "k": 10240, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 2335, + "n": 8192, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 2335, + "n": 65536, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 2420, + "n": 24, + "k": 28672, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/hc_attn", + "deepseek-ai/DeepSeek-V4-Pro-0813/hc_ffn", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.hc_attn", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.hc_ffn", + "deepseek-ai/DeepSeek-V4-Pro/hc_attn", + "deepseek-ai/DeepSeek-V4-Pro/hc_ffn", + "deepseek-ai/DeepSeek-V4-Pro/mtp.hc_attn", + "deepseek-ai/DeepSeek-V4-Pro/mtp.hc_ffn", + "sgl-project/DeepSeek-V4-Pro-FP8/hc_attn", + "sgl-project/DeepSeek-V4-Pro-FP8/hc_ffn", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.hc_attn", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.hc_ffn" + ] + }, + { + "type": "gemm", + "args": { + "m": 2420, + "n": 64, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.weights_proj", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.weights_proj", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.weights_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 2420, + "n": 256, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.compressor.wkv", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.compressor.wkv", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.compressor.wgate", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.compressor.wkv" + ] + }, + { + "type": "gemm", + "args": { + "m": 2420, + "n": 320, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 2420, + "n": 384, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.gate", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.gate", + "deepseek-ai/DeepSeek-V4-Pro/ffn.gate", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.gate", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.gate", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.gate" + ] + }, + { + "type": "gemm", + "args": { + "m": 2420, + "n": 384, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w3" + ] + }, + { + "type": "gemm", + "args": { + "m": 2420, + "n": 512, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wkv", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wkv", + "deepseek-ai/DeepSeek-V4-Pro/attn.wkv", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wkv", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wkv", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wkv" + ] + }, + { + "type": "gemm", + "args": { + "m": 2420, + "n": 512, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.compressor.wkv+wgate", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.compressor.wkv+wgate", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.compressor.wkv+wgate" + ] + }, + { + "type": "gemm", + "args": { + "m": 2420, + "n": 896, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 2420, + "n": 1024, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.compressor.wkv", + "deepseek-ai/DeepSeek-V4-Pro/attn.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro/attn.compressor.wkv", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.compressor.wgate", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.compressor.wkv" + ] + }, + { + "type": "gemm", + "args": { + "m": 2420, + "n": 1536, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w3" + ] + }, + { + "type": "gemm", + "args": { + "m": 2420, + "n": 4096, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 2420, + "n": 7168, + "k": 192, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w2" + ] + }, + { + "type": "gemm", + "args": { + "m": 2420, + "n": 7168, + "k": 448, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 2420, + "n": 7168, + "k": 1024, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 2671, + "n": 8192, + "k": 10240, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 2671, + "n": 10240, + "k": 4096, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 2671, + "n": 10240, + "k": 16384, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 2732, + "n": 2048, + "k": 10240, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 2732, + "n": 10240, + "k": 1024, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 2732, + "n": 10240, + "k": 4096, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 2732, + "n": 16384, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 2813, + "n": 24, + "k": 49152, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 2813, + "n": 64, + "k": 12288, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 2813, + "n": 120, + "k": 12288, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 2813, + "n": 256, + "k": 12288, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 2813, + "n": 512, + "k": 12288, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 2813, + "n": 512, + "k": 12288, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 2813, + "n": 768, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w3" + ] + }, + { + "type": "gemm", + "args": { + "m": 2813, + "n": 1024, + "k": 12288, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 2813, + "n": 1536, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w3" + ] + }, + { + "type": "gemm", + "args": { + "m": 2813, + "n": 1536, + "k": 12288, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 2813, + "n": 1792, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 2813, + "n": 5120, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 2813, + "n": 7168, + "k": 384, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w2" + ] + }, + { + "type": "gemm", + "args": { + "m": 2813, + "n": 7168, + "k": 768, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w2" + ] + }, + { + "type": "gemm", + "args": { + "m": 2813, + "n": 7168, + "k": 896, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 2813, + "n": 7168, + "k": 2048, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 2813, + "n": 7168, + "k": 4096, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 2813, + "n": 10240, + "k": 1024, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 2813, + "n": 12288, + "k": 1024, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 3010, + "n": 8192, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 3145, + "n": 24, + "k": 28672, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/hc_attn", + "deepseek-ai/DeepSeek-V4-Pro-0813/hc_ffn", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.hc_attn", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.hc_ffn", + "deepseek-ai/DeepSeek-V4-Pro/hc_attn", + "deepseek-ai/DeepSeek-V4-Pro/hc_ffn", + "deepseek-ai/DeepSeek-V4-Pro/mtp.hc_attn", + "deepseek-ai/DeepSeek-V4-Pro/mtp.hc_ffn", + "sgl-project/DeepSeek-V4-Pro-FP8/hc_attn", + "sgl-project/DeepSeek-V4-Pro-FP8/hc_ffn", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.hc_attn", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.hc_ffn" + ] + }, + { + "type": "gemm", + "args": { + "m": 3145, + "n": 64, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.weights_proj", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.weights_proj", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.weights_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 3145, + "n": 256, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.compressor.wkv", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.compressor.wkv", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.compressor.wgate", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.compressor.wkv" + ] + }, + { + "type": "gemm", + "args": { + "m": 3145, + "n": 384, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.gate", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.gate", + "deepseek-ai/DeepSeek-V4-Pro/ffn.gate", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.gate", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.gate", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.gate" + ] + }, + { + "type": "gemm", + "args": { + "m": 3145, + "n": 512, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wkv", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wkv", + "deepseek-ai/DeepSeek-V4-Pro/attn.wkv", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wkv", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wkv", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wkv" + ] + }, + { + "type": "gemm", + "args": { + "m": 3145, + "n": 512, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.compressor.wkv+wgate", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.compressor.wkv+wgate", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.compressor.wkv+wgate" + ] + }, + { + "type": "gemm", + "args": { + "m": 3145, + "n": 1024, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.compressor.wkv", + "deepseek-ai/DeepSeek-V4-Pro/attn.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro/attn.compressor.wkv", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.compressor.wgate", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.compressor.wkv" + ] + }, + { + "type": "gemm", + "args": { + "m": 3145, + "n": 1536, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w3" + ] + }, + { + "type": "gemm", + "args": { + "m": 3145, + "n": 6144, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3" + ] + }, + { + "type": "gemm", + "args": { + "m": 3145, + "n": 7168, + "k": 3072, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w2" + ] + }, + { + "type": "gemm", + "args": { + "m": 3145, + "n": 7168, + "k": 16384, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 3145, + "n": 65536, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 3161, + "n": 24, + "k": 40960, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 3161, + "n": 64, + "k": 10240, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 3161, + "n": 192, + "k": 10240, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 3161, + "n": 256, + "k": 10240, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 3161, + "n": 320, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 3161, + "n": 512, + "k": 10240, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 3161, + "n": 512, + "k": 10240, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 3161, + "n": 1024, + "k": 10240, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 3161, + "n": 1536, + "k": 10240, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 3161, + "n": 4096, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 3161, + "n": 7168, + "k": 1024, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 3698, + "n": 1792, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 3698, + "n": 7168, + "k": 896, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 3698, + "n": 7168, + "k": 2048, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 3951, + "n": 2048, + "k": 10240, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 3951, + "n": 8192, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 3951, + "n": 10240, + "k": 1024, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 3951, + "n": 10240, + "k": 4096, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 3951, + "n": 16384, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 4049, + "n": 24, + "k": 28672, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/hc_attn", + "deepseek-ai/DeepSeek-V4-Pro-0813/hc_ffn", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.hc_attn", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.hc_ffn", + "deepseek-ai/DeepSeek-V4-Pro/hc_attn", + "deepseek-ai/DeepSeek-V4-Pro/hc_ffn", + "deepseek-ai/DeepSeek-V4-Pro/mtp.hc_attn", + "deepseek-ai/DeepSeek-V4-Pro/mtp.hc_ffn", + "sgl-project/DeepSeek-V4-Pro-FP8/hc_attn", + "sgl-project/DeepSeek-V4-Pro-FP8/hc_ffn", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.hc_attn", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.hc_ffn" + ] + }, + { + "type": "gemm", + "args": { + "m": 4049, + "n": 64, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.weights_proj", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.weights_proj", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.weights_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 4049, + "n": 256, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.compressor.wkv", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.compressor.wkv", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.compressor.wgate", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.compressor.wkv" + ] + }, + { + "type": "gemm", + "args": { + "m": 4049, + "n": 320, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 4049, + "n": 512, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wkv", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wkv", + "deepseek-ai/DeepSeek-V4-Pro/attn.wkv", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wkv", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wkv", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wkv" + ] + }, + { + "type": "gemm", + "args": { + "m": 4049, + "n": 512, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.compressor.wkv+wgate", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.compressor.wkv+wgate", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.compressor.wkv+wgate" + ] + }, + { + "type": "gemm", + "args": { + "m": 4049, + "n": 1024, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.compressor.wkv", + "deepseek-ai/DeepSeek-V4-Pro/attn.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro/attn.compressor.wkv", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.compressor.wgate", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.compressor.wkv" + ] + }, + { + "type": "gemm", + "args": { + "m": 4049, + "n": 1536, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w3" + ] + }, + { + "type": "gemm", + "args": { + "m": 4285, + "n": 384, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.gate", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.gate", + "deepseek-ai/DeepSeek-V4-Pro/ffn.gate", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.gate", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.gate", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.gate" + ] + }, + { + "type": "gemm", + "args": { + "m": 4285, + "n": 384, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w3" + ] + }, + { + "type": "gemm", + "args": { + "m": 4285, + "n": 896, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 4285, + "n": 4096, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 4285, + "n": 7168, + "k": 192, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w2" + ] + }, + { + "type": "gemm", + "args": { + "m": 4285, + "n": 7168, + "k": 448, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 4285, + "n": 7168, + "k": 1024, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 4717, + "n": 24, + "k": 40960, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 4717, + "n": 64, + "k": 10240, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 4717, + "n": 192, + "k": 10240, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 4717, + "n": 256, + "k": 10240, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 4717, + "n": 512, + "k": 10240, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 4717, + "n": 512, + "k": 10240, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 4717, + "n": 1024, + "k": 10240, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 4717, + "n": 1536, + "k": 10240, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 4717, + "n": 10240, + "k": 16384, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 4717, + "n": 65536, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 4734, + "n": 7168, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.e_proj", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.h_proj", + "deepseek-ai/DeepSeek-V4-Pro/mtp.e_proj", + "deepseek-ai/DeepSeek-V4-Pro/mtp.h_proj", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.e_proj", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.h_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 4734, + "n": 7168, + "k": 16384, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 4734, + "n": 14336, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 4810, + "n": 8192, + "k": 10240, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 4810, + "n": 10240, + "k": 4096, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 4868, + "n": 24, + "k": 49152, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 4868, + "n": 64, + "k": 12288, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 4868, + "n": 120, + "k": 12288, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 4868, + "n": 256, + "k": 12288, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 4868, + "n": 512, + "k": 12288, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 4868, + "n": 512, + "k": 12288, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 4868, + "n": 1024, + "k": 12288, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 4868, + "n": 1024, + "k": 12288, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 4868, + "n": 1536, + "k": 12288, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 4868, + "n": 5120, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 4868, + "n": 12288, + "k": 512, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 4868, + "n": 12288, + "k": 1024, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 5144, + "n": 24, + "k": 28672, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/hc_attn", + "deepseek-ai/DeepSeek-V4-Pro-0813/hc_ffn", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.hc_attn", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.hc_ffn", + "deepseek-ai/DeepSeek-V4-Pro/hc_attn", + "deepseek-ai/DeepSeek-V4-Pro/hc_ffn", + "deepseek-ai/DeepSeek-V4-Pro/mtp.hc_attn", + "deepseek-ai/DeepSeek-V4-Pro/mtp.hc_ffn", + "sgl-project/DeepSeek-V4-Pro-FP8/hc_attn", + "sgl-project/DeepSeek-V4-Pro-FP8/hc_ffn", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.hc_attn", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.hc_ffn" + ] + }, + { + "type": "gemm", + "args": { + "m": 5144, + "n": 64, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.weights_proj", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.weights_proj", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.weights_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 5144, + "n": 256, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.compressor.wkv", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.compressor.wkv", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.compressor.wgate", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.compressor.wkv" + ] + }, + { + "type": "gemm", + "args": { + "m": 5144, + "n": 384, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.gate", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.gate", + "deepseek-ai/DeepSeek-V4-Pro/ffn.gate", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.gate", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.gate", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.gate" + ] + }, + { + "type": "gemm", + "args": { + "m": 5144, + "n": 512, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wkv", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wkv", + "deepseek-ai/DeepSeek-V4-Pro/attn.wkv", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wkv", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wkv", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wkv" + ] + }, + { + "type": "gemm", + "args": { + "m": 5144, + "n": 512, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.compressor.wkv+wgate", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.compressor.wkv+wgate", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.compressor.wkv+wgate" + ] + }, + { + "type": "gemm", + "args": { + "m": 5144, + "n": 1024, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.compressor.wkv", + "deepseek-ai/DeepSeek-V4-Pro/attn.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro/attn.compressor.wkv", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.compressor.wgate", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.compressor.wkv" + ] + }, + { + "type": "gemm", + "args": { + "m": 5144, + "n": 1536, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w3" + ] + }, + { + "type": "gemm", + "args": { + "m": 5144, + "n": 6144, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3" + ] + }, + { + "type": "gemm", + "args": { + "m": 5144, + "n": 7168, + "k": 3072, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w2" + ] + }, + { + "type": "gemm", + "args": { + "m": 5144, + "n": 7168, + "k": 16384, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 5144, + "n": 8192, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 5144, + "n": 65536, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 5290, + "n": 24, + "k": 28672, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/hc_attn", + "deepseek-ai/DeepSeek-V4-Pro-0813/hc_ffn", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.hc_attn", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.hc_ffn", + "deepseek-ai/DeepSeek-V4-Pro/hc_attn", + "deepseek-ai/DeepSeek-V4-Pro/hc_ffn", + "deepseek-ai/DeepSeek-V4-Pro/mtp.hc_attn", + "deepseek-ai/DeepSeek-V4-Pro/mtp.hc_ffn", + "sgl-project/DeepSeek-V4-Pro-FP8/hc_attn", + "sgl-project/DeepSeek-V4-Pro-FP8/hc_ffn", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.hc_attn", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.hc_ffn" + ] + }, + { + "type": "gemm", + "args": { + "m": 5290, + "n": 64, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.weights_proj", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.weights_proj", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.weights_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 5290, + "n": 256, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.compressor.wkv", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.compressor.wkv", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.compressor.wgate", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.compressor.wkv" + ] + }, + { + "type": "gemm", + "args": { + "m": 5290, + "n": 320, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 5290, + "n": 512, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wkv", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wkv", + "deepseek-ai/DeepSeek-V4-Pro/attn.wkv", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wkv", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wkv", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wkv" + ] + }, + { + "type": "gemm", + "args": { + "m": 5290, + "n": 512, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.compressor.wkv+wgate", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.compressor.wkv+wgate", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.compressor.wkv+wgate" + ] + }, + { + "type": "gemm", + "args": { + "m": 5290, + "n": 1024, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.compressor.wkv", + "deepseek-ai/DeepSeek-V4-Pro/attn.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro/attn.compressor.wkv", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.compressor.wgate", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.compressor.wkv" + ] + }, + { + "type": "gemm", + "args": { + "m": 5290, + "n": 1536, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w3" + ] + }, + { + "type": "gemm", + "args": { + "m": 5290, + "n": 8192, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 5668, + "n": 384, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.gate", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.gate", + "deepseek-ai/DeepSeek-V4-Pro/ffn.gate", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.gate", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.gate", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.gate" + ] + }, + { + "type": "gemm", + "args": { + "m": 5668, + "n": 4096, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 5668, + "n": 7168, + "k": 1024, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 5668, + "n": 10240, + "k": 1024, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 5709, + "n": 768, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w3" + ] + }, + { + "type": "gemm", + "args": { + "m": 5709, + "n": 7168, + "k": 384, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w2" + ] + }, + { + "type": "gemm", + "args": { + "m": 5709, + "n": 7168, + "k": 2048, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 5891, + "n": 896, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 5891, + "n": 7168, + "k": 448, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 5891, + "n": 7168, + "k": 4096, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 5891, + "n": 10240, + "k": 2048, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 5891, + "n": 16384, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 6042, + "n": 24, + "k": 40960, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 6042, + "n": 64, + "k": 10240, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 6042, + "n": 192, + "k": 10240, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 6042, + "n": 256, + "k": 10240, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 6042, + "n": 512, + "k": 10240, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 6042, + "n": 512, + "k": 10240, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 6042, + "n": 1024, + "k": 10240, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 6042, + "n": 1536, + "k": 10240, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 6042, + "n": 8192, + "k": 10240, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 6042, + "n": 10240, + "k": 4096, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 6042, + "n": 10240, + "k": 16384, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 6042, + "n": 65536, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 6069, + "n": 7168, + "k": 16384, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 6072, + "n": 24, + "k": 40960, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 6072, + "n": 24, + "k": 49152, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 6072, + "n": 64, + "k": 10240, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 6072, + "n": 64, + "k": 12288, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 6072, + "n": 120, + "k": 12288, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 6072, + "n": 192, + "k": 10240, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 6072, + "n": 256, + "k": 10240, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 6072, + "n": 256, + "k": 12288, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 6072, + "n": 384, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.gate", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.gate", + "deepseek-ai/DeepSeek-V4-Pro/ffn.gate", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.gate", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.gate", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.gate" + ] + }, + { + "type": "gemm", + "args": { + "m": 6072, + "n": 512, + "k": 10240, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 6072, + "n": 512, + "k": 10240, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 6072, + "n": 512, + "k": 12288, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 6072, + "n": 512, + "k": 12288, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 6072, + "n": 1024, + "k": 10240, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 6072, + "n": 1024, + "k": 12288, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 6072, + "n": 1536, + "k": 10240, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 6072, + "n": 1536, + "k": 12288, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 6072, + "n": 4096, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 6072, + "n": 5120, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 6072, + "n": 7168, + "k": 1024, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 6072, + "n": 10240, + "k": 1024, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 6072, + "n": 12288, + "k": 1024, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 6078, + "n": 2048, + "k": 10240, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 6078, + "n": 10240, + "k": 1024, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 6078, + "n": 10240, + "k": 4096, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 6515, + "n": 24, + "k": 28672, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/hc_attn", + "deepseek-ai/DeepSeek-V4-Pro-0813/hc_ffn", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.hc_attn", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.hc_ffn", + "deepseek-ai/DeepSeek-V4-Pro/hc_attn", + "deepseek-ai/DeepSeek-V4-Pro/hc_ffn", + "deepseek-ai/DeepSeek-V4-Pro/mtp.hc_attn", + "deepseek-ai/DeepSeek-V4-Pro/mtp.hc_ffn", + "sgl-project/DeepSeek-V4-Pro-FP8/hc_attn", + "sgl-project/DeepSeek-V4-Pro-FP8/hc_ffn", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.hc_attn", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.hc_ffn" + ] + }, + { + "type": "gemm", + "args": { + "m": 6515, + "n": 24, + "k": 49152, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 6515, + "n": 64, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.weights_proj", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.weights_proj", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.weights_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 6515, + "n": 64, + "k": 12288, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 6515, + "n": 120, + "k": 12288, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 6515, + "n": 256, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.compressor.wkv", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.compressor.wkv", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.compressor.wgate", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.compressor.wkv" + ] + }, + { + "type": "gemm", + "args": { + "m": 6515, + "n": 256, + "k": 12288, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 6515, + "n": 512, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wkv", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wkv", + "deepseek-ai/DeepSeek-V4-Pro/attn.wkv", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wkv", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wkv", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wkv" + ] + }, + { + "type": "gemm", + "args": { + "m": 6515, + "n": 512, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.compressor.wkv+wgate", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.compressor.wkv+wgate", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.compressor.wkv+wgate" + ] + }, + { + "type": "gemm", + "args": { + "m": 6515, + "n": 512, + "k": 12288, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 6515, + "n": 512, + "k": 12288, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 6515, + "n": 1024, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.compressor.wkv", + "deepseek-ai/DeepSeek-V4-Pro/attn.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro/attn.compressor.wkv", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.compressor.wgate", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.compressor.wkv" + ] + }, + { + "type": "gemm", + "args": { + "m": 6515, + "n": 1024, + "k": 12288, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 6515, + "n": 1536, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w3" + ] + }, + { + "type": "gemm", + "args": { + "m": 6515, + "n": 1536, + "k": 12288, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 6515, + "n": 5120, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 6515, + "n": 8192, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 6515, + "n": 12288, + "k": 1024, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 6736, + "n": 24, + "k": 28672, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/hc_attn", + "deepseek-ai/DeepSeek-V4-Pro-0813/hc_ffn", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.hc_attn", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.hc_ffn", + "deepseek-ai/DeepSeek-V4-Pro/hc_attn", + "deepseek-ai/DeepSeek-V4-Pro/hc_ffn", + "deepseek-ai/DeepSeek-V4-Pro/mtp.hc_attn", + "deepseek-ai/DeepSeek-V4-Pro/mtp.hc_ffn", + "sgl-project/DeepSeek-V4-Pro-FP8/hc_attn", + "sgl-project/DeepSeek-V4-Pro-FP8/hc_ffn", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.hc_attn", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.hc_ffn" + ] + }, + { + "type": "gemm", + "args": { + "m": 6736, + "n": 64, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.weights_proj", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.weights_proj", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.weights_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 6736, + "n": 256, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.compressor.wkv", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.compressor.wkv", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.compressor.wgate", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.compressor.wkv" + ] + }, + { + "type": "gemm", + "args": { + "m": 6736, + "n": 320, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 6736, + "n": 384, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.gate", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.gate", + "deepseek-ai/DeepSeek-V4-Pro/ffn.gate", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.gate", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.gate", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.gate" + ] + }, + { + "type": "gemm", + "args": { + "m": 6736, + "n": 512, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wkv", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wkv", + "deepseek-ai/DeepSeek-V4-Pro/attn.wkv", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wkv", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wkv", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wkv" + ] + }, + { + "type": "gemm", + "args": { + "m": 6736, + "n": 512, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.compressor.wkv+wgate", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.compressor.wkv+wgate", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.compressor.wkv+wgate" + ] + }, + { + "type": "gemm", + "args": { + "m": 6736, + "n": 1024, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.compressor.wkv", + "deepseek-ai/DeepSeek-V4-Pro/attn.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro/attn.compressor.wkv", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.compressor.wgate", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.compressor.wkv" + ] + }, + { + "type": "gemm", + "args": { + "m": 6736, + "n": 1536, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w3" + ] + }, + { + "type": "gemm", + "args": { + "m": 6736, + "n": 6144, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3" + ] + }, + { + "type": "gemm", + "args": { + "m": 6736, + "n": 7168, + "k": 3072, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w2" + ] + }, + { + "type": "gemm", + "args": { + "m": 6736, + "n": 7168, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.e_proj", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.h_proj", + "deepseek-ai/DeepSeek-V4-Pro/mtp.e_proj", + "deepseek-ai/DeepSeek-V4-Pro/mtp.h_proj", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.e_proj", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.h_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 6736, + "n": 7168, + "k": 16384, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 6736, + "n": 8192, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 6736, + "n": 14336, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 6736, + "n": 65536, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 6830, + "n": 320, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 6830, + "n": 1792, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 6830, + "n": 7168, + "k": 896, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 6837, + "n": 6144, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3" + ] + }, + { + "type": "gemm", + "args": { + "m": 6837, + "n": 7168, + "k": 3072, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w2" + ] + }, + { + "type": "gemm", + "args": { + "m": 7112, + "n": 24, + "k": 28672, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/hc_attn", + "deepseek-ai/DeepSeek-V4-Pro-0813/hc_ffn", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.hc_attn", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.hc_ffn", + "deepseek-ai/DeepSeek-V4-Pro/hc_attn", + "deepseek-ai/DeepSeek-V4-Pro/hc_ffn", + "deepseek-ai/DeepSeek-V4-Pro/mtp.hc_attn", + "deepseek-ai/DeepSeek-V4-Pro/mtp.hc_ffn", + "sgl-project/DeepSeek-V4-Pro-FP8/hc_attn", + "sgl-project/DeepSeek-V4-Pro-FP8/hc_ffn", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.hc_attn", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.hc_ffn" + ] + }, + { + "type": "gemm", + "args": { + "m": 7112, + "n": 24, + "k": 49152, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 7112, + "n": 64, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.weights_proj", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.weights_proj", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.weights_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 7112, + "n": 64, + "k": 12288, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 7112, + "n": 120, + "k": 12288, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 7112, + "n": 256, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.compressor.wkv", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.compressor.wkv", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.compressor.wgate", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.compressor.wkv" + ] + }, + { + "type": "gemm", + "args": { + "m": 7112, + "n": 256, + "k": 12288, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 7112, + "n": 320, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 7112, + "n": 384, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.gate", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.gate", + "deepseek-ai/DeepSeek-V4-Pro/ffn.gate", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.gate", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.gate", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.gate" + ] + }, + { + "type": "gemm", + "args": { + "m": 7112, + "n": 384, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w3" + ] + }, + { + "type": "gemm", + "args": { + "m": 7112, + "n": 512, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wkv", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wkv", + "deepseek-ai/DeepSeek-V4-Pro/attn.wkv", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wkv", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wkv", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wkv" + ] + }, + { + "type": "gemm", + "args": { + "m": 7112, + "n": 512, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.compressor.wkv+wgate", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.compressor.wkv+wgate", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.compressor.wkv+wgate" + ] + }, + { + "type": "gemm", + "args": { + "m": 7112, + "n": 512, + "k": 12288, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 7112, + "n": 512, + "k": 12288, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 7112, + "n": 768, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w3" + ] + }, + { + "type": "gemm", + "args": { + "m": 7112, + "n": 896, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 7112, + "n": 1024, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.compressor.wkv", + "deepseek-ai/DeepSeek-V4-Pro/attn.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro/attn.compressor.wkv", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.compressor.wgate", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.compressor.wkv" + ] + }, + { + "type": "gemm", + "args": { + "m": 7112, + "n": 1024, + "k": 12288, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 7112, + "n": 1024, + "k": 12288, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 7112, + "n": 1536, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w3" + ] + }, + { + "type": "gemm", + "args": { + "m": 7112, + "n": 1536, + "k": 12288, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 7112, + "n": 1792, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 7112, + "n": 2048, + "k": 10240, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 7112, + "n": 4096, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 7112, + "n": 5120, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 7112, + "n": 6144, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3" + ] + }, + { + "type": "gemm", + "args": { + "m": 7112, + "n": 7168, + "k": 192, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w2" + ] + }, + { + "type": "gemm", + "args": { + "m": 7112, + "n": 7168, + "k": 384, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w2" + ] + }, + { + "type": "gemm", + "args": { + "m": 7112, + "n": 7168, + "k": 448, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 7112, + "n": 7168, + "k": 896, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 7112, + "n": 7168, + "k": 1024, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 7112, + "n": 7168, + "k": 2048, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 7112, + "n": 7168, + "k": 3072, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w2" + ] + }, + { + "type": "gemm", + "args": { + "m": 7112, + "n": 7168, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.e_proj", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.h_proj", + "deepseek-ai/DeepSeek-V4-Pro/mtp.e_proj", + "deepseek-ai/DeepSeek-V4-Pro/mtp.h_proj", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.e_proj", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.h_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 7112, + "n": 7168, + "k": 16384, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 7112, + "n": 8192, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 7112, + "n": 8192, + "k": 10240, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 7112, + "n": 10240, + "k": 1024, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 7112, + "n": 10240, + "k": 4096, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 7112, + "n": 10240, + "k": 16384, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 7112, + "n": 12288, + "k": 512, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 7112, + "n": 12288, + "k": 1024, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 7112, + "n": 12288, + "k": 8192, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 7112, + "n": 12288, + "k": 16384, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 7112, + "n": 14336, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 7112, + "n": 16384, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 7112, + "n": 16384, + "k": 12288, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 7112, + "n": 65536, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 7112, + "n": 81920, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 7113, + "n": 4096, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 7113, + "n": 7168, + "k": 1024, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 7152, + "n": 24, + "k": 40960, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 7152, + "n": 64, + "k": 10240, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 7152, + "n": 192, + "k": 10240, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 7152, + "n": 256, + "k": 10240, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 7152, + "n": 384, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.gate", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.gate", + "deepseek-ai/DeepSeek-V4-Pro/ffn.gate", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.gate", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.gate", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.gate" + ] + }, + { + "type": "gemm", + "args": { + "m": 7152, + "n": 512, + "k": 10240, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 7152, + "n": 512, + "k": 10240, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 7152, + "n": 768, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w3" + ] + }, + { + "type": "gemm", + "args": { + "m": 7152, + "n": 1024, + "k": 10240, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 7152, + "n": 1536, + "k": 10240, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 7152, + "n": 1792, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 7152, + "n": 7168, + "k": 384, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w2" + ] + }, + { + "type": "gemm", + "args": { + "m": 7152, + "n": 7168, + "k": 896, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 7152, + "n": 7168, + "k": 2048, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 7152, + "n": 10240, + "k": 16384, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 7154, + "n": 384, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.gate", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.gate", + "deepseek-ai/DeepSeek-V4-Pro/ffn.gate", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.gate", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.gate", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.gate" + ] + }, + { + "type": "gemm", + "args": { + "m": 7154, + "n": 768, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w3" + ] + }, + { + "type": "gemm", + "args": { + "m": 7154, + "n": 7168, + "k": 384, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w2" + ] + }, + { + "type": "gemm", + "args": { + "m": 7154, + "n": 7168, + "k": 2048, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 7237, + "n": 384, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w3" + ] + }, + { + "type": "gemm", + "args": { + "m": 7237, + "n": 1024, + "k": 10240, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 7237, + "n": 1024, + "k": 12288, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 7237, + "n": 1536, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w3" + ] + }, + { + "type": "gemm", + "args": { + "m": 7237, + "n": 2048, + "k": 12288, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 7237, + "n": 6144, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3" + ] + }, + { + "type": "gemm", + "args": { + "m": 7237, + "n": 7168, + "k": 192, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w2" + ] + }, + { + "type": "gemm", + "args": { + "m": 7237, + "n": 7168, + "k": 768, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w2" + ] + }, + { + "type": "gemm", + "args": { + "m": 7237, + "n": 7168, + "k": 3072, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w2" + ] + }, + { + "type": "gemm", + "args": { + "m": 7237, + "n": 7168, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.e_proj", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.h_proj", + "deepseek-ai/DeepSeek-V4-Pro/mtp.e_proj", + "deepseek-ai/DeepSeek-V4-Pro/mtp.h_proj", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.e_proj", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.h_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 7237, + "n": 10240, + "k": 512, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 7237, + "n": 10240, + "k": 1024, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 7237, + "n": 10240, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 7237, + "n": 12288, + "k": 512, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 7237, + "n": 12288, + "k": 1024, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 7237, + "n": 12288, + "k": 2048, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 7237, + "n": 12288, + "k": 8192, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 7237, + "n": 12288, + "k": 16384, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 7237, + "n": 14336, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 7237, + "n": 16384, + "k": 12288, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 7237, + "n": 81920, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 7388, + "n": 896, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 7388, + "n": 7168, + "k": 448, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 7586, + "n": 24, + "k": 40960, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 7586, + "n": 64, + "k": 10240, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 7586, + "n": 192, + "k": 10240, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 7586, + "n": 256, + "k": 10240, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 7586, + "n": 512, + "k": 10240, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 7586, + "n": 512, + "k": 10240, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 7586, + "n": 1024, + "k": 10240, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 7586, + "n": 1536, + "k": 10240, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 7586, + "n": 10240, + "k": 16384, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 7586, + "n": 65536, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 7647, + "n": 7168, + "k": 16384, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 7648, + "n": 24, + "k": 28672, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/hc_attn", + "deepseek-ai/DeepSeek-V4-Pro-0813/hc_ffn", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.hc_attn", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.hc_ffn", + "deepseek-ai/DeepSeek-V4-Pro/hc_attn", + "deepseek-ai/DeepSeek-V4-Pro/hc_ffn", + "deepseek-ai/DeepSeek-V4-Pro/mtp.hc_attn", + "deepseek-ai/DeepSeek-V4-Pro/mtp.hc_ffn", + "sgl-project/DeepSeek-V4-Pro-FP8/hc_attn", + "sgl-project/DeepSeek-V4-Pro-FP8/hc_ffn", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.hc_attn", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.hc_ffn" + ] + }, + { + "type": "gemm", + "args": { + "m": 7648, + "n": 64, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.weights_proj", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.weights_proj", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.weights_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 7648, + "n": 256, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.compressor.wkv", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.compressor.wkv", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.compressor.wgate", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.compressor.wkv" + ] + }, + { + "type": "gemm", + "args": { + "m": 7648, + "n": 384, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.gate", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.gate", + "deepseek-ai/DeepSeek-V4-Pro/ffn.gate", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.gate", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.gate", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.gate" + ] + }, + { + "type": "gemm", + "args": { + "m": 7648, + "n": 512, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wkv", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wkv", + "deepseek-ai/DeepSeek-V4-Pro/attn.wkv", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wkv", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wkv", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wkv" + ] + }, + { + "type": "gemm", + "args": { + "m": 7648, + "n": 512, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.compressor.wkv+wgate", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.compressor.wkv+wgate", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.compressor.wkv+wgate" + ] + }, + { + "type": "gemm", + "args": { + "m": 7648, + "n": 768, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w3" + ] + }, + { + "type": "gemm", + "args": { + "m": 7648, + "n": 1024, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.compressor.wkv", + "deepseek-ai/DeepSeek-V4-Pro/attn.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro/attn.compressor.wkv", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.compressor.wgate", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.compressor.wkv" + ] + }, + { + "type": "gemm", + "args": { + "m": 7648, + "n": 1536, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w3" + ] + }, + { + "type": "gemm", + "args": { + "m": 7648, + "n": 7168, + "k": 384, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w2" + ] + }, + { + "type": "gemm", + "args": { + "m": 7648, + "n": 7168, + "k": 2048, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 7648, + "n": 8192, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 7769, + "n": 8192, + "k": 10240, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 7769, + "n": 10240, + "k": 4096, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 7936, + "n": 24, + "k": 28672, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/hc_attn", + "deepseek-ai/DeepSeek-V4-Pro-0813/hc_ffn", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.hc_attn", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.hc_ffn", + "deepseek-ai/DeepSeek-V4-Pro/hc_attn", + "deepseek-ai/DeepSeek-V4-Pro/hc_ffn", + "deepseek-ai/DeepSeek-V4-Pro/mtp.hc_attn", + "deepseek-ai/DeepSeek-V4-Pro/mtp.hc_ffn", + "sgl-project/DeepSeek-V4-Pro-FP8/hc_attn", + "sgl-project/DeepSeek-V4-Pro-FP8/hc_ffn", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.hc_attn", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.hc_ffn" + ] + }, + { + "type": "gemm", + "args": { + "m": 7936, + "n": 24, + "k": 40960, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 7936, + "n": 24, + "k": 49152, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 7936, + "n": 64, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.weights_proj", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.weights_proj", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.weights_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 7936, + "n": 64, + "k": 10240, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 7936, + "n": 64, + "k": 12288, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 7936, + "n": 120, + "k": 12288, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 7936, + "n": 192, + "k": 10240, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 7936, + "n": 256, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.compressor.wkv", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.compressor.wkv", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.compressor.wgate", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.compressor.wkv" + ] + }, + { + "type": "gemm", + "args": { + "m": 7936, + "n": 256, + "k": 10240, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 7936, + "n": 256, + "k": 12288, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 7936, + "n": 320, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 7936, + "n": 384, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.gate", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.gate", + "deepseek-ai/DeepSeek-V4-Pro/ffn.gate", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.gate", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.gate", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.gate" + ] + }, + { + "type": "gemm", + "args": { + "m": 7936, + "n": 512, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wkv", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wkv", + "deepseek-ai/DeepSeek-V4-Pro/attn.wkv", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wkv", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wkv", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wkv" + ] + }, + { + "type": "gemm", + "args": { + "m": 7936, + "n": 512, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.compressor.wkv+wgate", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.compressor.wkv+wgate", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.compressor.wkv+wgate" + ] + }, + { + "type": "gemm", + "args": { + "m": 7936, + "n": 512, + "k": 10240, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 7936, + "n": 512, + "k": 10240, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 7936, + "n": 512, + "k": 12288, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 7936, + "n": 512, + "k": 12288, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 7936, + "n": 768, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w3" + ] + }, + { + "type": "gemm", + "args": { + "m": 7936, + "n": 1024, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.compressor.wkv", + "deepseek-ai/DeepSeek-V4-Pro/attn.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro/attn.compressor.wkv", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.compressor.wgate", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.compressor.wkv" + ] + }, + { + "type": "gemm", + "args": { + "m": 7936, + "n": 1024, + "k": 10240, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 7936, + "n": 1024, + "k": 12288, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 7936, + "n": 1536, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w3" + ] + }, + { + "type": "gemm", + "args": { + "m": 7936, + "n": 1536, + "k": 10240, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 7936, + "n": 1536, + "k": 12288, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 7936, + "n": 1792, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 7936, + "n": 7168, + "k": 384, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w2" + ] + }, + { + "type": "gemm", + "args": { + "m": 7936, + "n": 7168, + "k": 896, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 7936, + "n": 7168, + "k": 2048, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 7936, + "n": 7168, + "k": 16384, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 7936, + "n": 8192, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 7936, + "n": 8192, + "k": 10240, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 7936, + "n": 10240, + "k": 4096, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 7936, + "n": 10240, + "k": 16384, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 7936, + "n": 12288, + "k": 8192, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 7936, + "n": 12288, + "k": 16384, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 7936, + "n": 16384, + "k": 12288, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 7936, + "n": 65536, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 7936, + "n": 81920, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 8064, + "n": 24, + "k": 28672, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/hc_attn", + "deepseek-ai/DeepSeek-V4-Pro-0813/hc_ffn", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.hc_attn", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.hc_ffn", + "deepseek-ai/DeepSeek-V4-Pro/hc_attn", + "deepseek-ai/DeepSeek-V4-Pro/hc_ffn", + "deepseek-ai/DeepSeek-V4-Pro/mtp.hc_attn", + "deepseek-ai/DeepSeek-V4-Pro/mtp.hc_ffn", + "sgl-project/DeepSeek-V4-Pro-FP8/hc_attn", + "sgl-project/DeepSeek-V4-Pro-FP8/hc_ffn", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.hc_attn", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.hc_ffn" + ] + }, + { + "type": "gemm", + "args": { + "m": 8064, + "n": 64, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.weights_proj", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.weights_proj", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.weights_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 8064, + "n": 256, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.compressor.wkv", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.compressor.wkv", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.compressor.wgate", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.compressor.wkv" + ] + }, + { + "type": "gemm", + "args": { + "m": 8064, + "n": 384, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.gate", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.gate", + "deepseek-ai/DeepSeek-V4-Pro/ffn.gate", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.gate", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.gate", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.gate" + ] + }, + { + "type": "gemm", + "args": { + "m": 8064, + "n": 512, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wkv", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wkv", + "deepseek-ai/DeepSeek-V4-Pro/attn.wkv", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wkv", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wkv", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wkv" + ] + }, + { + "type": "gemm", + "args": { + "m": 8064, + "n": 512, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.compressor.wkv+wgate", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.compressor.wkv+wgate", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.compressor.wkv+wgate" + ] + }, + { + "type": "gemm", + "args": { + "m": 8064, + "n": 768, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w3" + ] + }, + { + "type": "gemm", + "args": { + "m": 8064, + "n": 1024, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.compressor.wkv", + "deepseek-ai/DeepSeek-V4-Pro/attn.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro/attn.compressor.wkv", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.compressor.wgate", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.compressor.wkv" + ] + }, + { + "type": "gemm", + "args": { + "m": 8064, + "n": 1536, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w3" + ] + }, + { + "type": "gemm", + "args": { + "m": 8064, + "n": 7168, + "k": 384, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w2" + ] + }, + { + "type": "gemm", + "args": { + "m": 8064, + "n": 7168, + "k": 2048, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 8064, + "n": 8192, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 8177, + "n": 24, + "k": 28672, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/hc_attn", + "deepseek-ai/DeepSeek-V4-Pro-0813/hc_ffn", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.hc_attn", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.hc_ffn", + "deepseek-ai/DeepSeek-V4-Pro/hc_attn", + "deepseek-ai/DeepSeek-V4-Pro/hc_ffn", + "deepseek-ai/DeepSeek-V4-Pro/mtp.hc_attn", + "deepseek-ai/DeepSeek-V4-Pro/mtp.hc_ffn", + "sgl-project/DeepSeek-V4-Pro-FP8/hc_attn", + "sgl-project/DeepSeek-V4-Pro-FP8/hc_ffn", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.hc_attn", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.hc_ffn" + ] + }, + { + "type": "gemm", + "args": { + "m": 8177, + "n": 24, + "k": 49152, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 8177, + "n": 64, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.weights_proj", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.weights_proj", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.weights_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 8177, + "n": 64, + "k": 12288, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 8177, + "n": 120, + "k": 12288, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 8177, + "n": 256, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.compressor.wkv", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.compressor.wkv", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.compressor.wgate", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.compressor.wkv" + ] + }, + { + "type": "gemm", + "args": { + "m": 8177, + "n": 256, + "k": 12288, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 8177, + "n": 512, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wkv", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wkv", + "deepseek-ai/DeepSeek-V4-Pro/attn.wkv", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wkv", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wkv", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wkv" + ] + }, + { + "type": "gemm", + "args": { + "m": 8177, + "n": 512, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.compressor.wkv+wgate", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.compressor.wkv+wgate", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.compressor.wkv+wgate" + ] + }, + { + "type": "gemm", + "args": { + "m": 8177, + "n": 512, + "k": 12288, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 8177, + "n": 512, + "k": 12288, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 8177, + "n": 1024, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.compressor.wkv", + "deepseek-ai/DeepSeek-V4-Pro/attn.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro/attn.compressor.wkv", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.compressor.wgate", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.compressor.wkv" + ] + }, + { + "type": "gemm", + "args": { + "m": 8177, + "n": 1024, + "k": 12288, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 8177, + "n": 1536, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w3" + ] + }, + { + "type": "gemm", + "args": { + "m": 8177, + "n": 1536, + "k": 12288, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 8177, + "n": 8192, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 8192, + "n": 24, + "k": 28672, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/hc_attn", + "deepseek-ai/DeepSeek-V4-Pro-0813/hc_ffn", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.hc_attn", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.hc_ffn", + "deepseek-ai/DeepSeek-V4-Pro/hc_attn", + "deepseek-ai/DeepSeek-V4-Pro/hc_ffn", + "deepseek-ai/DeepSeek-V4-Pro/mtp.hc_attn", + "deepseek-ai/DeepSeek-V4-Pro/mtp.hc_ffn", + "sgl-project/DeepSeek-V4-Pro-FP8/hc_attn", + "sgl-project/DeepSeek-V4-Pro-FP8/hc_ffn", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.hc_attn", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.hc_ffn" + ] + }, + { + "type": "gemm", + "args": { + "m": 8192, + "n": 24, + "k": 49152, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 8192, + "n": 64, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.weights_proj", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.weights_proj", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.weights_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 8192, + "n": 64, + "k": 12288, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 8192, + "n": 120, + "k": 12288, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 8192, + "n": 256, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.compressor.wkv", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.compressor.wkv", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.compressor.wgate", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.compressor.wkv" + ] + }, + { + "type": "gemm", + "args": { + "m": 8192, + "n": 256, + "k": 12288, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 8192, + "n": 384, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w3" + ] + }, + { + "type": "gemm", + "args": { + "m": 8192, + "n": 512, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wkv", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wkv", + "deepseek-ai/DeepSeek-V4-Pro/attn.wkv", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wkv", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wkv", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wkv" + ] + }, + { + "type": "gemm", + "args": { + "m": 8192, + "n": 512, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.compressor.wkv+wgate", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.compressor.wkv+wgate", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.compressor.wkv+wgate" + ] + }, + { + "type": "gemm", + "args": { + "m": 8192, + "n": 512, + "k": 12288, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 8192, + "n": 512, + "k": 12288, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 8192, + "n": 896, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 8192, + "n": 1024, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.compressor.wkv", + "deepseek-ai/DeepSeek-V4-Pro/attn.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro/attn.compressor.wkv", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.compressor.wgate", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.compressor.wkv" + ] + }, + { + "type": "gemm", + "args": { + "m": 8192, + "n": 1024, + "k": 12288, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 8192, + "n": 1536, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w3" + ] + }, + { + "type": "gemm", + "args": { + "m": 8192, + "n": 1536, + "k": 12288, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 8192, + "n": 4096, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 8192, + "n": 5120, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 8192, + "n": 6144, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3" + ] + }, + { + "type": "gemm", + "args": { + "m": 8192, + "n": 7168, + "k": 192, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w2" + ] + }, + { + "type": "gemm", + "args": { + "m": 8192, + "n": 7168, + "k": 448, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 8192, + "n": 7168, + "k": 768, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w2" + ] + }, + { + "type": "gemm", + "args": { + "m": 8192, + "n": 7168, + "k": 1024, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 8192, + "n": 7168, + "k": 3072, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w2" + ] + }, + { + "type": "gemm", + "args": { + "m": 8192, + "n": 7168, + "k": 4096, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 8192, + "n": 8192, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 8192, + "n": 10240, + "k": 1024, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 8192, + "n": 10240, + "k": 2048, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 8192, + "n": 12288, + "k": 1024, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 8192, + "n": 16384, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 9446, + "n": 320, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 9446, + "n": 12288, + "k": 16384, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 9446, + "n": 81920, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 10010, + "n": 384, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.gate", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.gate", + "deepseek-ai/DeepSeek-V4-Pro/ffn.gate", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.gate", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.gate", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.gate" + ] + }, + { + "type": "gemm", + "args": { + "m": 10010, + "n": 6144, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3" + ] + }, + { + "type": "gemm", + "args": { + "m": 10010, + "n": 7168, + "k": 3072, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w2" + ] + }, + { + "type": "gemm", + "args": { + "m": 10010, + "n": 7168, + "k": 16384, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 10010, + "n": 65536, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 10128, + "n": 24, + "k": 28672, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/hc_attn", + "deepseek-ai/DeepSeek-V4-Pro-0813/hc_ffn", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.hc_attn", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.hc_ffn", + "deepseek-ai/DeepSeek-V4-Pro/hc_attn", + "deepseek-ai/DeepSeek-V4-Pro/hc_ffn", + "deepseek-ai/DeepSeek-V4-Pro/mtp.hc_attn", + "deepseek-ai/DeepSeek-V4-Pro/mtp.hc_ffn", + "sgl-project/DeepSeek-V4-Pro-FP8/hc_attn", + "sgl-project/DeepSeek-V4-Pro-FP8/hc_ffn", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.hc_attn", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.hc_ffn" + ] + }, + { + "type": "gemm", + "args": { + "m": 10128, + "n": 64, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.weights_proj", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.weights_proj", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.weights_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 10128, + "n": 256, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.compressor.wkv", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.compressor.wkv", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.compressor.wgate", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.compressor.wkv" + ] + }, + { + "type": "gemm", + "args": { + "m": 10128, + "n": 320, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 10128, + "n": 512, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wkv", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wkv", + "deepseek-ai/DeepSeek-V4-Pro/attn.wkv", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wkv", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wkv", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wkv" + ] + }, + { + "type": "gemm", + "args": { + "m": 10128, + "n": 512, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.compressor.wkv+wgate", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.compressor.wkv+wgate", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.compressor.wkv+wgate" + ] + }, + { + "type": "gemm", + "args": { + "m": 10128, + "n": 1024, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.compressor.wkv", + "deepseek-ai/DeepSeek-V4-Pro/attn.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro/attn.compressor.wkv", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.compressor.wgate", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.compressor.wkv" + ] + }, + { + "type": "gemm", + "args": { + "m": 10128, + "n": 1536, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w3" + ] + }, + { + "type": "gemm", + "args": { + "m": 10128, + "n": 7168, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.e_proj", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.h_proj", + "deepseek-ai/DeepSeek-V4-Pro/mtp.e_proj", + "deepseek-ai/DeepSeek-V4-Pro/mtp.h_proj", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.e_proj", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.h_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 10128, + "n": 7168, + "k": 16384, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 10128, + "n": 8192, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 10128, + "n": 14336, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 10128, + "n": 65536, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 10236, + "n": 24, + "k": 28672, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/hc_attn", + "deepseek-ai/DeepSeek-V4-Pro-0813/hc_ffn", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.hc_attn", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.hc_ffn", + "deepseek-ai/DeepSeek-V4-Pro/hc_attn", + "deepseek-ai/DeepSeek-V4-Pro/hc_ffn", + "deepseek-ai/DeepSeek-V4-Pro/mtp.hc_attn", + "deepseek-ai/DeepSeek-V4-Pro/mtp.hc_ffn", + "sgl-project/DeepSeek-V4-Pro-FP8/hc_attn", + "sgl-project/DeepSeek-V4-Pro-FP8/hc_ffn", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.hc_attn", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.hc_ffn" + ] + }, + { + "type": "gemm", + "args": { + "m": 10236, + "n": 24, + "k": 40960, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 10236, + "n": 64, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.weights_proj", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.weights_proj", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.weights_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 10236, + "n": 64, + "k": 10240, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 10236, + "n": 192, + "k": 10240, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 10236, + "n": 256, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.compressor.wkv", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.compressor.wkv", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.compressor.wgate", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.compressor.wkv" + ] + }, + { + "type": "gemm", + "args": { + "m": 10236, + "n": 256, + "k": 10240, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 10236, + "n": 512, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wkv", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wkv", + "deepseek-ai/DeepSeek-V4-Pro/attn.wkv", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wkv", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wkv", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wkv" + ] + }, + { + "type": "gemm", + "args": { + "m": 10236, + "n": 512, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.compressor.wkv+wgate", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.compressor.wkv+wgate", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.compressor.wkv+wgate" + ] + }, + { + "type": "gemm", + "args": { + "m": 10236, + "n": 512, + "k": 10240, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 10236, + "n": 512, + "k": 10240, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 10236, + "n": 1024, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.compressor.wkv", + "deepseek-ai/DeepSeek-V4-Pro/attn.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro/attn.compressor.wkv", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.compressor.wgate", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.compressor.wkv" + ] + }, + { + "type": "gemm", + "args": { + "m": 10236, + "n": 1024, + "k": 10240, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 10236, + "n": 1536, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w3" + ] + }, + { + "type": "gemm", + "args": { + "m": 10236, + "n": 1536, + "k": 10240, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 10236, + "n": 8192, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 10236, + "n": 10240, + "k": 16384, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 10442, + "n": 24, + "k": 49152, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 10442, + "n": 64, + "k": 12288, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 10442, + "n": 120, + "k": 12288, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 10442, + "n": 256, + "k": 12288, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 10442, + "n": 512, + "k": 12288, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 10442, + "n": 512, + "k": 12288, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 10442, + "n": 1024, + "k": 12288, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 10442, + "n": 1536, + "k": 12288, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 10442, + "n": 5120, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 10442, + "n": 12288, + "k": 1024, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 10776, + "n": 24, + "k": 28672, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/hc_attn", + "deepseek-ai/DeepSeek-V4-Pro-0813/hc_ffn", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.hc_attn", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.hc_ffn", + "deepseek-ai/DeepSeek-V4-Pro/hc_attn", + "deepseek-ai/DeepSeek-V4-Pro/hc_ffn", + "deepseek-ai/DeepSeek-V4-Pro/mtp.hc_attn", + "deepseek-ai/DeepSeek-V4-Pro/mtp.hc_ffn", + "sgl-project/DeepSeek-V4-Pro-FP8/hc_attn", + "sgl-project/DeepSeek-V4-Pro-FP8/hc_ffn", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.hc_attn", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.hc_ffn" + ] + }, + { + "type": "gemm", + "args": { + "m": 10776, + "n": 24, + "k": 49152, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 10776, + "n": 64, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.weights_proj", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.weights_proj", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.weights_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 10776, + "n": 64, + "k": 12288, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 10776, + "n": 120, + "k": 12288, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 10776, + "n": 256, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.compressor.wkv", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.compressor.wkv", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.compressor.wgate", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.compressor.wkv" + ] + }, + { + "type": "gemm", + "args": { + "m": 10776, + "n": 256, + "k": 12288, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 10776, + "n": 384, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.gate", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.gate", + "deepseek-ai/DeepSeek-V4-Pro/ffn.gate", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.gate", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.gate", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.gate" + ] + }, + { + "type": "gemm", + "args": { + "m": 10776, + "n": 512, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wkv", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wkv", + "deepseek-ai/DeepSeek-V4-Pro/attn.wkv", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wkv", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wkv", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wkv" + ] + }, + { + "type": "gemm", + "args": { + "m": 10776, + "n": 512, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.compressor.wkv+wgate", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.compressor.wkv+wgate", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.compressor.wkv+wgate" + ] + }, + { + "type": "gemm", + "args": { + "m": 10776, + "n": 512, + "k": 12288, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 10776, + "n": 512, + "k": 12288, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 10776, + "n": 1024, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.compressor.wkv", + "deepseek-ai/DeepSeek-V4-Pro/attn.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro/attn.compressor.wkv", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.compressor.wgate", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.compressor.wkv" + ] + }, + { + "type": "gemm", + "args": { + "m": 10776, + "n": 1024, + "k": 12288, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 10776, + "n": 1536, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w3" + ] + }, + { + "type": "gemm", + "args": { + "m": 10776, + "n": 1536, + "k": 12288, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 10776, + "n": 7168, + "k": 16384, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 10776, + "n": 8192, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 10776, + "n": 12288, + "k": 16384, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 10776, + "n": 65536, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 10776, + "n": 81920, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 11068, + "n": 7168, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.e_proj", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.h_proj", + "deepseek-ai/DeepSeek-V4-Pro/mtp.e_proj", + "deepseek-ai/DeepSeek-V4-Pro/mtp.h_proj", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.e_proj", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.h_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 11068, + "n": 8192, + "k": 10240, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 11068, + "n": 10240, + "k": 4096, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 11068, + "n": 14336, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 11304, + "n": 24, + "k": 40960, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 11304, + "n": 64, + "k": 10240, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 11304, + "n": 192, + "k": 10240, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 11304, + "n": 256, + "k": 10240, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 11304, + "n": 512, + "k": 10240, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 11304, + "n": 512, + "k": 10240, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 11304, + "n": 1024, + "k": 10240, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 11304, + "n": 1536, + "k": 10240, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 11304, + "n": 10240, + "k": 16384, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 11724, + "n": 12288, + "k": 8192, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 11724, + "n": 16384, + "k": 12288, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 11872, + "n": 24, + "k": 28672, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/hc_attn", + "deepseek-ai/DeepSeek-V4-Pro-0813/hc_ffn", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.hc_attn", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.hc_ffn", + "deepseek-ai/DeepSeek-V4-Pro/hc_attn", + "deepseek-ai/DeepSeek-V4-Pro/hc_ffn", + "deepseek-ai/DeepSeek-V4-Pro/mtp.hc_attn", + "deepseek-ai/DeepSeek-V4-Pro/mtp.hc_ffn", + "sgl-project/DeepSeek-V4-Pro-FP8/hc_attn", + "sgl-project/DeepSeek-V4-Pro-FP8/hc_ffn", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.hc_attn", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.hc_ffn" + ] + }, + { + "type": "gemm", + "args": { + "m": 11872, + "n": 24, + "k": 40960, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 11872, + "n": 64, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.weights_proj", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.weights_proj", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.weights_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 11872, + "n": 64, + "k": 10240, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 11872, + "n": 192, + "k": 10240, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 11872, + "n": 256, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.compressor.wkv", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.compressor.wkv", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.compressor.wgate", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.compressor.wkv" + ] + }, + { + "type": "gemm", + "args": { + "m": 11872, + "n": 256, + "k": 10240, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 11872, + "n": 384, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.gate", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.gate", + "deepseek-ai/DeepSeek-V4-Pro/ffn.gate", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.gate", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.gate", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.gate" + ] + }, + { + "type": "gemm", + "args": { + "m": 11872, + "n": 512, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wkv", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wkv", + "deepseek-ai/DeepSeek-V4-Pro/attn.wkv", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wkv", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wkv", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wkv" + ] + }, + { + "type": "gemm", + "args": { + "m": 11872, + "n": 512, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.compressor.wkv+wgate", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.compressor.wkv+wgate", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.compressor.wkv+wgate" + ] + }, + { + "type": "gemm", + "args": { + "m": 11872, + "n": 512, + "k": 10240, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 11872, + "n": 512, + "k": 10240, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 11872, + "n": 1024, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.compressor.wkv", + "deepseek-ai/DeepSeek-V4-Pro/attn.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro/attn.compressor.wkv", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.compressor.wgate", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.compressor.wkv" + ] + }, + { + "type": "gemm", + "args": { + "m": 11872, + "n": 1024, + "k": 10240, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 11872, + "n": 1536, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w3" + ] + }, + { + "type": "gemm", + "args": { + "m": 11872, + "n": 1536, + "k": 10240, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 11872, + "n": 7168, + "k": 16384, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 11872, + "n": 8192, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 11872, + "n": 10240, + "k": 16384, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 11872, + "n": 65536, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 12134, + "n": 12288, + "k": 16384, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 12134, + "n": 81920, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 12136, + "n": 320, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 12136, + "n": 6144, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3" + ] + }, + { + "type": "gemm", + "args": { + "m": 12136, + "n": 7168, + "k": 3072, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w2" + ] + }, + { + "type": "gemm", + "args": { + "m": 12264, + "n": 12288, + "k": 16384, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 12264, + "n": 81920, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 12560, + "n": 24, + "k": 28672, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/hc_attn", + "deepseek-ai/DeepSeek-V4-Pro-0813/hc_ffn", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.hc_attn", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.hc_ffn", + "deepseek-ai/DeepSeek-V4-Pro/hc_attn", + "deepseek-ai/DeepSeek-V4-Pro/hc_ffn", + "deepseek-ai/DeepSeek-V4-Pro/mtp.hc_attn", + "deepseek-ai/DeepSeek-V4-Pro/mtp.hc_ffn", + "sgl-project/DeepSeek-V4-Pro-FP8/hc_attn", + "sgl-project/DeepSeek-V4-Pro-FP8/hc_ffn", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.hc_attn", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.hc_ffn" + ] + }, + { + "type": "gemm", + "args": { + "m": 12560, + "n": 24, + "k": 40960, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 12560, + "n": 64, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.weights_proj", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.weights_proj", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.weights_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 12560, + "n": 64, + "k": 10240, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 12560, + "n": 192, + "k": 10240, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 12560, + "n": 256, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.compressor.wkv", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.compressor.wkv", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.compressor.wgate", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.compressor.wkv" + ] + }, + { + "type": "gemm", + "args": { + "m": 12560, + "n": 256, + "k": 10240, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 12560, + "n": 384, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.gate", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.gate", + "deepseek-ai/DeepSeek-V4-Pro/ffn.gate", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.gate", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.gate", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.gate" + ] + }, + { + "type": "gemm", + "args": { + "m": 12560, + "n": 512, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wkv", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wkv", + "deepseek-ai/DeepSeek-V4-Pro/attn.wkv", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wkv", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wkv", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wkv" + ] + }, + { + "type": "gemm", + "args": { + "m": 12560, + "n": 512, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.compressor.wkv+wgate", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.compressor.wkv+wgate", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.compressor.wkv+wgate" + ] + }, + { + "type": "gemm", + "args": { + "m": 12560, + "n": 512, + "k": 10240, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 12560, + "n": 512, + "k": 10240, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 12560, + "n": 1024, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.compressor.wkv", + "deepseek-ai/DeepSeek-V4-Pro/attn.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro/attn.compressor.wkv", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.compressor.wgate", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.compressor.wkv" + ] + }, + { + "type": "gemm", + "args": { + "m": 12560, + "n": 1024, + "k": 10240, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 12560, + "n": 1536, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w3" + ] + }, + { + "type": "gemm", + "args": { + "m": 12560, + "n": 1536, + "k": 10240, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 12560, + "n": 6144, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3" + ] + }, + { + "type": "gemm", + "args": { + "m": 12560, + "n": 7168, + "k": 3072, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w2" + ] + }, + { + "type": "gemm", + "args": { + "m": 12560, + "n": 7168, + "k": 16384, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 12560, + "n": 8192, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 12560, + "n": 10240, + "k": 16384, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 12560, + "n": 65536, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 12598, + "n": 65536, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 12930, + "n": 24, + "k": 28672, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/hc_attn", + "deepseek-ai/DeepSeek-V4-Pro-0813/hc_ffn", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.hc_attn", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.hc_ffn", + "deepseek-ai/DeepSeek-V4-Pro/hc_attn", + "deepseek-ai/DeepSeek-V4-Pro/hc_ffn", + "deepseek-ai/DeepSeek-V4-Pro/mtp.hc_attn", + "deepseek-ai/DeepSeek-V4-Pro/mtp.hc_ffn", + "sgl-project/DeepSeek-V4-Pro-FP8/hc_attn", + "sgl-project/DeepSeek-V4-Pro-FP8/hc_ffn", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.hc_attn", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.hc_ffn" + ] + }, + { + "type": "gemm", + "args": { + "m": 12930, + "n": 24, + "k": 40960, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 12930, + "n": 64, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.weights_proj", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.weights_proj", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.weights_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 12930, + "n": 64, + "k": 10240, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 12930, + "n": 192, + "k": 10240, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 12930, + "n": 256, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.compressor.wkv", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.compressor.wkv", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.compressor.wgate", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.compressor.wkv" + ] + }, + { + "type": "gemm", + "args": { + "m": 12930, + "n": 256, + "k": 10240, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 12930, + "n": 384, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.gate", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.gate", + "deepseek-ai/DeepSeek-V4-Pro/ffn.gate", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.gate", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.gate", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.gate" + ] + }, + { + "type": "gemm", + "args": { + "m": 12930, + "n": 512, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wkv", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wkv", + "deepseek-ai/DeepSeek-V4-Pro/attn.wkv", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wkv", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wkv", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wkv" + ] + }, + { + "type": "gemm", + "args": { + "m": 12930, + "n": 512, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.compressor.wkv+wgate", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.compressor.wkv+wgate", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.compressor.wkv+wgate" + ] + }, + { + "type": "gemm", + "args": { + "m": 12930, + "n": 512, + "k": 10240, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 12930, + "n": 512, + "k": 10240, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 12930, + "n": 1024, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.compressor.wkv", + "deepseek-ai/DeepSeek-V4-Pro/attn.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro/attn.compressor.wkv", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.compressor.wgate", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.compressor.wkv" + ] + }, + { + "type": "gemm", + "args": { + "m": 12930, + "n": 1024, + "k": 10240, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 12930, + "n": 1536, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w3" + ] + }, + { + "type": "gemm", + "args": { + "m": 12930, + "n": 1536, + "k": 10240, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 12930, + "n": 7168, + "k": 16384, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 12930, + "n": 8192, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 12930, + "n": 10240, + "k": 16384, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 12936, + "n": 24, + "k": 40960, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 12936, + "n": 64, + "k": 10240, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 12936, + "n": 192, + "k": 10240, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 12936, + "n": 256, + "k": 10240, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 12936, + "n": 512, + "k": 10240, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 12936, + "n": 512, + "k": 10240, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 12936, + "n": 1024, + "k": 10240, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 12936, + "n": 1536, + "k": 10240, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 12936, + "n": 8192, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 12936, + "n": 10240, + "k": 16384, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 12936, + "n": 65536, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 12960, + "n": 24, + "k": 28672, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/hc_attn", + "deepseek-ai/DeepSeek-V4-Pro-0813/hc_ffn", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.hc_attn", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.hc_ffn", + "deepseek-ai/DeepSeek-V4-Pro/hc_attn", + "deepseek-ai/DeepSeek-V4-Pro/hc_ffn", + "deepseek-ai/DeepSeek-V4-Pro/mtp.hc_attn", + "deepseek-ai/DeepSeek-V4-Pro/mtp.hc_ffn", + "sgl-project/DeepSeek-V4-Pro-FP8/hc_attn", + "sgl-project/DeepSeek-V4-Pro-FP8/hc_ffn", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.hc_attn", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.hc_ffn" + ] + }, + { + "type": "gemm", + "args": { + "m": 12960, + "n": 24, + "k": 40960, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 12960, + "n": 64, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.weights_proj", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.weights_proj", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.weights_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 12960, + "n": 64, + "k": 10240, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 12960, + "n": 192, + "k": 10240, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 12960, + "n": 256, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.compressor.wkv", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.compressor.wkv", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.compressor.wgate", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.compressor.wkv" + ] + }, + { + "type": "gemm", + "args": { + "m": 12960, + "n": 256, + "k": 10240, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 12960, + "n": 384, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.gate", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.gate", + "deepseek-ai/DeepSeek-V4-Pro/ffn.gate", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.gate", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.gate", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.gate" + ] + }, + { + "type": "gemm", + "args": { + "m": 12960, + "n": 512, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wkv", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wkv", + "deepseek-ai/DeepSeek-V4-Pro/attn.wkv", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wkv", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wkv", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wkv" + ] + }, + { + "type": "gemm", + "args": { + "m": 12960, + "n": 512, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.compressor.wkv+wgate", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.compressor.wkv+wgate", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.compressor.wkv+wgate" + ] + }, + { + "type": "gemm", + "args": { + "m": 12960, + "n": 512, + "k": 10240, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 12960, + "n": 512, + "k": 10240, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 12960, + "n": 1024, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.compressor.wkv", + "deepseek-ai/DeepSeek-V4-Pro/attn.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro/attn.compressor.wkv", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.compressor.wgate", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.compressor.wkv" + ] + }, + { + "type": "gemm", + "args": { + "m": 12960, + "n": 1024, + "k": 10240, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 12960, + "n": 1536, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w3" + ] + }, + { + "type": "gemm", + "args": { + "m": 12960, + "n": 1536, + "k": 10240, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 12960, + "n": 7168, + "k": 16384, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 12960, + "n": 8192, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 12960, + "n": 10240, + "k": 16384, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 12960, + "n": 65536, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 13032, + "n": 12288, + "k": 8192, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 13032, + "n": 16384, + "k": 12288, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 13072, + "n": 24, + "k": 40960, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 13072, + "n": 24, + "k": 49152, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 13072, + "n": 64, + "k": 10240, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 13072, + "n": 64, + "k": 12288, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 13072, + "n": 120, + "k": 12288, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 13072, + "n": 192, + "k": 10240, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 13072, + "n": 256, + "k": 10240, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 13072, + "n": 256, + "k": 12288, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 13072, + "n": 512, + "k": 10240, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 13072, + "n": 512, + "k": 10240, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 13072, + "n": 512, + "k": 12288, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 13072, + "n": 512, + "k": 12288, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 13072, + "n": 1024, + "k": 10240, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 13072, + "n": 1024, + "k": 12288, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 13072, + "n": 1536, + "k": 10240, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 13072, + "n": 1536, + "k": 12288, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 13072, + "n": 8192, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 13072, + "n": 10240, + "k": 16384, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 13072, + "n": 12288, + "k": 8192, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 13072, + "n": 12288, + "k": 16384, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 13072, + "n": 16384, + "k": 12288, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 13072, + "n": 81920, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 13116, + "n": 24, + "k": 49152, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 13116, + "n": 64, + "k": 12288, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 13116, + "n": 120, + "k": 12288, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 13116, + "n": 256, + "k": 12288, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 13116, + "n": 512, + "k": 12288, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 13116, + "n": 512, + "k": 12288, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 13116, + "n": 1024, + "k": 12288, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 13116, + "n": 1536, + "k": 12288, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 13658, + "n": 2048, + "k": 10240, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 13658, + "n": 10240, + "k": 1024, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 13658, + "n": 10240, + "k": 4096, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 13658, + "n": 16384, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 13790, + "n": 320, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 13790, + "n": 7168, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.e_proj", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.h_proj", + "deepseek-ai/DeepSeek-V4-Pro/mtp.e_proj", + "deepseek-ai/DeepSeek-V4-Pro/mtp.h_proj", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.e_proj", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.h_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 13790, + "n": 14336, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 13796, + "n": 768, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w3" + ] + }, + { + "type": "gemm", + "args": { + "m": 13796, + "n": 7168, + "k": 384, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w2" + ] + }, + { + "type": "gemm", + "args": { + "m": 13796, + "n": 7168, + "k": 2048, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 13824, + "n": 24, + "k": 28672, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/hc_attn", + "deepseek-ai/DeepSeek-V4-Pro-0813/hc_ffn", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.hc_attn", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.hc_ffn", + "deepseek-ai/DeepSeek-V4-Pro/hc_attn", + "deepseek-ai/DeepSeek-V4-Pro/hc_ffn", + "deepseek-ai/DeepSeek-V4-Pro/mtp.hc_attn", + "deepseek-ai/DeepSeek-V4-Pro/mtp.hc_ffn", + "sgl-project/DeepSeek-V4-Pro-FP8/hc_attn", + "sgl-project/DeepSeek-V4-Pro-FP8/hc_ffn", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.hc_attn", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.hc_ffn" + ] + }, + { + "type": "gemm", + "args": { + "m": 13824, + "n": 64, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.weights_proj", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.weights_proj", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.weights_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 13824, + "n": 256, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.compressor.wkv", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.compressor.wkv", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.compressor.wgate", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.compressor.wkv" + ] + }, + { + "type": "gemm", + "args": { + "m": 13824, + "n": 384, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.gate", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.gate", + "deepseek-ai/DeepSeek-V4-Pro/ffn.gate", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.gate", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.gate", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.gate" + ] + }, + { + "type": "gemm", + "args": { + "m": 13824, + "n": 384, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w3" + ] + }, + { + "type": "gemm", + "args": { + "m": 13824, + "n": 512, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wkv", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wkv", + "deepseek-ai/DeepSeek-V4-Pro/attn.wkv", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wkv", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wkv", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wkv" + ] + }, + { + "type": "gemm", + "args": { + "m": 13824, + "n": 512, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.compressor.wkv+wgate", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.compressor.wkv+wgate", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.compressor.wkv+wgate" + ] + }, + { + "type": "gemm", + "args": { + "m": 13824, + "n": 1024, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.compressor.wkv", + "deepseek-ai/DeepSeek-V4-Pro/attn.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro/attn.compressor.wkv", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.compressor.wgate", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.compressor.wkv" + ] + }, + { + "type": "gemm", + "args": { + "m": 13824, + "n": 1536, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w3" + ] + }, + { + "type": "gemm", + "args": { + "m": 13824, + "n": 4096, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 13824, + "n": 7168, + "k": 192, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w2" + ] + }, + { + "type": "gemm", + "args": { + "m": 13824, + "n": 7168, + "k": 1024, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 13824, + "n": 8192, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 13990, + "n": 6144, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3" + ] + }, + { + "type": "gemm", + "args": { + "m": 13990, + "n": 7168, + "k": 3072, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w2" + ] + }, + { + "type": "gemm", + "args": { + "m": 14000, + "n": 24, + "k": 28672, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/hc_attn", + "deepseek-ai/DeepSeek-V4-Pro-0813/hc_ffn", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.hc_attn", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.hc_ffn", + "deepseek-ai/DeepSeek-V4-Pro/hc_attn", + "deepseek-ai/DeepSeek-V4-Pro/hc_ffn", + "deepseek-ai/DeepSeek-V4-Pro/mtp.hc_attn", + "deepseek-ai/DeepSeek-V4-Pro/mtp.hc_ffn", + "sgl-project/DeepSeek-V4-Pro-FP8/hc_attn", + "sgl-project/DeepSeek-V4-Pro-FP8/hc_ffn", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.hc_attn", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.hc_ffn" + ] + }, + { + "type": "gemm", + "args": { + "m": 14000, + "n": 64, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.weights_proj", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.weights_proj", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.weights_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 14000, + "n": 256, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.compressor.wkv", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.compressor.wkv", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.compressor.wgate", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.compressor.wkv" + ] + }, + { + "type": "gemm", + "args": { + "m": 14000, + "n": 512, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wkv", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wkv", + "deepseek-ai/DeepSeek-V4-Pro/attn.wkv", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wkv", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wkv", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wkv" + ] + }, + { + "type": "gemm", + "args": { + "m": 14000, + "n": 512, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.compressor.wkv+wgate", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.compressor.wkv+wgate", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.compressor.wkv+wgate" + ] + }, + { + "type": "gemm", + "args": { + "m": 14000, + "n": 1024, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.compressor.wkv", + "deepseek-ai/DeepSeek-V4-Pro/attn.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro/attn.compressor.wkv", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.compressor.wgate", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.compressor.wkv" + ] + }, + { + "type": "gemm", + "args": { + "m": 14000, + "n": 1536, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w3" + ] + }, + { + "type": "gemm", + "args": { + "m": 14169, + "n": 1024, + "k": 10240, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 14169, + "n": 2048, + "k": 12288, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 14169, + "n": 10240, + "k": 512, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 14169, + "n": 10240, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 14169, + "n": 10240, + "k": 2048, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 14169, + "n": 12288, + "k": 1024, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 14169, + "n": 12288, + "k": 2048, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 14922, + "n": 8192, + "k": 10240, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 14922, + "n": 10240, + "k": 4096, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 15000, + "n": 24, + "k": 28672, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/hc_attn", + "deepseek-ai/DeepSeek-V4-Pro-0813/hc_ffn", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.hc_attn", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.hc_ffn", + "deepseek-ai/DeepSeek-V4-Pro/hc_attn", + "deepseek-ai/DeepSeek-V4-Pro/hc_ffn", + "deepseek-ai/DeepSeek-V4-Pro/mtp.hc_attn", + "deepseek-ai/DeepSeek-V4-Pro/mtp.hc_ffn", + "sgl-project/DeepSeek-V4-Pro-FP8/hc_attn", + "sgl-project/DeepSeek-V4-Pro-FP8/hc_ffn", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.hc_attn", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.hc_ffn" + ] + }, + { + "type": "gemm", + "args": { + "m": 15000, + "n": 64, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.weights_proj", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.weights_proj", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.weights_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 15000, + "n": 256, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.compressor.wkv", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.compressor.wkv", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.compressor.wgate", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.compressor.wkv" + ] + }, + { + "type": "gemm", + "args": { + "m": 15000, + "n": 512, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wkv", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wkv", + "deepseek-ai/DeepSeek-V4-Pro/attn.wkv", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wkv", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wkv", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wkv" + ] + }, + { + "type": "gemm", + "args": { + "m": 15000, + "n": 512, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.compressor.wkv+wgate", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.compressor.wkv+wgate", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.compressor.wkv+wgate" + ] + }, + { + "type": "gemm", + "args": { + "m": 15000, + "n": 1024, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.compressor.wkv", + "deepseek-ai/DeepSeek-V4-Pro/attn.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro/attn.compressor.wkv", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.compressor.wgate", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.compressor.wkv" + ] + }, + { + "type": "gemm", + "args": { + "m": 15000, + "n": 1536, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w3" + ] + }, + { + "type": "gemm", + "args": { + "m": 15000, + "n": 7168, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.e_proj", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.h_proj", + "deepseek-ai/DeepSeek-V4-Pro/mtp.e_proj", + "deepseek-ai/DeepSeek-V4-Pro/mtp.h_proj", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.e_proj", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.h_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 15000, + "n": 7168, + "k": 16384, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 15000, + "n": 8192, + "k": 10240, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 15000, + "n": 10240, + "k": 4096, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 15000, + "n": 14336, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 15312, + "n": 320, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 15312, + "n": 7168, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.e_proj", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.h_proj", + "deepseek-ai/DeepSeek-V4-Pro/mtp.e_proj", + "deepseek-ai/DeepSeek-V4-Pro/mtp.h_proj", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.e_proj", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.h_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 15312, + "n": 14336, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 15332, + "n": 12288, + "k": 16384, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 15332, + "n": 81920, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 15982, + "n": 320, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 15982, + "n": 7168, + "k": 16384, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 15982, + "n": 65536, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 16347, + "n": 24, + "k": 28672, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/hc_attn", + "deepseek-ai/DeepSeek-V4-Pro-0813/hc_ffn", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.hc_attn", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.hc_ffn", + "deepseek-ai/DeepSeek-V4-Pro/hc_attn", + "deepseek-ai/DeepSeek-V4-Pro/hc_ffn", + "deepseek-ai/DeepSeek-V4-Pro/mtp.hc_attn", + "deepseek-ai/DeepSeek-V4-Pro/mtp.hc_ffn", + "sgl-project/DeepSeek-V4-Pro-FP8/hc_attn", + "sgl-project/DeepSeek-V4-Pro-FP8/hc_ffn", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.hc_attn", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.hc_ffn" + ] + }, + { + "type": "gemm", + "args": { + "m": 16347, + "n": 64, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.weights_proj", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.weights_proj", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.weights_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 16347, + "n": 256, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.compressor.wkv", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.compressor.wkv", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.compressor.wgate", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.compressor.wkv" + ] + }, + { + "type": "gemm", + "args": { + "m": 16347, + "n": 384, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.gate", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.gate", + "deepseek-ai/DeepSeek-V4-Pro/ffn.gate", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.gate", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.gate", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.gate" + ] + }, + { + "type": "gemm", + "args": { + "m": 16347, + "n": 512, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wkv", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wkv", + "deepseek-ai/DeepSeek-V4-Pro/attn.wkv", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wkv", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wkv", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wkv" + ] + }, + { + "type": "gemm", + "args": { + "m": 16347, + "n": 512, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.compressor.wkv+wgate", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.compressor.wkv+wgate", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.compressor.wkv+wgate" + ] + }, + { + "type": "gemm", + "args": { + "m": 16347, + "n": 1024, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.compressor.wkv", + "deepseek-ai/DeepSeek-V4-Pro/attn.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro/attn.compressor.wkv", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.compressor.wgate", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.compressor.wkv" + ] + }, + { + "type": "gemm", + "args": { + "m": 16347, + "n": 1536, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w3" + ] + }, + { + "type": "gemm", + "args": { + "m": 16347, + "n": 8192, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 16384, + "n": 24, + "k": 28672, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/hc_attn", + "deepseek-ai/DeepSeek-V4-Pro-0813/hc_ffn", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.hc_attn", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.hc_ffn", + "deepseek-ai/DeepSeek-V4-Pro/hc_attn", + "deepseek-ai/DeepSeek-V4-Pro/hc_ffn", + "deepseek-ai/DeepSeek-V4-Pro/mtp.hc_attn", + "deepseek-ai/DeepSeek-V4-Pro/mtp.hc_ffn", + "sgl-project/DeepSeek-V4-Pro-FP8/hc_attn", + "sgl-project/DeepSeek-V4-Pro-FP8/hc_ffn", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.hc_attn", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.hc_ffn" + ] + }, + { + "type": "gemm", + "args": { + "m": 16384, + "n": 24, + "k": 40960, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 16384, + "n": 24, + "k": 49152, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 16384, + "n": 64, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.weights_proj", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.weights_proj", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.weights_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 16384, + "n": 64, + "k": 10240, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 16384, + "n": 64, + "k": 12288, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 16384, + "n": 120, + "k": 12288, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 16384, + "n": 192, + "k": 10240, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 16384, + "n": 256, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.compressor.wkv", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.compressor.wkv", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.compressor.wgate", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.compressor.wkv" + ] + }, + { + "type": "gemm", + "args": { + "m": 16384, + "n": 256, + "k": 10240, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 16384, + "n": 256, + "k": 12288, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 16384, + "n": 384, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.gate", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.gate", + "deepseek-ai/DeepSeek-V4-Pro/ffn.gate", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.gate", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.gate", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.gate" + ] + }, + { + "type": "gemm", + "args": { + "m": 16384, + "n": 512, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wkv", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wkv", + "deepseek-ai/DeepSeek-V4-Pro/attn.wkv", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wkv", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wkv", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wkv" + ] + }, + { + "type": "gemm", + "args": { + "m": 16384, + "n": 512, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.compressor.wkv+wgate", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.compressor.wkv+wgate", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.compressor.wkv+wgate" + ] + }, + { + "type": "gemm", + "args": { + "m": 16384, + "n": 512, + "k": 10240, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 16384, + "n": 512, + "k": 10240, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 16384, + "n": 512, + "k": 12288, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 16384, + "n": 512, + "k": 12288, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 16384, + "n": 768, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w3" + ] + }, + { + "type": "gemm", + "args": { + "m": 16384, + "n": 1024, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.compressor.wkv", + "deepseek-ai/DeepSeek-V4-Pro/attn.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro/attn.compressor.wkv", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.compressor.wgate", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.compressor.wkv" + ] + }, + { + "type": "gemm", + "args": { + "m": 16384, + "n": 1024, + "k": 10240, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 16384, + "n": 1024, + "k": 10240, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 16384, + "n": 1024, + "k": 12288, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 16384, + "n": 1024, + "k": 12288, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 16384, + "n": 1536, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w3" + ] + }, + { + "type": "gemm", + "args": { + "m": 16384, + "n": 1536, + "k": 10240, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 16384, + "n": 1536, + "k": 12288, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 16384, + "n": 2048, + "k": 10240, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 16384, + "n": 2048, + "k": 12288, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 16384, + "n": 5120, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 16384, + "n": 7168, + "k": 384, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w2" + ] + }, + { + "type": "gemm", + "args": { + "m": 16384, + "n": 7168, + "k": 2048, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 16384, + "n": 7168, + "k": 16384, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 16384, + "n": 8192, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 16384, + "n": 8192, + "k": 10240, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 16384, + "n": 10240, + "k": 512, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 16384, + "n": 10240, + "k": 1024, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 16384, + "n": 10240, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 16384, + "n": 10240, + "k": 2048, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 16384, + "n": 10240, + "k": 4096, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 16384, + "n": 10240, + "k": 16384, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 16384, + "n": 12288, + "k": 512, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 16384, + "n": 12288, + "k": 1024, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 16384, + "n": 12288, + "k": 2048, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 16384, + "n": 16384, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 16384, + "n": 65536, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 16386, + "n": 24, + "k": 40960, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 16386, + "n": 24, + "k": 49152, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 16386, + "n": 64, + "k": 10240, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 16386, + "n": 64, + "k": 12288, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 16386, + "n": 120, + "k": 12288, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 16386, + "n": 192, + "k": 10240, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 16386, + "n": 256, + "k": 10240, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 16386, + "n": 256, + "k": 12288, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 16386, + "n": 512, + "k": 10240, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 16386, + "n": 512, + "k": 10240, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 16386, + "n": 512, + "k": 12288, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 16386, + "n": 512, + "k": 12288, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 16386, + "n": 1024, + "k": 10240, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 16386, + "n": 1024, + "k": 12288, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 16386, + "n": 1024, + "k": 12288, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 16386, + "n": 1536, + "k": 10240, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 16386, + "n": 1536, + "k": 12288, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 16386, + "n": 5120, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 16386, + "n": 6144, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3" + ] + }, + { + "type": "gemm", + "args": { + "m": 16386, + "n": 7168, + "k": 3072, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w2" + ] + }, + { + "type": "gemm", + "args": { + "m": 16386, + "n": 10240, + "k": 16384, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 16386, + "n": 12288, + "k": 512, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 16386, + "n": 12288, + "k": 1024, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 16386, + "n": 12288, + "k": 8192, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 16386, + "n": 16384, + "k": 12288, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 22400, + "n": 24, + "k": 28672, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/hc_attn", + "deepseek-ai/DeepSeek-V4-Pro-0813/hc_ffn", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.hc_attn", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.hc_ffn", + "deepseek-ai/DeepSeek-V4-Pro/hc_attn", + "deepseek-ai/DeepSeek-V4-Pro/hc_ffn", + "deepseek-ai/DeepSeek-V4-Pro/mtp.hc_attn", + "deepseek-ai/DeepSeek-V4-Pro/mtp.hc_ffn", + "sgl-project/DeepSeek-V4-Pro-FP8/hc_attn", + "sgl-project/DeepSeek-V4-Pro-FP8/hc_ffn", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.hc_attn", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.hc_ffn" + ] + }, + { + "type": "gemm", + "args": { + "m": 22400, + "n": 64, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.weights_proj", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.weights_proj", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.weights_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 22400, + "n": 256, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.compressor.wkv", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.compressor.wkv", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.compressor.wgate", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.compressor.wkv" + ] + }, + { + "type": "gemm", + "args": { + "m": 22400, + "n": 384, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.gate", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.gate", + "deepseek-ai/DeepSeek-V4-Pro/ffn.gate", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.gate", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.gate", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.gate" + ] + }, + { + "type": "gemm", + "args": { + "m": 22400, + "n": 512, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wkv", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wkv", + "deepseek-ai/DeepSeek-V4-Pro/attn.wkv", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wkv", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wkv", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wkv" + ] + }, + { + "type": "gemm", + "args": { + "m": 22400, + "n": 512, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.compressor.wkv+wgate", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.compressor.wkv+wgate", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.compressor.wkv+wgate" + ] + }, + { + "type": "gemm", + "args": { + "m": 22400, + "n": 1024, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.compressor.wkv", + "deepseek-ai/DeepSeek-V4-Pro/attn.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro/attn.compressor.wkv", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.compressor.wgate", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.compressor.wkv" + ] + }, + { + "type": "gemm", + "args": { + "m": 22400, + "n": 1536, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w3" + ] + }, + { + "type": "gemm", + "args": { + "m": 22400, + "n": 6144, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3" + ] + }, + { + "type": "gemm", + "args": { + "m": 22400, + "n": 7168, + "k": 3072, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w2" + ] + }, + { + "type": "gemm", + "args": { + "m": 22400, + "n": 7168, + "k": 16384, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 22400, + "n": 8192, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 22400, + "n": 65536, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 29291, + "n": 24, + "k": 28672, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/hc_attn", + "deepseek-ai/DeepSeek-V4-Pro-0813/hc_ffn", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.hc_attn", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.hc_ffn", + "deepseek-ai/DeepSeek-V4-Pro/hc_attn", + "deepseek-ai/DeepSeek-V4-Pro/hc_ffn", + "deepseek-ai/DeepSeek-V4-Pro/mtp.hc_attn", + "deepseek-ai/DeepSeek-V4-Pro/mtp.hc_ffn", + "sgl-project/DeepSeek-V4-Pro-FP8/hc_attn", + "sgl-project/DeepSeek-V4-Pro-FP8/hc_ffn", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.hc_attn", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.hc_ffn" + ] + }, + { + "type": "gemm", + "args": { + "m": 29291, + "n": 64, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.weights_proj", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.weights_proj", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.weights_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 29291, + "n": 256, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.compressor.wkv", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.compressor.wkv", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.compressor.wgate", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.compressor.wkv" + ] + }, + { + "type": "gemm", + "args": { + "m": 29291, + "n": 384, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.gate", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.gate", + "deepseek-ai/DeepSeek-V4-Pro/ffn.gate", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.gate", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.gate", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.gate" + ] + }, + { + "type": "gemm", + "args": { + "m": 29291, + "n": 512, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wkv", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wkv", + "deepseek-ai/DeepSeek-V4-Pro/attn.wkv", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wkv", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wkv", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wkv" + ] + }, + { + "type": "gemm", + "args": { + "m": 29291, + "n": 512, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.compressor.wkv+wgate", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.compressor.wkv+wgate", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.compressor.wkv+wgate" + ] + }, + { + "type": "gemm", + "args": { + "m": 29291, + "n": 1024, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.compressor.wkv", + "deepseek-ai/DeepSeek-V4-Pro/attn.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro/attn.compressor.wkv", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.compressor.wgate", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.compressor.wkv" + ] + }, + { + "type": "gemm", + "args": { + "m": 29291, + "n": 1536, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w3" + ] + }, + { + "type": "gemm", + "args": { + "m": 29291, + "n": 6144, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3" + ] + }, + { + "type": "gemm", + "args": { + "m": 29291, + "n": 7168, + "k": 3072, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w2" + ] + }, + { + "type": "gemm", + "args": { + "m": 29291, + "n": 7168, + "k": 16384, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 29291, + "n": 8192, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 29291, + "n": 65536, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 32028, + "n": 6144, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3" + ] + }, + { + "type": "gemm", + "args": { + "m": 32028, + "n": 7168, + "k": 3072, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w2" + ] + }, + { + "type": "gemm", + "args": { + "m": 32768, + "n": 24, + "k": 28672, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/hc_attn", + "deepseek-ai/DeepSeek-V4-Pro-0813/hc_ffn", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.hc_attn", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.hc_ffn", + "deepseek-ai/DeepSeek-V4-Pro/hc_attn", + "deepseek-ai/DeepSeek-V4-Pro/hc_ffn", + "deepseek-ai/DeepSeek-V4-Pro/mtp.hc_attn", + "deepseek-ai/DeepSeek-V4-Pro/mtp.hc_ffn", + "sgl-project/DeepSeek-V4-Pro-FP8/hc_attn", + "sgl-project/DeepSeek-V4-Pro-FP8/hc_ffn", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.hc_attn", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.hc_ffn" + ] + }, + { + "type": "gemm", + "args": { + "m": 32768, + "n": 64, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.weights_proj", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.weights_proj", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.weights_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 32768, + "n": 256, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.compressor.wkv", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.compressor.wkv", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.compressor.wgate", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.compressor.wkv" + ] + }, + { + "type": "gemm", + "args": { + "m": 32768, + "n": 384, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.gate", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.gate", + "deepseek-ai/DeepSeek-V4-Pro/ffn.gate", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.gate", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.gate", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.gate" + ] + }, + { + "type": "gemm", + "args": { + "m": 32768, + "n": 384, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w3" + ] + }, + { + "type": "gemm", + "args": { + "m": 32768, + "n": 512, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wkv", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wkv", + "deepseek-ai/DeepSeek-V4-Pro/attn.wkv", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wkv", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wkv", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wkv" + ] + }, + { + "type": "gemm", + "args": { + "m": 32768, + "n": 512, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.compressor.wkv+wgate", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.compressor.wkv+wgate", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.compressor.wkv+wgate" + ] + }, + { + "type": "gemm", + "args": { + "m": 32768, + "n": 1024, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.compressor.wkv", + "deepseek-ai/DeepSeek-V4-Pro/attn.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro/attn.compressor.wkv", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.compressor.wgate", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.compressor.wkv" + ] + }, + { + "type": "gemm", + "args": { + "m": 32768, + "n": 1536, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w3" + ] + }, + { + "type": "gemm", + "args": { + "m": 32768, + "n": 4096, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 32768, + "n": 6144, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3" + ] + }, + { + "type": "gemm", + "args": { + "m": 32768, + "n": 7168, + "k": 192, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w2" + ] + }, + { + "type": "gemm", + "args": { + "m": 32768, + "n": 7168, + "k": 1024, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 32768, + "n": 7168, + "k": 3072, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w2" + ] + }, + { + "type": "gemm", + "args": { + "m": 32768, + "n": 7168, + "k": 16384, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 32768, + "n": 8192, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 32768, + "n": 65536, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_b" + ] + } +] diff --git a/operatorx/testlists/gemm_serving_all_min.json b/operatorx/testlists/gemm_serving_all_min.json new file mode 100644 index 0000000000..6b06be1f49 --- /dev/null +++ b/operatorx/testlists/gemm_serving_all_min.json @@ -0,0 +1,241390 @@ +[ + { + "type": "gemm", + "args": { + "m": 1, + "n": 24, + "k": 28672, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/hc_attn", + "deepseek-ai/DeepSeek-V4-Pro-0813/hc_ffn", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.hc_attn", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.hc_ffn", + "deepseek-ai/DeepSeek-V4-Pro/hc_attn", + "deepseek-ai/DeepSeek-V4-Pro/hc_ffn", + "deepseek-ai/DeepSeek-V4-Pro/mtp.hc_attn", + "deepseek-ai/DeepSeek-V4-Pro/mtp.hc_ffn", + "sgl-project/DeepSeek-V4-Pro-FP8/hc_attn", + "sgl-project/DeepSeek-V4-Pro-FP8/hc_ffn", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.hc_attn", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.hc_ffn" + ] + }, + { + "type": "gemm", + "args": { + "m": 1, + "n": 24, + "k": 40960, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 1, + "n": 24, + "k": 49152, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 1, + "n": 64, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.weights_proj", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.weights_proj", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.weights_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 1, + "n": 64, + "k": 10240, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 1, + "n": 64, + "k": 12288, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 1, + "n": 120, + "k": 12288, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 1, + "n": 192, + "k": 10240, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 1, + "n": 256, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.compressor.wkv", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.compressor.wkv", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.compressor.wgate", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.compressor.wkv" + ] + }, + { + "type": "gemm", + "args": { + "m": 1, + "n": 256, + "k": 10240, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 1, + "n": 256, + "k": 12288, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 1, + "n": 320, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 1, + "n": 384, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.gate", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.gate", + "deepseek-ai/DeepSeek-V4-Pro/ffn.gate", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.gate", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.gate", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.gate" + ] + }, + { + "type": "gemm", + "args": { + "m": 1, + "n": 384, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w3" + ] + }, + { + "type": "gemm", + "args": { + "m": 1, + "n": 512, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wkv", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wkv", + "deepseek-ai/DeepSeek-V4-Pro/attn.wkv", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wkv", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wkv", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wkv" + ] + }, + { + "type": "gemm", + "args": { + "m": 1, + "n": 512, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.compressor.wkv+wgate", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.compressor.wkv+wgate", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.compressor.wkv+wgate" + ] + }, + { + "type": "gemm", + "args": { + "m": 1, + "n": 512, + "k": 10240, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 1, + "n": 512, + "k": 10240, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 1, + "n": 512, + "k": 12288, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 1, + "n": 512, + "k": 12288, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 1, + "n": 768, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w3" + ] + }, + { + "type": "gemm", + "args": { + "m": 1, + "n": 896, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 1, + "n": 1024, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.compressor.wkv", + "deepseek-ai/DeepSeek-V4-Pro/attn.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro/attn.compressor.wkv", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.compressor.wgate", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.compressor.wkv" + ] + }, + { + "type": "gemm", + "args": { + "m": 1, + "n": 1024, + "k": 10240, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 1, + "n": 1024, + "k": 10240, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 1, + "n": 1024, + "k": 12288, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 1, + "n": 1536, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w3" + ] + }, + { + "type": "gemm", + "args": { + "m": 1, + "n": 1536, + "k": 10240, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 1, + "n": 1536, + "k": 12288, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 1, + "n": 1792, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 1, + "n": 2048, + "k": 12288, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 1, + "n": 4096, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 1, + "n": 5120, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 1, + "n": 6144, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3" + ] + }, + { + "type": "gemm", + "args": { + "m": 1, + "n": 7168, + "k": 192, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w2" + ] + }, + { + "type": "gemm", + "args": { + "m": 1, + "n": 7168, + "k": 384, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w2" + ] + }, + { + "type": "gemm", + "args": { + "m": 1, + "n": 7168, + "k": 448, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 1, + "n": 7168, + "k": 768, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w2" + ] + }, + { + "type": "gemm", + "args": { + "m": 1, + "n": 7168, + "k": 896, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 1, + "n": 7168, + "k": 1024, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 1, + "n": 7168, + "k": 2048, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 1, + "n": 7168, + "k": 3072, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w2" + ] + }, + { + "type": "gemm", + "args": { + "m": 1, + "n": 7168, + "k": 4096, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 1, + "n": 7168, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.e_proj", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.h_proj", + "deepseek-ai/DeepSeek-V4-Pro/mtp.e_proj", + "deepseek-ai/DeepSeek-V4-Pro/mtp.h_proj", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.e_proj", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.h_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 1, + "n": 7168, + "k": 16384, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 1, + "n": 8080, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/head", + "deepseek-ai/DeepSeek-V4-Pro/head", + "sgl-project/DeepSeek-V4-Pro-FP8/head" + ] + }, + { + "type": "gemm", + "args": { + "m": 1, + "n": 8080, + "k": 10240, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 1, + "n": 8080, + "k": 12288, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 1, + "n": 8192, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 1, + "n": 8192, + "k": 10240, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 1, + "n": 10240, + "k": 512, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 1, + "n": 10240, + "k": 1024, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 1, + "n": 10240, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 1, + "n": 10240, + "k": 2048, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 1, + "n": 10240, + "k": 4096, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 1, + "n": 10240, + "k": 16384, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 1, + "n": 12288, + "k": 1024, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 1, + "n": 12288, + "k": 2048, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 1, + "n": 12288, + "k": 8192, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 1, + "n": 12288, + "k": 16384, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 1, + "n": 14336, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 1, + "n": 16160, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/head", + "deepseek-ai/DeepSeek-V4-Pro/head", + "sgl-project/DeepSeek-V4-Pro-FP8/head" + ] + }, + { + "type": "gemm", + "args": { + "m": 1, + "n": 16384, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 1, + "n": 16384, + "k": 12288, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 1, + "n": 32320, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/head", + "deepseek-ai/DeepSeek-V4-Pro/head", + "sgl-project/DeepSeek-V4-Pro-FP8/head" + ] + }, + { + "type": "gemm", + "args": { + "m": 1, + "n": 65536, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 1, + "n": 81920, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 1, + "n": 129280, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/head", + "deepseek-ai/DeepSeek-V4-Pro/head", + "sgl-project/DeepSeek-V4-Pro-FP8/head" + ] + }, + { + "type": "gemm", + "args": { + "m": 1, + "n": 129280, + "k": 10240, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 1, + "n": 129280, + "k": 12288, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 2, + "n": 24, + "k": 28672, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/hc_attn", + "deepseek-ai/DeepSeek-V4-Pro-0813/hc_ffn", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.hc_attn", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.hc_ffn", + "deepseek-ai/DeepSeek-V4-Pro/hc_attn", + "deepseek-ai/DeepSeek-V4-Pro/hc_ffn", + "deepseek-ai/DeepSeek-V4-Pro/mtp.hc_attn", + "deepseek-ai/DeepSeek-V4-Pro/mtp.hc_ffn", + "sgl-project/DeepSeek-V4-Pro-FP8/hc_attn", + "sgl-project/DeepSeek-V4-Pro-FP8/hc_ffn", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.hc_attn", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.hc_ffn" + ] + }, + { + "type": "gemm", + "args": { + "m": 2, + "n": 24, + "k": 40960, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 2, + "n": 24, + "k": 49152, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 2, + "n": 64, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.weights_proj", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.weights_proj", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.weights_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 2, + "n": 64, + "k": 10240, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 2, + "n": 64, + "k": 12288, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 2, + "n": 120, + "k": 12288, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 2, + "n": 192, + "k": 10240, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 2, + "n": 256, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.compressor.wkv", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.compressor.wkv", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.compressor.wgate", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.compressor.wkv" + ] + }, + { + "type": "gemm", + "args": { + "m": 2, + "n": 256, + "k": 10240, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 2, + "n": 256, + "k": 12288, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 2, + "n": 320, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 2, + "n": 384, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.gate", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.gate", + "deepseek-ai/DeepSeek-V4-Pro/ffn.gate", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.gate", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.gate", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.gate" + ] + }, + { + "type": "gemm", + "args": { + "m": 2, + "n": 384, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w3" + ] + }, + { + "type": "gemm", + "args": { + "m": 2, + "n": 512, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wkv", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wkv", + "deepseek-ai/DeepSeek-V4-Pro/attn.wkv", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wkv", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wkv", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wkv" + ] + }, + { + "type": "gemm", + "args": { + "m": 2, + "n": 512, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.compressor.wkv+wgate", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.compressor.wkv+wgate", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.compressor.wkv+wgate" + ] + }, + { + "type": "gemm", + "args": { + "m": 2, + "n": 512, + "k": 10240, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 2, + "n": 512, + "k": 10240, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 2, + "n": 512, + "k": 12288, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 2, + "n": 512, + "k": 12288, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 2, + "n": 768, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w3" + ] + }, + { + "type": "gemm", + "args": { + "m": 2, + "n": 896, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 2, + "n": 1024, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.compressor.wkv", + "deepseek-ai/DeepSeek-V4-Pro/attn.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro/attn.compressor.wkv", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.compressor.wgate", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.compressor.wkv" + ] + }, + { + "type": "gemm", + "args": { + "m": 2, + "n": 1024, + "k": 10240, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 2, + "n": 1024, + "k": 10240, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 2, + "n": 1024, + "k": 12288, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 2, + "n": 1536, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w3" + ] + }, + { + "type": "gemm", + "args": { + "m": 2, + "n": 1536, + "k": 10240, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 2, + "n": 1536, + "k": 12288, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 2, + "n": 1792, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 2, + "n": 4096, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 2, + "n": 5120, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 2, + "n": 6144, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3" + ] + }, + { + "type": "gemm", + "args": { + "m": 2, + "n": 7168, + "k": 192, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w2" + ] + }, + { + "type": "gemm", + "args": { + "m": 2, + "n": 7168, + "k": 384, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w2" + ] + }, + { + "type": "gemm", + "args": { + "m": 2, + "n": 7168, + "k": 448, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 2, + "n": 7168, + "k": 768, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w2" + ] + }, + { + "type": "gemm", + "args": { + "m": 2, + "n": 7168, + "k": 896, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 2, + "n": 7168, + "k": 1024, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 2, + "n": 7168, + "k": 2048, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 2, + "n": 7168, + "k": 3072, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w2" + ] + }, + { + "type": "gemm", + "args": { + "m": 2, + "n": 7168, + "k": 4096, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 2, + "n": 7168, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.e_proj", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.h_proj", + "deepseek-ai/DeepSeek-V4-Pro/mtp.e_proj", + "deepseek-ai/DeepSeek-V4-Pro/mtp.h_proj", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.e_proj", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.h_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 2, + "n": 7168, + "k": 16384, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 2, + "n": 8080, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/head", + "deepseek-ai/DeepSeek-V4-Pro/head", + "sgl-project/DeepSeek-V4-Pro-FP8/head" + ] + }, + { + "type": "gemm", + "args": { + "m": 2, + "n": 8080, + "k": 10240, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 2, + "n": 8080, + "k": 12288, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 2, + "n": 8192, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 2, + "n": 8192, + "k": 10240, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 2, + "n": 10240, + "k": 512, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 2, + "n": 10240, + "k": 1024, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 2, + "n": 10240, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 2, + "n": 10240, + "k": 2048, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 2, + "n": 10240, + "k": 4096, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 2, + "n": 10240, + "k": 16384, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 2, + "n": 12288, + "k": 1024, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 2, + "n": 12288, + "k": 2048, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 2, + "n": 12288, + "k": 8192, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 2, + "n": 12288, + "k": 16384, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 2, + "n": 14336, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 2, + "n": 16160, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/head", + "deepseek-ai/DeepSeek-V4-Pro/head", + "sgl-project/DeepSeek-V4-Pro-FP8/head" + ] + }, + { + "type": "gemm", + "args": { + "m": 2, + "n": 16384, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 2, + "n": 16384, + "k": 12288, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 2, + "n": 32320, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/head", + "deepseek-ai/DeepSeek-V4-Pro/head", + "sgl-project/DeepSeek-V4-Pro-FP8/head" + ] + }, + { + "type": "gemm", + "args": { + "m": 2, + "n": 65536, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 2, + "n": 81920, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 2, + "n": 129280, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/head", + "deepseek-ai/DeepSeek-V4-Pro/head", + "sgl-project/DeepSeek-V4-Pro-FP8/head" + ] + }, + { + "type": "gemm", + "args": { + "m": 2, + "n": 129280, + "k": 10240, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 2, + "n": 129280, + "k": 12288, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 3, + "n": 24, + "k": 28672, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/hc_attn", + "deepseek-ai/DeepSeek-V4-Pro-0813/hc_ffn", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.hc_attn", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.hc_ffn", + "deepseek-ai/DeepSeek-V4-Pro/hc_attn", + "deepseek-ai/DeepSeek-V4-Pro/hc_ffn", + "deepseek-ai/DeepSeek-V4-Pro/mtp.hc_attn", + "deepseek-ai/DeepSeek-V4-Pro/mtp.hc_ffn", + "sgl-project/DeepSeek-V4-Pro-FP8/hc_attn", + "sgl-project/DeepSeek-V4-Pro-FP8/hc_ffn", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.hc_attn", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.hc_ffn" + ] + }, + { + "type": "gemm", + "args": { + "m": 3, + "n": 24, + "k": 40960, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 3, + "n": 24, + "k": 49152, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 3, + "n": 64, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.weights_proj", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.weights_proj", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.weights_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 3, + "n": 64, + "k": 10240, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 3, + "n": 64, + "k": 12288, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 3, + "n": 120, + "k": 12288, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 3, + "n": 192, + "k": 10240, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 3, + "n": 256, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.compressor.wkv", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.compressor.wkv", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.compressor.wgate", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.compressor.wkv" + ] + }, + { + "type": "gemm", + "args": { + "m": 3, + "n": 256, + "k": 10240, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 3, + "n": 256, + "k": 12288, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 3, + "n": 320, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 3, + "n": 384, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.gate", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.gate", + "deepseek-ai/DeepSeek-V4-Pro/ffn.gate", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.gate", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.gate", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.gate" + ] + }, + { + "type": "gemm", + "args": { + "m": 3, + "n": 384, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w3" + ] + }, + { + "type": "gemm", + "args": { + "m": 3, + "n": 512, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wkv", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wkv", + "deepseek-ai/DeepSeek-V4-Pro/attn.wkv", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wkv", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wkv", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wkv" + ] + }, + { + "type": "gemm", + "args": { + "m": 3, + "n": 512, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.compressor.wkv+wgate", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.compressor.wkv+wgate", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.compressor.wkv+wgate" + ] + }, + { + "type": "gemm", + "args": { + "m": 3, + "n": 512, + "k": 10240, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 3, + "n": 512, + "k": 10240, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 3, + "n": 512, + "k": 12288, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 3, + "n": 512, + "k": 12288, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 3, + "n": 768, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w3" + ] + }, + { + "type": "gemm", + "args": { + "m": 3, + "n": 896, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 3, + "n": 1024, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.compressor.wkv", + "deepseek-ai/DeepSeek-V4-Pro/attn.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro/attn.compressor.wkv", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.compressor.wgate", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.compressor.wkv" + ] + }, + { + "type": "gemm", + "args": { + "m": 3, + "n": 1024, + "k": 10240, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 3, + "n": 1024, + "k": 10240, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 3, + "n": 1024, + "k": 12288, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 3, + "n": 1536, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w3" + ] + }, + { + "type": "gemm", + "args": { + "m": 3, + "n": 1536, + "k": 10240, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 3, + "n": 1536, + "k": 12288, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 3, + "n": 1792, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 3, + "n": 4096, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 3, + "n": 5120, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 3, + "n": 6144, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3" + ] + }, + { + "type": "gemm", + "args": { + "m": 3, + "n": 7168, + "k": 192, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w2" + ] + }, + { + "type": "gemm", + "args": { + "m": 3, + "n": 7168, + "k": 384, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w2" + ] + }, + { + "type": "gemm", + "args": { + "m": 3, + "n": 7168, + "k": 448, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 3, + "n": 7168, + "k": 768, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w2" + ] + }, + { + "type": "gemm", + "args": { + "m": 3, + "n": 7168, + "k": 896, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 3, + "n": 7168, + "k": 1024, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 3, + "n": 7168, + "k": 2048, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 3, + "n": 7168, + "k": 3072, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w2" + ] + }, + { + "type": "gemm", + "args": { + "m": 3, + "n": 7168, + "k": 4096, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 3, + "n": 7168, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.e_proj", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.h_proj", + "deepseek-ai/DeepSeek-V4-Pro/mtp.e_proj", + "deepseek-ai/DeepSeek-V4-Pro/mtp.h_proj", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.e_proj", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.h_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 3, + "n": 7168, + "k": 16384, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 3, + "n": 8080, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/head", + "deepseek-ai/DeepSeek-V4-Pro/head", + "sgl-project/DeepSeek-V4-Pro-FP8/head" + ] + }, + { + "type": "gemm", + "args": { + "m": 3, + "n": 8080, + "k": 10240, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 3, + "n": 8080, + "k": 12288, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 3, + "n": 8192, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 3, + "n": 8192, + "k": 10240, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 3, + "n": 10240, + "k": 512, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 3, + "n": 10240, + "k": 1024, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 3, + "n": 10240, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 3, + "n": 10240, + "k": 2048, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 3, + "n": 10240, + "k": 4096, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 3, + "n": 10240, + "k": 16384, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 3, + "n": 12288, + "k": 1024, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 3, + "n": 12288, + "k": 2048, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 3, + "n": 12288, + "k": 8192, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 3, + "n": 12288, + "k": 16384, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 3, + "n": 14336, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 3, + "n": 16160, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/head", + "deepseek-ai/DeepSeek-V4-Pro/head", + "sgl-project/DeepSeek-V4-Pro-FP8/head" + ] + }, + { + "type": "gemm", + "args": { + "m": 3, + "n": 16384, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 3, + "n": 16384, + "k": 12288, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 3, + "n": 32320, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/head", + "deepseek-ai/DeepSeek-V4-Pro/head", + "sgl-project/DeepSeek-V4-Pro-FP8/head" + ] + }, + { + "type": "gemm", + "args": { + "m": 3, + "n": 65536, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 3, + "n": 81920, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 3, + "n": 129280, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/head", + "deepseek-ai/DeepSeek-V4-Pro/head", + "sgl-project/DeepSeek-V4-Pro-FP8/head" + ] + }, + { + "type": "gemm", + "args": { + "m": 3, + "n": 129280, + "k": 10240, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 3, + "n": 129280, + "k": 12288, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 4, + "n": 24, + "k": 28672, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/hc_attn", + "deepseek-ai/DeepSeek-V4-Pro-0813/hc_ffn", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.hc_attn", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.hc_ffn", + "deepseek-ai/DeepSeek-V4-Pro/hc_attn", + "deepseek-ai/DeepSeek-V4-Pro/hc_ffn", + "deepseek-ai/DeepSeek-V4-Pro/mtp.hc_attn", + "deepseek-ai/DeepSeek-V4-Pro/mtp.hc_ffn", + "sgl-project/DeepSeek-V4-Pro-FP8/hc_attn", + "sgl-project/DeepSeek-V4-Pro-FP8/hc_ffn", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.hc_attn", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.hc_ffn" + ] + }, + { + "type": "gemm", + "args": { + "m": 4, + "n": 24, + "k": 40960, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 4, + "n": 24, + "k": 49152, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 4, + "n": 64, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.weights_proj", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.weights_proj", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.weights_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 4, + "n": 64, + "k": 10240, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 4, + "n": 64, + "k": 12288, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 4, + "n": 120, + "k": 12288, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 4, + "n": 192, + "k": 10240, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 4, + "n": 256, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.compressor.wkv", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.compressor.wkv", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.compressor.wgate", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.compressor.wkv" + ] + }, + { + "type": "gemm", + "args": { + "m": 4, + "n": 256, + "k": 10240, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 4, + "n": 256, + "k": 12288, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 4, + "n": 320, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 4, + "n": 384, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.gate", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.gate", + "deepseek-ai/DeepSeek-V4-Pro/ffn.gate", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.gate", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.gate", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.gate" + ] + }, + { + "type": "gemm", + "args": { + "m": 4, + "n": 384, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w3" + ] + }, + { + "type": "gemm", + "args": { + "m": 4, + "n": 512, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wkv", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wkv", + "deepseek-ai/DeepSeek-V4-Pro/attn.wkv", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wkv", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wkv", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wkv" + ] + }, + { + "type": "gemm", + "args": { + "m": 4, + "n": 512, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.compressor.wkv+wgate", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.compressor.wkv+wgate", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.compressor.wkv+wgate" + ] + }, + { + "type": "gemm", + "args": { + "m": 4, + "n": 512, + "k": 10240, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 4, + "n": 512, + "k": 10240, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 4, + "n": 512, + "k": 12288, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 4, + "n": 512, + "k": 12288, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 4, + "n": 768, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w3" + ] + }, + { + "type": "gemm", + "args": { + "m": 4, + "n": 896, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 4, + "n": 1024, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.compressor.wkv", + "deepseek-ai/DeepSeek-V4-Pro/attn.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro/attn.compressor.wkv", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.compressor.wgate", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.compressor.wkv" + ] + }, + { + "type": "gemm", + "args": { + "m": 4, + "n": 1024, + "k": 10240, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 4, + "n": 1024, + "k": 10240, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 4, + "n": 1024, + "k": 12288, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 4, + "n": 1536, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w3" + ] + }, + { + "type": "gemm", + "args": { + "m": 4, + "n": 1536, + "k": 10240, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 4, + "n": 1536, + "k": 12288, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 4, + "n": 1792, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 4, + "n": 4096, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 4, + "n": 5120, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 4, + "n": 6144, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3" + ] + }, + { + "type": "gemm", + "args": { + "m": 4, + "n": 7168, + "k": 192, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w2" + ] + }, + { + "type": "gemm", + "args": { + "m": 4, + "n": 7168, + "k": 384, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w2" + ] + }, + { + "type": "gemm", + "args": { + "m": 4, + "n": 7168, + "k": 448, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 4, + "n": 7168, + "k": 768, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w2" + ] + }, + { + "type": "gemm", + "args": { + "m": 4, + "n": 7168, + "k": 896, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 4, + "n": 7168, + "k": 1024, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 4, + "n": 7168, + "k": 2048, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 4, + "n": 7168, + "k": 3072, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w2" + ] + }, + { + "type": "gemm", + "args": { + "m": 4, + "n": 7168, + "k": 4096, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 4, + "n": 7168, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.e_proj", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.h_proj", + "deepseek-ai/DeepSeek-V4-Pro/mtp.e_proj", + "deepseek-ai/DeepSeek-V4-Pro/mtp.h_proj", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.e_proj", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.h_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 4, + "n": 7168, + "k": 16384, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 4, + "n": 8080, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/head", + "deepseek-ai/DeepSeek-V4-Pro/head", + "sgl-project/DeepSeek-V4-Pro-FP8/head" + ] + }, + { + "type": "gemm", + "args": { + "m": 4, + "n": 8080, + "k": 10240, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 4, + "n": 8080, + "k": 12288, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 4, + "n": 8192, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 4, + "n": 8192, + "k": 10240, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 4, + "n": 10240, + "k": 512, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 4, + "n": 10240, + "k": 1024, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 4, + "n": 10240, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 4, + "n": 10240, + "k": 2048, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 4, + "n": 10240, + "k": 4096, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 4, + "n": 10240, + "k": 16384, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 4, + "n": 12288, + "k": 1024, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 4, + "n": 12288, + "k": 2048, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 4, + "n": 12288, + "k": 8192, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 4, + "n": 12288, + "k": 16384, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 4, + "n": 14336, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 4, + "n": 16160, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/head", + "deepseek-ai/DeepSeek-V4-Pro/head", + "sgl-project/DeepSeek-V4-Pro-FP8/head" + ] + }, + { + "type": "gemm", + "args": { + "m": 4, + "n": 16384, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 4, + "n": 16384, + "k": 12288, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 4, + "n": 32320, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/head", + "deepseek-ai/DeepSeek-V4-Pro/head", + "sgl-project/DeepSeek-V4-Pro-FP8/head" + ] + }, + { + "type": "gemm", + "args": { + "m": 4, + "n": 65536, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 4, + "n": 81920, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 4, + "n": 129280, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/head", + "deepseek-ai/DeepSeek-V4-Pro/head", + "sgl-project/DeepSeek-V4-Pro-FP8/head" + ] + }, + { + "type": "gemm", + "args": { + "m": 4, + "n": 129280, + "k": 10240, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 4, + "n": 129280, + "k": 12288, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 6, + "n": 24, + "k": 28672, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/hc_attn", + "deepseek-ai/DeepSeek-V4-Pro-0813/hc_ffn", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.hc_attn", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.hc_ffn", + "deepseek-ai/DeepSeek-V4-Pro/hc_attn", + "deepseek-ai/DeepSeek-V4-Pro/hc_ffn", + "deepseek-ai/DeepSeek-V4-Pro/mtp.hc_attn", + "deepseek-ai/DeepSeek-V4-Pro/mtp.hc_ffn", + "sgl-project/DeepSeek-V4-Pro-FP8/hc_attn", + "sgl-project/DeepSeek-V4-Pro-FP8/hc_ffn", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.hc_attn", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.hc_ffn" + ] + }, + { + "type": "gemm", + "args": { + "m": 6, + "n": 24, + "k": 40960, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 6, + "n": 24, + "k": 49152, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 6, + "n": 64, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.weights_proj", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.weights_proj", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.weights_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 6, + "n": 64, + "k": 10240, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 6, + "n": 64, + "k": 12288, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 6, + "n": 120, + "k": 12288, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 6, + "n": 192, + "k": 10240, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 6, + "n": 256, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.compressor.wkv", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.compressor.wkv", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.compressor.wgate", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.compressor.wkv" + ] + }, + { + "type": "gemm", + "args": { + "m": 6, + "n": 256, + "k": 10240, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 6, + "n": 256, + "k": 12288, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 6, + "n": 320, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 6, + "n": 384, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.gate", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.gate", + "deepseek-ai/DeepSeek-V4-Pro/ffn.gate", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.gate", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.gate", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.gate" + ] + }, + { + "type": "gemm", + "args": { + "m": 6, + "n": 384, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w3" + ] + }, + { + "type": "gemm", + "args": { + "m": 6, + "n": 512, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wkv", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wkv", + "deepseek-ai/DeepSeek-V4-Pro/attn.wkv", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wkv", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wkv", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wkv" + ] + }, + { + "type": "gemm", + "args": { + "m": 6, + "n": 512, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.compressor.wkv+wgate", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.compressor.wkv+wgate", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.compressor.wkv+wgate" + ] + }, + { + "type": "gemm", + "args": { + "m": 6, + "n": 512, + "k": 10240, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 6, + "n": 512, + "k": 10240, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 6, + "n": 512, + "k": 12288, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 6, + "n": 512, + "k": 12288, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 6, + "n": 768, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w3" + ] + }, + { + "type": "gemm", + "args": { + "m": 6, + "n": 896, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 6, + "n": 1024, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.compressor.wkv", + "deepseek-ai/DeepSeek-V4-Pro/attn.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro/attn.compressor.wkv", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.compressor.wgate", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.compressor.wkv" + ] + }, + { + "type": "gemm", + "args": { + "m": 6, + "n": 1024, + "k": 10240, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 6, + "n": 1024, + "k": 10240, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 6, + "n": 1024, + "k": 12288, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 6, + "n": 1536, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w3" + ] + }, + { + "type": "gemm", + "args": { + "m": 6, + "n": 1536, + "k": 10240, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 6, + "n": 1536, + "k": 12288, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 6, + "n": 1792, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 6, + "n": 4096, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 6, + "n": 5120, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 6, + "n": 6144, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3" + ] + }, + { + "type": "gemm", + "args": { + "m": 6, + "n": 7168, + "k": 192, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w2" + ] + }, + { + "type": "gemm", + "args": { + "m": 6, + "n": 7168, + "k": 384, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w2" + ] + }, + { + "type": "gemm", + "args": { + "m": 6, + "n": 7168, + "k": 448, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 6, + "n": 7168, + "k": 768, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w2" + ] + }, + { + "type": "gemm", + "args": { + "m": 6, + "n": 7168, + "k": 896, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 6, + "n": 7168, + "k": 1024, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 6, + "n": 7168, + "k": 2048, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 6, + "n": 7168, + "k": 3072, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w2" + ] + }, + { + "type": "gemm", + "args": { + "m": 6, + "n": 7168, + "k": 4096, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 6, + "n": 7168, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.e_proj", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.h_proj", + "deepseek-ai/DeepSeek-V4-Pro/mtp.e_proj", + "deepseek-ai/DeepSeek-V4-Pro/mtp.h_proj", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.e_proj", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.h_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 6, + "n": 7168, + "k": 16384, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 6, + "n": 8080, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/head", + "deepseek-ai/DeepSeek-V4-Pro/head", + "sgl-project/DeepSeek-V4-Pro-FP8/head" + ] + }, + { + "type": "gemm", + "args": { + "m": 6, + "n": 8080, + "k": 10240, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 6, + "n": 8080, + "k": 12288, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 6, + "n": 8192, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 6, + "n": 8192, + "k": 10240, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 6, + "n": 10240, + "k": 512, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 6, + "n": 10240, + "k": 1024, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 6, + "n": 10240, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 6, + "n": 10240, + "k": 2048, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 6, + "n": 10240, + "k": 4096, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 6, + "n": 10240, + "k": 16384, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 6, + "n": 12288, + "k": 1024, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 6, + "n": 12288, + "k": 2048, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 6, + "n": 12288, + "k": 8192, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 6, + "n": 12288, + "k": 16384, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 6, + "n": 14336, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 6, + "n": 16160, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/head", + "deepseek-ai/DeepSeek-V4-Pro/head", + "sgl-project/DeepSeek-V4-Pro-FP8/head" + ] + }, + { + "type": "gemm", + "args": { + "m": 6, + "n": 16384, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 6, + "n": 16384, + "k": 12288, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 6, + "n": 32320, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/head", + "deepseek-ai/DeepSeek-V4-Pro/head", + "sgl-project/DeepSeek-V4-Pro-FP8/head" + ] + }, + { + "type": "gemm", + "args": { + "m": 6, + "n": 65536, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 6, + "n": 81920, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 6, + "n": 129280, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/head", + "deepseek-ai/DeepSeek-V4-Pro/head", + "sgl-project/DeepSeek-V4-Pro-FP8/head" + ] + }, + { + "type": "gemm", + "args": { + "m": 6, + "n": 129280, + "k": 10240, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 6, + "n": 129280, + "k": 12288, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 8, + "n": 24, + "k": 28672, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/hc_attn", + "deepseek-ai/DeepSeek-V4-Pro-0813/hc_ffn", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.hc_attn", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.hc_ffn", + "deepseek-ai/DeepSeek-V4-Pro/hc_attn", + "deepseek-ai/DeepSeek-V4-Pro/hc_ffn", + "deepseek-ai/DeepSeek-V4-Pro/mtp.hc_attn", + "deepseek-ai/DeepSeek-V4-Pro/mtp.hc_ffn", + "sgl-project/DeepSeek-V4-Pro-FP8/hc_attn", + "sgl-project/DeepSeek-V4-Pro-FP8/hc_ffn", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.hc_attn", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.hc_ffn" + ] + }, + { + "type": "gemm", + "args": { + "m": 8, + "n": 24, + "k": 40960, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 8, + "n": 24, + "k": 49152, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 8, + "n": 64, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.weights_proj", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.weights_proj", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.weights_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 8, + "n": 64, + "k": 10240, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 8, + "n": 64, + "k": 12288, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 8, + "n": 120, + "k": 12288, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 8, + "n": 192, + "k": 10240, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 8, + "n": 256, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.compressor.wkv", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.compressor.wkv", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.compressor.wgate", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.compressor.wkv" + ] + }, + { + "type": "gemm", + "args": { + "m": 8, + "n": 256, + "k": 10240, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 8, + "n": 256, + "k": 12288, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 8, + "n": 320, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 8, + "n": 384, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.gate", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.gate", + "deepseek-ai/DeepSeek-V4-Pro/ffn.gate", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.gate", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.gate", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.gate" + ] + }, + { + "type": "gemm", + "args": { + "m": 8, + "n": 384, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w3" + ] + }, + { + "type": "gemm", + "args": { + "m": 8, + "n": 512, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wkv", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wkv", + "deepseek-ai/DeepSeek-V4-Pro/attn.wkv", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wkv", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wkv", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wkv" + ] + }, + { + "type": "gemm", + "args": { + "m": 8, + "n": 512, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.compressor.wkv+wgate", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.compressor.wkv+wgate", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.compressor.wkv+wgate" + ] + }, + { + "type": "gemm", + "args": { + "m": 8, + "n": 512, + "k": 10240, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 8, + "n": 512, + "k": 10240, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 8, + "n": 512, + "k": 12288, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 8, + "n": 512, + "k": 12288, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 8, + "n": 768, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w3" + ] + }, + { + "type": "gemm", + "args": { + "m": 8, + "n": 896, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 8, + "n": 1024, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.compressor.wkv", + "deepseek-ai/DeepSeek-V4-Pro/attn.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro/attn.compressor.wkv", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.compressor.wgate", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.compressor.wkv" + ] + }, + { + "type": "gemm", + "args": { + "m": 8, + "n": 1024, + "k": 10240, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 8, + "n": 1024, + "k": 12288, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 8, + "n": 1536, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w3" + ] + }, + { + "type": "gemm", + "args": { + "m": 8, + "n": 1536, + "k": 10240, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 8, + "n": 1536, + "k": 12288, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 8, + "n": 1792, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 8, + "n": 4096, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 8, + "n": 5120, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 8, + "n": 6144, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3" + ] + }, + { + "type": "gemm", + "args": { + "m": 8, + "n": 7168, + "k": 192, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w2" + ] + }, + { + "type": "gemm", + "args": { + "m": 8, + "n": 7168, + "k": 384, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w2" + ] + }, + { + "type": "gemm", + "args": { + "m": 8, + "n": 7168, + "k": 448, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 8, + "n": 7168, + "k": 768, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w2" + ] + }, + { + "type": "gemm", + "args": { + "m": 8, + "n": 7168, + "k": 896, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 8, + "n": 7168, + "k": 1024, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 8, + "n": 7168, + "k": 2048, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 8, + "n": 7168, + "k": 3072, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w2" + ] + }, + { + "type": "gemm", + "args": { + "m": 8, + "n": 7168, + "k": 4096, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 8, + "n": 7168, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.e_proj", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.h_proj", + "deepseek-ai/DeepSeek-V4-Pro/mtp.e_proj", + "deepseek-ai/DeepSeek-V4-Pro/mtp.h_proj", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.e_proj", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.h_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 8, + "n": 7168, + "k": 16384, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 8, + "n": 8080, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/head", + "deepseek-ai/DeepSeek-V4-Pro/head", + "sgl-project/DeepSeek-V4-Pro-FP8/head" + ] + }, + { + "type": "gemm", + "args": { + "m": 8, + "n": 8080, + "k": 10240, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 8, + "n": 8080, + "k": 12288, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 8, + "n": 8192, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 8, + "n": 8192, + "k": 10240, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 8, + "n": 10240, + "k": 1024, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 8, + "n": 10240, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 8, + "n": 10240, + "k": 2048, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 8, + "n": 10240, + "k": 4096, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 8, + "n": 10240, + "k": 16384, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 8, + "n": 12288, + "k": 1024, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 8, + "n": 12288, + "k": 2048, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 8, + "n": 12288, + "k": 8192, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 8, + "n": 12288, + "k": 16384, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 8, + "n": 14336, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 8, + "n": 16160, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/head", + "deepseek-ai/DeepSeek-V4-Pro/head", + "sgl-project/DeepSeek-V4-Pro-FP8/head" + ] + }, + { + "type": "gemm", + "args": { + "m": 8, + "n": 16384, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 8, + "n": 16384, + "k": 12288, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 8, + "n": 32320, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/head", + "deepseek-ai/DeepSeek-V4-Pro/head", + "sgl-project/DeepSeek-V4-Pro-FP8/head" + ] + }, + { + "type": "gemm", + "args": { + "m": 8, + "n": 65536, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 8, + "n": 81920, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 8, + "n": 129280, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/head", + "deepseek-ai/DeepSeek-V4-Pro/head", + "sgl-project/DeepSeek-V4-Pro-FP8/head" + ] + }, + { + "type": "gemm", + "args": { + "m": 8, + "n": 129280, + "k": 10240, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 8, + "n": 129280, + "k": 12288, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 9, + "n": 24, + "k": 28672, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/hc_attn", + "deepseek-ai/DeepSeek-V4-Pro-0813/hc_ffn", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.hc_attn", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.hc_ffn", + "deepseek-ai/DeepSeek-V4-Pro/hc_attn", + "deepseek-ai/DeepSeek-V4-Pro/hc_ffn", + "deepseek-ai/DeepSeek-V4-Pro/mtp.hc_attn", + "deepseek-ai/DeepSeek-V4-Pro/mtp.hc_ffn", + "sgl-project/DeepSeek-V4-Pro-FP8/hc_attn", + "sgl-project/DeepSeek-V4-Pro-FP8/hc_ffn", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.hc_attn", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.hc_ffn" + ] + }, + { + "type": "gemm", + "args": { + "m": 9, + "n": 24, + "k": 40960, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 9, + "n": 24, + "k": 49152, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 9, + "n": 64, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.weights_proj", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.weights_proj", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.weights_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 9, + "n": 64, + "k": 10240, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 9, + "n": 64, + "k": 12288, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 9, + "n": 120, + "k": 12288, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 9, + "n": 192, + "k": 10240, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 9, + "n": 256, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.compressor.wkv", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.compressor.wkv", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.compressor.wgate", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.compressor.wkv" + ] + }, + { + "type": "gemm", + "args": { + "m": 9, + "n": 256, + "k": 10240, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 9, + "n": 256, + "k": 12288, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 9, + "n": 320, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 9, + "n": 384, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.gate", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.gate", + "deepseek-ai/DeepSeek-V4-Pro/ffn.gate", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.gate", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.gate", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.gate" + ] + }, + { + "type": "gemm", + "args": { + "m": 9, + "n": 384, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w3" + ] + }, + { + "type": "gemm", + "args": { + "m": 9, + "n": 512, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wkv", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wkv", + "deepseek-ai/DeepSeek-V4-Pro/attn.wkv", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wkv", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wkv", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wkv" + ] + }, + { + "type": "gemm", + "args": { + "m": 9, + "n": 512, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.compressor.wkv+wgate", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.compressor.wkv+wgate", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.compressor.wkv+wgate" + ] + }, + { + "type": "gemm", + "args": { + "m": 9, + "n": 512, + "k": 10240, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 9, + "n": 512, + "k": 10240, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 9, + "n": 512, + "k": 12288, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 9, + "n": 512, + "k": 12288, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 9, + "n": 768, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w3" + ] + }, + { + "type": "gemm", + "args": { + "m": 9, + "n": 896, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 9, + "n": 1024, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.compressor.wkv", + "deepseek-ai/DeepSeek-V4-Pro/attn.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro/attn.compressor.wkv", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.compressor.wgate", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.compressor.wkv" + ] + }, + { + "type": "gemm", + "args": { + "m": 9, + "n": 1024, + "k": 10240, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 9, + "n": 1024, + "k": 12288, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 9, + "n": 1536, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w3" + ] + }, + { + "type": "gemm", + "args": { + "m": 9, + "n": 1536, + "k": 10240, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 9, + "n": 1536, + "k": 12288, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 9, + "n": 1792, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 9, + "n": 4096, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 9, + "n": 5120, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 9, + "n": 6144, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3" + ] + }, + { + "type": "gemm", + "args": { + "m": 9, + "n": 7168, + "k": 192, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w2" + ] + }, + { + "type": "gemm", + "args": { + "m": 9, + "n": 7168, + "k": 384, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w2" + ] + }, + { + "type": "gemm", + "args": { + "m": 9, + "n": 7168, + "k": 448, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 9, + "n": 7168, + "k": 768, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w2" + ] + }, + { + "type": "gemm", + "args": { + "m": 9, + "n": 7168, + "k": 896, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 9, + "n": 7168, + "k": 1024, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 9, + "n": 7168, + "k": 2048, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 9, + "n": 7168, + "k": 3072, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w2" + ] + }, + { + "type": "gemm", + "args": { + "m": 9, + "n": 7168, + "k": 4096, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 9, + "n": 7168, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.e_proj", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.h_proj", + "deepseek-ai/DeepSeek-V4-Pro/mtp.e_proj", + "deepseek-ai/DeepSeek-V4-Pro/mtp.h_proj", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.e_proj", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.h_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 9, + "n": 7168, + "k": 16384, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 9, + "n": 8080, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/head", + "deepseek-ai/DeepSeek-V4-Pro/head", + "sgl-project/DeepSeek-V4-Pro-FP8/head" + ] + }, + { + "type": "gemm", + "args": { + "m": 9, + "n": 8080, + "k": 10240, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 9, + "n": 8080, + "k": 12288, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 9, + "n": 8192, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 9, + "n": 8192, + "k": 10240, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 9, + "n": 10240, + "k": 1024, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 9, + "n": 10240, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 9, + "n": 10240, + "k": 2048, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 9, + "n": 10240, + "k": 4096, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 9, + "n": 10240, + "k": 16384, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 9, + "n": 12288, + "k": 1024, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 9, + "n": 12288, + "k": 2048, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 9, + "n": 12288, + "k": 8192, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 9, + "n": 12288, + "k": 16384, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 9, + "n": 14336, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 9, + "n": 16160, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/head", + "deepseek-ai/DeepSeek-V4-Pro/head", + "sgl-project/DeepSeek-V4-Pro-FP8/head" + ] + }, + { + "type": "gemm", + "args": { + "m": 9, + "n": 16384, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 9, + "n": 16384, + "k": 12288, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 9, + "n": 32320, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/head", + "deepseek-ai/DeepSeek-V4-Pro/head", + "sgl-project/DeepSeek-V4-Pro-FP8/head" + ] + }, + { + "type": "gemm", + "args": { + "m": 9, + "n": 65536, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 9, + "n": 81920, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 9, + "n": 129280, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/head", + "deepseek-ai/DeepSeek-V4-Pro/head", + "sgl-project/DeepSeek-V4-Pro-FP8/head" + ] + }, + { + "type": "gemm", + "args": { + "m": 9, + "n": 129280, + "k": 10240, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 9, + "n": 129280, + "k": 12288, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 11, + "n": 24, + "k": 28672, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/hc_attn", + "deepseek-ai/DeepSeek-V4-Pro-0813/hc_ffn", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.hc_attn", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.hc_ffn", + "deepseek-ai/DeepSeek-V4-Pro/hc_attn", + "deepseek-ai/DeepSeek-V4-Pro/hc_ffn", + "deepseek-ai/DeepSeek-V4-Pro/mtp.hc_attn", + "deepseek-ai/DeepSeek-V4-Pro/mtp.hc_ffn", + "sgl-project/DeepSeek-V4-Pro-FP8/hc_attn", + "sgl-project/DeepSeek-V4-Pro-FP8/hc_ffn", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.hc_attn", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.hc_ffn" + ] + }, + { + "type": "gemm", + "args": { + "m": 11, + "n": 24, + "k": 40960, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 11, + "n": 24, + "k": 49152, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 11, + "n": 64, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.weights_proj", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.weights_proj", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.weights_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 11, + "n": 64, + "k": 10240, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 11, + "n": 64, + "k": 12288, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 11, + "n": 120, + "k": 12288, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 11, + "n": 192, + "k": 10240, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 11, + "n": 256, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.compressor.wkv", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.compressor.wkv", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.compressor.wgate", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.compressor.wkv" + ] + }, + { + "type": "gemm", + "args": { + "m": 11, + "n": 256, + "k": 10240, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 11, + "n": 256, + "k": 12288, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 11, + "n": 320, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 11, + "n": 384, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.gate", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.gate", + "deepseek-ai/DeepSeek-V4-Pro/ffn.gate", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.gate", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.gate", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.gate" + ] + }, + { + "type": "gemm", + "args": { + "m": 11, + "n": 384, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w3" + ] + }, + { + "type": "gemm", + "args": { + "m": 11, + "n": 512, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wkv", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wkv", + "deepseek-ai/DeepSeek-V4-Pro/attn.wkv", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wkv", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wkv", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wkv" + ] + }, + { + "type": "gemm", + "args": { + "m": 11, + "n": 512, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.compressor.wkv+wgate", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.compressor.wkv+wgate", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.compressor.wkv+wgate" + ] + }, + { + "type": "gemm", + "args": { + "m": 11, + "n": 512, + "k": 10240, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 11, + "n": 512, + "k": 10240, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 11, + "n": 512, + "k": 12288, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 11, + "n": 512, + "k": 12288, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 11, + "n": 768, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w3" + ] + }, + { + "type": "gemm", + "args": { + "m": 11, + "n": 896, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 11, + "n": 1024, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.compressor.wkv", + "deepseek-ai/DeepSeek-V4-Pro/attn.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro/attn.compressor.wkv", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.compressor.wgate", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.compressor.wkv" + ] + }, + { + "type": "gemm", + "args": { + "m": 11, + "n": 1024, + "k": 10240, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 11, + "n": 1024, + "k": 12288, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 11, + "n": 1536, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w3" + ] + }, + { + "type": "gemm", + "args": { + "m": 11, + "n": 1536, + "k": 10240, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 11, + "n": 1536, + "k": 12288, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 11, + "n": 1792, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 11, + "n": 4096, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 11, + "n": 5120, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 11, + "n": 6144, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3" + ] + }, + { + "type": "gemm", + "args": { + "m": 11, + "n": 7168, + "k": 192, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w2" + ] + }, + { + "type": "gemm", + "args": { + "m": 11, + "n": 7168, + "k": 384, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w2" + ] + }, + { + "type": "gemm", + "args": { + "m": 11, + "n": 7168, + "k": 448, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 11, + "n": 7168, + "k": 768, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w2" + ] + }, + { + "type": "gemm", + "args": { + "m": 11, + "n": 7168, + "k": 896, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 11, + "n": 7168, + "k": 1024, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 11, + "n": 7168, + "k": 2048, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 11, + "n": 7168, + "k": 3072, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w2" + ] + }, + { + "type": "gemm", + "args": { + "m": 11, + "n": 7168, + "k": 4096, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 11, + "n": 7168, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.e_proj", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.h_proj", + "deepseek-ai/DeepSeek-V4-Pro/mtp.e_proj", + "deepseek-ai/DeepSeek-V4-Pro/mtp.h_proj", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.e_proj", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.h_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 11, + "n": 7168, + "k": 16384, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 11, + "n": 8080, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/head", + "deepseek-ai/DeepSeek-V4-Pro/head", + "sgl-project/DeepSeek-V4-Pro-FP8/head" + ] + }, + { + "type": "gemm", + "args": { + "m": 11, + "n": 8080, + "k": 10240, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 11, + "n": 8080, + "k": 12288, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 11, + "n": 8192, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 11, + "n": 8192, + "k": 10240, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 11, + "n": 10240, + "k": 1024, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 11, + "n": 10240, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 11, + "n": 10240, + "k": 2048, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 11, + "n": 10240, + "k": 4096, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 11, + "n": 10240, + "k": 16384, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 11, + "n": 12288, + "k": 1024, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 11, + "n": 12288, + "k": 2048, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 11, + "n": 12288, + "k": 8192, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 11, + "n": 12288, + "k": 16384, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 11, + "n": 14336, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 11, + "n": 16160, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/head", + "deepseek-ai/DeepSeek-V4-Pro/head", + "sgl-project/DeepSeek-V4-Pro-FP8/head" + ] + }, + { + "type": "gemm", + "args": { + "m": 11, + "n": 16384, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 11, + "n": 16384, + "k": 12288, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 11, + "n": 32320, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/head", + "deepseek-ai/DeepSeek-V4-Pro/head", + "sgl-project/DeepSeek-V4-Pro-FP8/head" + ] + }, + { + "type": "gemm", + "args": { + "m": 11, + "n": 65536, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 11, + "n": 81920, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 11, + "n": 129280, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/head", + "deepseek-ai/DeepSeek-V4-Pro/head", + "sgl-project/DeepSeek-V4-Pro-FP8/head" + ] + }, + { + "type": "gemm", + "args": { + "m": 11, + "n": 129280, + "k": 10240, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 11, + "n": 129280, + "k": 12288, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 14, + "n": 24, + "k": 28672, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/hc_attn", + "deepseek-ai/DeepSeek-V4-Pro-0813/hc_ffn", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.hc_attn", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.hc_ffn", + "deepseek-ai/DeepSeek-V4-Pro/hc_attn", + "deepseek-ai/DeepSeek-V4-Pro/hc_ffn", + "deepseek-ai/DeepSeek-V4-Pro/mtp.hc_attn", + "deepseek-ai/DeepSeek-V4-Pro/mtp.hc_ffn", + "sgl-project/DeepSeek-V4-Pro-FP8/hc_attn", + "sgl-project/DeepSeek-V4-Pro-FP8/hc_ffn", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.hc_attn", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.hc_ffn" + ] + }, + { + "type": "gemm", + "args": { + "m": 14, + "n": 24, + "k": 40960, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 14, + "n": 24, + "k": 49152, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 14, + "n": 64, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.weights_proj", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.weights_proj", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.weights_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 14, + "n": 64, + "k": 10240, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 14, + "n": 64, + "k": 12288, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 14, + "n": 120, + "k": 12288, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 14, + "n": 192, + "k": 10240, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 14, + "n": 256, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.compressor.wkv", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.compressor.wkv", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.compressor.wgate", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.compressor.wkv" + ] + }, + { + "type": "gemm", + "args": { + "m": 14, + "n": 256, + "k": 10240, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 14, + "n": 256, + "k": 12288, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 14, + "n": 320, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 14, + "n": 384, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.gate", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.gate", + "deepseek-ai/DeepSeek-V4-Pro/ffn.gate", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.gate", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.gate", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.gate" + ] + }, + { + "type": "gemm", + "args": { + "m": 14, + "n": 384, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w3" + ] + }, + { + "type": "gemm", + "args": { + "m": 14, + "n": 512, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wkv", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wkv", + "deepseek-ai/DeepSeek-V4-Pro/attn.wkv", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wkv", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wkv", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wkv" + ] + }, + { + "type": "gemm", + "args": { + "m": 14, + "n": 512, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.compressor.wkv+wgate", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.compressor.wkv+wgate", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.compressor.wkv+wgate" + ] + }, + { + "type": "gemm", + "args": { + "m": 14, + "n": 512, + "k": 10240, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 14, + "n": 512, + "k": 10240, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 14, + "n": 512, + "k": 12288, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 14, + "n": 512, + "k": 12288, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 14, + "n": 768, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w3" + ] + }, + { + "type": "gemm", + "args": { + "m": 14, + "n": 896, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 14, + "n": 1024, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.compressor.wkv", + "deepseek-ai/DeepSeek-V4-Pro/attn.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro/attn.compressor.wkv", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.compressor.wgate", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.compressor.wkv" + ] + }, + { + "type": "gemm", + "args": { + "m": 14, + "n": 1024, + "k": 10240, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 14, + "n": 1024, + "k": 12288, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 14, + "n": 1536, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w3" + ] + }, + { + "type": "gemm", + "args": { + "m": 14, + "n": 1536, + "k": 10240, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 14, + "n": 1536, + "k": 12288, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 14, + "n": 1792, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 14, + "n": 4096, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 14, + "n": 5120, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 14, + "n": 6144, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3" + ] + }, + { + "type": "gemm", + "args": { + "m": 14, + "n": 7168, + "k": 192, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w2" + ] + }, + { + "type": "gemm", + "args": { + "m": 14, + "n": 7168, + "k": 384, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w2" + ] + }, + { + "type": "gemm", + "args": { + "m": 14, + "n": 7168, + "k": 448, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 14, + "n": 7168, + "k": 768, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w2" + ] + }, + { + "type": "gemm", + "args": { + "m": 14, + "n": 7168, + "k": 896, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 14, + "n": 7168, + "k": 1024, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 14, + "n": 7168, + "k": 2048, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 14, + "n": 7168, + "k": 3072, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w2" + ] + }, + { + "type": "gemm", + "args": { + "m": 14, + "n": 7168, + "k": 4096, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 14, + "n": 7168, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.e_proj", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.h_proj", + "deepseek-ai/DeepSeek-V4-Pro/mtp.e_proj", + "deepseek-ai/DeepSeek-V4-Pro/mtp.h_proj", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.e_proj", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.h_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 14, + "n": 7168, + "k": 16384, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 14, + "n": 8080, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/head", + "deepseek-ai/DeepSeek-V4-Pro/head", + "sgl-project/DeepSeek-V4-Pro-FP8/head" + ] + }, + { + "type": "gemm", + "args": { + "m": 14, + "n": 8080, + "k": 10240, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 14, + "n": 8080, + "k": 12288, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 14, + "n": 8192, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 14, + "n": 8192, + "k": 10240, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 14, + "n": 10240, + "k": 1024, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 14, + "n": 10240, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 14, + "n": 10240, + "k": 2048, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 14, + "n": 10240, + "k": 4096, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 14, + "n": 10240, + "k": 16384, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 14, + "n": 12288, + "k": 1024, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 14, + "n": 12288, + "k": 2048, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 14, + "n": 12288, + "k": 8192, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 14, + "n": 12288, + "k": 16384, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 14, + "n": 14336, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 14, + "n": 16160, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/head", + "deepseek-ai/DeepSeek-V4-Pro/head", + "sgl-project/DeepSeek-V4-Pro-FP8/head" + ] + }, + { + "type": "gemm", + "args": { + "m": 14, + "n": 16384, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 14, + "n": 16384, + "k": 12288, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 14, + "n": 32320, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/head", + "deepseek-ai/DeepSeek-V4-Pro/head", + "sgl-project/DeepSeek-V4-Pro-FP8/head" + ] + }, + { + "type": "gemm", + "args": { + "m": 14, + "n": 65536, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 14, + "n": 81920, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 14, + "n": 129280, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/head", + "deepseek-ai/DeepSeek-V4-Pro/head", + "sgl-project/DeepSeek-V4-Pro-FP8/head" + ] + }, + { + "type": "gemm", + "args": { + "m": 14, + "n": 129280, + "k": 10240, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 14, + "n": 129280, + "k": 12288, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 15, + "n": 6144, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3" + ] + }, + { + "type": "gemm", + "args": { + "m": 15, + "n": 7168, + "k": 3072, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w2" + ] + }, + { + "type": "gemm", + "args": { + "m": 16, + "n": 24, + "k": 28672, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/hc_attn", + "deepseek-ai/DeepSeek-V4-Pro-0813/hc_ffn", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.hc_attn", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.hc_ffn", + "deepseek-ai/DeepSeek-V4-Pro/hc_attn", + "deepseek-ai/DeepSeek-V4-Pro/hc_ffn", + "deepseek-ai/DeepSeek-V4-Pro/mtp.hc_attn", + "deepseek-ai/DeepSeek-V4-Pro/mtp.hc_ffn", + "sgl-project/DeepSeek-V4-Pro-FP8/hc_attn", + "sgl-project/DeepSeek-V4-Pro-FP8/hc_ffn", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.hc_attn", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.hc_ffn" + ] + }, + { + "type": "gemm", + "args": { + "m": 16, + "n": 24, + "k": 40960, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 16, + "n": 24, + "k": 49152, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 16, + "n": 64, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.weights_proj", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.weights_proj", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.weights_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 16, + "n": 64, + "k": 10240, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 16, + "n": 64, + "k": 12288, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 16, + "n": 120, + "k": 12288, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 16, + "n": 192, + "k": 10240, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 16, + "n": 256, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.compressor.wkv", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.compressor.wkv", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.compressor.wgate", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.compressor.wkv" + ] + }, + { + "type": "gemm", + "args": { + "m": 16, + "n": 256, + "k": 10240, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 16, + "n": 256, + "k": 12288, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 16, + "n": 320, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 16, + "n": 384, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.gate", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.gate", + "deepseek-ai/DeepSeek-V4-Pro/ffn.gate", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.gate", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.gate", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.gate" + ] + }, + { + "type": "gemm", + "args": { + "m": 16, + "n": 384, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w3" + ] + }, + { + "type": "gemm", + "args": { + "m": 16, + "n": 512, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wkv", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wkv", + "deepseek-ai/DeepSeek-V4-Pro/attn.wkv", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wkv", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wkv", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wkv" + ] + }, + { + "type": "gemm", + "args": { + "m": 16, + "n": 512, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.compressor.wkv+wgate", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.compressor.wkv+wgate", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.compressor.wkv+wgate" + ] + }, + { + "type": "gemm", + "args": { + "m": 16, + "n": 512, + "k": 10240, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 16, + "n": 512, + "k": 10240, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 16, + "n": 512, + "k": 12288, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 16, + "n": 512, + "k": 12288, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 16, + "n": 768, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w3" + ] + }, + { + "type": "gemm", + "args": { + "m": 16, + "n": 896, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 16, + "n": 1024, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.compressor.wkv", + "deepseek-ai/DeepSeek-V4-Pro/attn.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro/attn.compressor.wkv", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.compressor.wgate", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.compressor.wkv" + ] + }, + { + "type": "gemm", + "args": { + "m": 16, + "n": 1024, + "k": 10240, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 16, + "n": 1024, + "k": 12288, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 16, + "n": 1536, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w3" + ] + }, + { + "type": "gemm", + "args": { + "m": 16, + "n": 1536, + "k": 10240, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 16, + "n": 1536, + "k": 12288, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 16, + "n": 1792, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 16, + "n": 4096, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 16, + "n": 5120, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 16, + "n": 6144, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3" + ] + }, + { + "type": "gemm", + "args": { + "m": 16, + "n": 7168, + "k": 192, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w2" + ] + }, + { + "type": "gemm", + "args": { + "m": 16, + "n": 7168, + "k": 384, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w2" + ] + }, + { + "type": "gemm", + "args": { + "m": 16, + "n": 7168, + "k": 448, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 16, + "n": 7168, + "k": 768, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w2" + ] + }, + { + "type": "gemm", + "args": { + "m": 16, + "n": 7168, + "k": 896, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 16, + "n": 7168, + "k": 1024, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 16, + "n": 7168, + "k": 2048, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 16, + "n": 7168, + "k": 3072, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w2" + ] + }, + { + "type": "gemm", + "args": { + "m": 16, + "n": 7168, + "k": 4096, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 16, + "n": 7168, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.e_proj", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.h_proj", + "deepseek-ai/DeepSeek-V4-Pro/mtp.e_proj", + "deepseek-ai/DeepSeek-V4-Pro/mtp.h_proj", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.e_proj", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.h_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 16, + "n": 7168, + "k": 16384, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 16, + "n": 8080, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/head", + "deepseek-ai/DeepSeek-V4-Pro/head", + "sgl-project/DeepSeek-V4-Pro-FP8/head" + ] + }, + { + "type": "gemm", + "args": { + "m": 16, + "n": 8080, + "k": 10240, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 16, + "n": 8080, + "k": 12288, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 16, + "n": 8192, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 16, + "n": 8192, + "k": 10240, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 16, + "n": 10240, + "k": 1024, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 16, + "n": 10240, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 16, + "n": 10240, + "k": 2048, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 16, + "n": 10240, + "k": 4096, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 16, + "n": 10240, + "k": 16384, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 16, + "n": 12288, + "k": 1024, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 16, + "n": 12288, + "k": 2048, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 16, + "n": 12288, + "k": 8192, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 16, + "n": 12288, + "k": 16384, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 16, + "n": 14336, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 16, + "n": 16160, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/head", + "deepseek-ai/DeepSeek-V4-Pro/head", + "sgl-project/DeepSeek-V4-Pro-FP8/head" + ] + }, + { + "type": "gemm", + "args": { + "m": 16, + "n": 16384, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 16, + "n": 16384, + "k": 12288, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 16, + "n": 32320, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/head", + "deepseek-ai/DeepSeek-V4-Pro/head", + "sgl-project/DeepSeek-V4-Pro-FP8/head" + ] + }, + { + "type": "gemm", + "args": { + "m": 16, + "n": 65536, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 16, + "n": 81920, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 16, + "n": 129280, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/head", + "deepseek-ai/DeepSeek-V4-Pro/head", + "sgl-project/DeepSeek-V4-Pro-FP8/head" + ] + }, + { + "type": "gemm", + "args": { + "m": 16, + "n": 129280, + "k": 10240, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 16, + "n": 129280, + "k": 12288, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 18, + "n": 24, + "k": 28672, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/hc_attn", + "deepseek-ai/DeepSeek-V4-Pro-0813/hc_ffn", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.hc_attn", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.hc_ffn", + "deepseek-ai/DeepSeek-V4-Pro/hc_attn", + "deepseek-ai/DeepSeek-V4-Pro/hc_ffn", + "deepseek-ai/DeepSeek-V4-Pro/mtp.hc_attn", + "deepseek-ai/DeepSeek-V4-Pro/mtp.hc_ffn", + "sgl-project/DeepSeek-V4-Pro-FP8/hc_attn", + "sgl-project/DeepSeek-V4-Pro-FP8/hc_ffn", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.hc_attn", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.hc_ffn" + ] + }, + { + "type": "gemm", + "args": { + "m": 18, + "n": 24, + "k": 40960, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 18, + "n": 24, + "k": 49152, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 18, + "n": 64, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.weights_proj", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.weights_proj", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.weights_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 18, + "n": 64, + "k": 10240, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 18, + "n": 64, + "k": 12288, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 18, + "n": 120, + "k": 12288, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 18, + "n": 192, + "k": 10240, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 18, + "n": 256, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.compressor.wkv", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.compressor.wkv", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.compressor.wgate", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.compressor.wkv" + ] + }, + { + "type": "gemm", + "args": { + "m": 18, + "n": 256, + "k": 10240, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 18, + "n": 256, + "k": 12288, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 18, + "n": 320, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 18, + "n": 384, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.gate", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.gate", + "deepseek-ai/DeepSeek-V4-Pro/ffn.gate", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.gate", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.gate", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.gate" + ] + }, + { + "type": "gemm", + "args": { + "m": 18, + "n": 384, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w3" + ] + }, + { + "type": "gemm", + "args": { + "m": 18, + "n": 512, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wkv", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wkv", + "deepseek-ai/DeepSeek-V4-Pro/attn.wkv", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wkv", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wkv", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wkv" + ] + }, + { + "type": "gemm", + "args": { + "m": 18, + "n": 512, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.compressor.wkv+wgate", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.compressor.wkv+wgate", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.compressor.wkv+wgate" + ] + }, + { + "type": "gemm", + "args": { + "m": 18, + "n": 512, + "k": 10240, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 18, + "n": 512, + "k": 10240, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 18, + "n": 512, + "k": 12288, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 18, + "n": 512, + "k": 12288, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 18, + "n": 768, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w3" + ] + }, + { + "type": "gemm", + "args": { + "m": 18, + "n": 896, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 18, + "n": 1024, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.compressor.wkv", + "deepseek-ai/DeepSeek-V4-Pro/attn.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro/attn.compressor.wkv", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.compressor.wgate", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.compressor.wkv" + ] + }, + { + "type": "gemm", + "args": { + "m": 18, + "n": 1024, + "k": 10240, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 18, + "n": 1024, + "k": 12288, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 18, + "n": 1536, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w3" + ] + }, + { + "type": "gemm", + "args": { + "m": 18, + "n": 1536, + "k": 10240, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 18, + "n": 1536, + "k": 12288, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 18, + "n": 1792, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 18, + "n": 4096, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 18, + "n": 5120, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 18, + "n": 6144, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3" + ] + }, + { + "type": "gemm", + "args": { + "m": 18, + "n": 7168, + "k": 192, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w2" + ] + }, + { + "type": "gemm", + "args": { + "m": 18, + "n": 7168, + "k": 384, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w2" + ] + }, + { + "type": "gemm", + "args": { + "m": 18, + "n": 7168, + "k": 448, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 18, + "n": 7168, + "k": 768, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w2" + ] + }, + { + "type": "gemm", + "args": { + "m": 18, + "n": 7168, + "k": 896, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 18, + "n": 7168, + "k": 1024, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 18, + "n": 7168, + "k": 2048, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 18, + "n": 7168, + "k": 3072, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w2" + ] + }, + { + "type": "gemm", + "args": { + "m": 18, + "n": 7168, + "k": 4096, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 18, + "n": 7168, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.e_proj", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.h_proj", + "deepseek-ai/DeepSeek-V4-Pro/mtp.e_proj", + "deepseek-ai/DeepSeek-V4-Pro/mtp.h_proj", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.e_proj", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.h_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 18, + "n": 7168, + "k": 16384, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 18, + "n": 8080, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/head", + "deepseek-ai/DeepSeek-V4-Pro/head", + "sgl-project/DeepSeek-V4-Pro-FP8/head" + ] + }, + { + "type": "gemm", + "args": { + "m": 18, + "n": 8080, + "k": 10240, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 18, + "n": 8080, + "k": 12288, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 18, + "n": 8192, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 18, + "n": 8192, + "k": 10240, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 18, + "n": 10240, + "k": 1024, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 18, + "n": 10240, + "k": 2048, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 18, + "n": 10240, + "k": 4096, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 18, + "n": 10240, + "k": 16384, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 18, + "n": 12288, + "k": 1024, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 18, + "n": 12288, + "k": 8192, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 18, + "n": 12288, + "k": 16384, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 18, + "n": 14336, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 18, + "n": 16160, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/head", + "deepseek-ai/DeepSeek-V4-Pro/head", + "sgl-project/DeepSeek-V4-Pro-FP8/head" + ] + }, + { + "type": "gemm", + "args": { + "m": 18, + "n": 16384, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 18, + "n": 16384, + "k": 12288, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 18, + "n": 32320, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/head", + "deepseek-ai/DeepSeek-V4-Pro/head", + "sgl-project/DeepSeek-V4-Pro-FP8/head" + ] + }, + { + "type": "gemm", + "args": { + "m": 18, + "n": 65536, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 18, + "n": 81920, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 18, + "n": 129280, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/head", + "deepseek-ai/DeepSeek-V4-Pro/head", + "sgl-project/DeepSeek-V4-Pro-FP8/head" + ] + }, + { + "type": "gemm", + "args": { + "m": 18, + "n": 129280, + "k": 10240, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 18, + "n": 129280, + "k": 12288, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 19, + "n": 10240, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 19, + "n": 12288, + "k": 2048, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 22, + "n": 6144, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3" + ] + }, + { + "type": "gemm", + "args": { + "m": 22, + "n": 7168, + "k": 3072, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w2" + ] + }, + { + "type": "gemm", + "args": { + "m": 23, + "n": 24, + "k": 28672, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/hc_attn", + "deepseek-ai/DeepSeek-V4-Pro-0813/hc_ffn", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.hc_attn", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.hc_ffn", + "deepseek-ai/DeepSeek-V4-Pro/hc_attn", + "deepseek-ai/DeepSeek-V4-Pro/hc_ffn", + "deepseek-ai/DeepSeek-V4-Pro/mtp.hc_attn", + "deepseek-ai/DeepSeek-V4-Pro/mtp.hc_ffn", + "sgl-project/DeepSeek-V4-Pro-FP8/hc_attn", + "sgl-project/DeepSeek-V4-Pro-FP8/hc_ffn", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.hc_attn", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.hc_ffn" + ] + }, + { + "type": "gemm", + "args": { + "m": 23, + "n": 24, + "k": 40960, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 23, + "n": 24, + "k": 49152, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 23, + "n": 64, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.weights_proj", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.weights_proj", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.weights_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 23, + "n": 64, + "k": 10240, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 23, + "n": 64, + "k": 12288, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 23, + "n": 120, + "k": 12288, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 23, + "n": 192, + "k": 10240, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 23, + "n": 256, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.compressor.wkv", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.compressor.wkv", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.compressor.wgate", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.compressor.wkv" + ] + }, + { + "type": "gemm", + "args": { + "m": 23, + "n": 256, + "k": 10240, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 23, + "n": 256, + "k": 12288, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 23, + "n": 320, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 23, + "n": 384, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.gate", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.gate", + "deepseek-ai/DeepSeek-V4-Pro/ffn.gate", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.gate", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.gate", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.gate" + ] + }, + { + "type": "gemm", + "args": { + "m": 23, + "n": 384, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w3" + ] + }, + { + "type": "gemm", + "args": { + "m": 23, + "n": 512, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wkv", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wkv", + "deepseek-ai/DeepSeek-V4-Pro/attn.wkv", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wkv", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wkv", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wkv" + ] + }, + { + "type": "gemm", + "args": { + "m": 23, + "n": 512, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.compressor.wkv+wgate", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.compressor.wkv+wgate", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.compressor.wkv+wgate" + ] + }, + { + "type": "gemm", + "args": { + "m": 23, + "n": 512, + "k": 10240, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 23, + "n": 512, + "k": 10240, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 23, + "n": 512, + "k": 12288, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 23, + "n": 512, + "k": 12288, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 23, + "n": 768, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w3" + ] + }, + { + "type": "gemm", + "args": { + "m": 23, + "n": 896, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 23, + "n": 1024, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.compressor.wkv", + "deepseek-ai/DeepSeek-V4-Pro/attn.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro/attn.compressor.wkv", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.compressor.wgate", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.compressor.wkv" + ] + }, + { + "type": "gemm", + "args": { + "m": 23, + "n": 1024, + "k": 10240, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 23, + "n": 1024, + "k": 12288, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 23, + "n": 1536, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w3" + ] + }, + { + "type": "gemm", + "args": { + "m": 23, + "n": 1536, + "k": 10240, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 23, + "n": 1536, + "k": 12288, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 23, + "n": 1792, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 23, + "n": 4096, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 23, + "n": 5120, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 23, + "n": 6144, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3" + ] + }, + { + "type": "gemm", + "args": { + "m": 23, + "n": 7168, + "k": 192, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w2" + ] + }, + { + "type": "gemm", + "args": { + "m": 23, + "n": 7168, + "k": 384, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w2" + ] + }, + { + "type": "gemm", + "args": { + "m": 23, + "n": 7168, + "k": 448, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 23, + "n": 7168, + "k": 768, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w2" + ] + }, + { + "type": "gemm", + "args": { + "m": 23, + "n": 7168, + "k": 896, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 23, + "n": 7168, + "k": 1024, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 23, + "n": 7168, + "k": 2048, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 23, + "n": 7168, + "k": 3072, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w2" + ] + }, + { + "type": "gemm", + "args": { + "m": 23, + "n": 7168, + "k": 4096, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 23, + "n": 7168, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.e_proj", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.h_proj", + "deepseek-ai/DeepSeek-V4-Pro/mtp.e_proj", + "deepseek-ai/DeepSeek-V4-Pro/mtp.h_proj", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.e_proj", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.h_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 23, + "n": 7168, + "k": 16384, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 23, + "n": 8080, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/head", + "deepseek-ai/DeepSeek-V4-Pro/head", + "sgl-project/DeepSeek-V4-Pro-FP8/head" + ] + }, + { + "type": "gemm", + "args": { + "m": 23, + "n": 8080, + "k": 10240, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 23, + "n": 8192, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 23, + "n": 8192, + "k": 10240, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 23, + "n": 10240, + "k": 1024, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 23, + "n": 10240, + "k": 2048, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 23, + "n": 10240, + "k": 4096, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 23, + "n": 10240, + "k": 16384, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 23, + "n": 12288, + "k": 1024, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 23, + "n": 14336, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 23, + "n": 16160, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/head", + "deepseek-ai/DeepSeek-V4-Pro/head", + "sgl-project/DeepSeek-V4-Pro-FP8/head" + ] + }, + { + "type": "gemm", + "args": { + "m": 23, + "n": 16384, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 23, + "n": 32320, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/head", + "deepseek-ai/DeepSeek-V4-Pro/head", + "sgl-project/DeepSeek-V4-Pro-FP8/head" + ] + }, + { + "type": "gemm", + "args": { + "m": 23, + "n": 65536, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 23, + "n": 129280, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/head", + "deepseek-ai/DeepSeek-V4-Pro/head", + "sgl-project/DeepSeek-V4-Pro-FP8/head" + ] + }, + { + "type": "gemm", + "args": { + "m": 23, + "n": 129280, + "k": 10240, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 24, + "n": 24, + "k": 28672, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/hc_attn", + "deepseek-ai/DeepSeek-V4-Pro-0813/hc_ffn", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.hc_attn", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.hc_ffn", + "deepseek-ai/DeepSeek-V4-Pro/hc_attn", + "deepseek-ai/DeepSeek-V4-Pro/hc_ffn", + "deepseek-ai/DeepSeek-V4-Pro/mtp.hc_attn", + "deepseek-ai/DeepSeek-V4-Pro/mtp.hc_ffn", + "sgl-project/DeepSeek-V4-Pro-FP8/hc_attn", + "sgl-project/DeepSeek-V4-Pro-FP8/hc_ffn", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.hc_attn", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.hc_ffn" + ] + }, + { + "type": "gemm", + "args": { + "m": 24, + "n": 24, + "k": 40960, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 24, + "n": 24, + "k": 49152, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 24, + "n": 64, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.weights_proj", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.weights_proj", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.weights_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 24, + "n": 64, + "k": 10240, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 24, + "n": 64, + "k": 12288, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 24, + "n": 120, + "k": 12288, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 24, + "n": 192, + "k": 10240, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 24, + "n": 256, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.compressor.wkv", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.compressor.wkv", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.compressor.wgate", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.compressor.wkv" + ] + }, + { + "type": "gemm", + "args": { + "m": 24, + "n": 256, + "k": 10240, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 24, + "n": 256, + "k": 12288, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 24, + "n": 320, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 24, + "n": 384, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.gate", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.gate", + "deepseek-ai/DeepSeek-V4-Pro/ffn.gate", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.gate", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.gate", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.gate" + ] + }, + { + "type": "gemm", + "args": { + "m": 24, + "n": 384, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w3" + ] + }, + { + "type": "gemm", + "args": { + "m": 24, + "n": 512, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wkv", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wkv", + "deepseek-ai/DeepSeek-V4-Pro/attn.wkv", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wkv", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wkv", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wkv" + ] + }, + { + "type": "gemm", + "args": { + "m": 24, + "n": 512, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.compressor.wkv+wgate", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.compressor.wkv+wgate", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.compressor.wkv+wgate" + ] + }, + { + "type": "gemm", + "args": { + "m": 24, + "n": 512, + "k": 10240, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 24, + "n": 512, + "k": 10240, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 24, + "n": 512, + "k": 12288, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 24, + "n": 512, + "k": 12288, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 24, + "n": 768, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w3" + ] + }, + { + "type": "gemm", + "args": { + "m": 24, + "n": 896, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 24, + "n": 1024, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.compressor.wkv", + "deepseek-ai/DeepSeek-V4-Pro/attn.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro/attn.compressor.wkv", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.compressor.wgate", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.compressor.wkv" + ] + }, + { + "type": "gemm", + "args": { + "m": 24, + "n": 1024, + "k": 10240, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 24, + "n": 1024, + "k": 12288, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 24, + "n": 1536, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w3" + ] + }, + { + "type": "gemm", + "args": { + "m": 24, + "n": 1536, + "k": 10240, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 24, + "n": 1536, + "k": 12288, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 24, + "n": 1792, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 24, + "n": 4096, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 24, + "n": 5120, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 24, + "n": 6144, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3" + ] + }, + { + "type": "gemm", + "args": { + "m": 24, + "n": 7168, + "k": 192, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w2" + ] + }, + { + "type": "gemm", + "args": { + "m": 24, + "n": 7168, + "k": 384, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w2" + ] + }, + { + "type": "gemm", + "args": { + "m": 24, + "n": 7168, + "k": 448, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 24, + "n": 7168, + "k": 768, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w2" + ] + }, + { + "type": "gemm", + "args": { + "m": 24, + "n": 7168, + "k": 896, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 24, + "n": 7168, + "k": 1024, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 24, + "n": 7168, + "k": 2048, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 24, + "n": 7168, + "k": 3072, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w2" + ] + }, + { + "type": "gemm", + "args": { + "m": 24, + "n": 7168, + "k": 4096, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 24, + "n": 7168, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.e_proj", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.h_proj", + "deepseek-ai/DeepSeek-V4-Pro/mtp.e_proj", + "deepseek-ai/DeepSeek-V4-Pro/mtp.h_proj", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.e_proj", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.h_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 24, + "n": 7168, + "k": 16384, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 24, + "n": 8080, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/head", + "deepseek-ai/DeepSeek-V4-Pro/head", + "sgl-project/DeepSeek-V4-Pro-FP8/head" + ] + }, + { + "type": "gemm", + "args": { + "m": 24, + "n": 8080, + "k": 10240, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 24, + "n": 8192, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 24, + "n": 8192, + "k": 10240, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 24, + "n": 10240, + "k": 1024, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 24, + "n": 10240, + "k": 2048, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 24, + "n": 10240, + "k": 4096, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 24, + "n": 10240, + "k": 16384, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 24, + "n": 12288, + "k": 1024, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 24, + "n": 12288, + "k": 8192, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 24, + "n": 12288, + "k": 16384, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 24, + "n": 14336, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 24, + "n": 16160, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/head", + "deepseek-ai/DeepSeek-V4-Pro/head", + "sgl-project/DeepSeek-V4-Pro-FP8/head" + ] + }, + { + "type": "gemm", + "args": { + "m": 24, + "n": 16384, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 24, + "n": 16384, + "k": 12288, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 24, + "n": 32320, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/head", + "deepseek-ai/DeepSeek-V4-Pro/head", + "sgl-project/DeepSeek-V4-Pro-FP8/head" + ] + }, + { + "type": "gemm", + "args": { + "m": 24, + "n": 65536, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 24, + "n": 81920, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 24, + "n": 129280, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/head", + "deepseek-ai/DeepSeek-V4-Pro/head", + "sgl-project/DeepSeek-V4-Pro-FP8/head" + ] + }, + { + "type": "gemm", + "args": { + "m": 24, + "n": 129280, + "k": 10240, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 24, + "n": 129280, + "k": 12288, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 29, + "n": 24, + "k": 28672, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/hc_attn", + "deepseek-ai/DeepSeek-V4-Pro-0813/hc_ffn", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.hc_attn", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.hc_ffn", + "deepseek-ai/DeepSeek-V4-Pro/hc_attn", + "deepseek-ai/DeepSeek-V4-Pro/hc_ffn", + "deepseek-ai/DeepSeek-V4-Pro/mtp.hc_attn", + "deepseek-ai/DeepSeek-V4-Pro/mtp.hc_ffn", + "sgl-project/DeepSeek-V4-Pro-FP8/hc_attn", + "sgl-project/DeepSeek-V4-Pro-FP8/hc_ffn", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.hc_attn", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.hc_ffn" + ] + }, + { + "type": "gemm", + "args": { + "m": 29, + "n": 24, + "k": 40960, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 29, + "n": 24, + "k": 49152, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 29, + "n": 64, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.weights_proj", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.weights_proj", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.weights_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 29, + "n": 64, + "k": 10240, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 29, + "n": 64, + "k": 12288, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 29, + "n": 120, + "k": 12288, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 29, + "n": 192, + "k": 10240, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 29, + "n": 256, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.compressor.wkv", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.compressor.wkv", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.compressor.wgate", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.compressor.wkv" + ] + }, + { + "type": "gemm", + "args": { + "m": 29, + "n": 256, + "k": 10240, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 29, + "n": 256, + "k": 12288, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 29, + "n": 320, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 29, + "n": 384, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.gate", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.gate", + "deepseek-ai/DeepSeek-V4-Pro/ffn.gate", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.gate", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.gate", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.gate" + ] + }, + { + "type": "gemm", + "args": { + "m": 29, + "n": 384, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w3" + ] + }, + { + "type": "gemm", + "args": { + "m": 29, + "n": 512, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wkv", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wkv", + "deepseek-ai/DeepSeek-V4-Pro/attn.wkv", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wkv", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wkv", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wkv" + ] + }, + { + "type": "gemm", + "args": { + "m": 29, + "n": 512, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.compressor.wkv+wgate", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.compressor.wkv+wgate", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.compressor.wkv+wgate" + ] + }, + { + "type": "gemm", + "args": { + "m": 29, + "n": 512, + "k": 10240, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 29, + "n": 512, + "k": 10240, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 29, + "n": 512, + "k": 12288, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 29, + "n": 512, + "k": 12288, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 29, + "n": 768, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w3" + ] + }, + { + "type": "gemm", + "args": { + "m": 29, + "n": 896, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 29, + "n": 1024, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.compressor.wkv", + "deepseek-ai/DeepSeek-V4-Pro/attn.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro/attn.compressor.wkv", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.compressor.wgate", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.compressor.wkv" + ] + }, + { + "type": "gemm", + "args": { + "m": 29, + "n": 1024, + "k": 10240, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 29, + "n": 1024, + "k": 12288, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 29, + "n": 1536, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w3" + ] + }, + { + "type": "gemm", + "args": { + "m": 29, + "n": 1536, + "k": 10240, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 29, + "n": 1536, + "k": 12288, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 29, + "n": 1792, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 29, + "n": 4096, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 29, + "n": 5120, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 29, + "n": 6144, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3" + ] + }, + { + "type": "gemm", + "args": { + "m": 29, + "n": 7168, + "k": 192, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w2" + ] + }, + { + "type": "gemm", + "args": { + "m": 29, + "n": 7168, + "k": 384, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w2" + ] + }, + { + "type": "gemm", + "args": { + "m": 29, + "n": 7168, + "k": 448, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 29, + "n": 7168, + "k": 768, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w2" + ] + }, + { + "type": "gemm", + "args": { + "m": 29, + "n": 7168, + "k": 896, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 29, + "n": 7168, + "k": 1024, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 29, + "n": 7168, + "k": 2048, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 29, + "n": 7168, + "k": 3072, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w2" + ] + }, + { + "type": "gemm", + "args": { + "m": 29, + "n": 7168, + "k": 4096, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 29, + "n": 7168, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.e_proj", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.h_proj", + "deepseek-ai/DeepSeek-V4-Pro/mtp.e_proj", + "deepseek-ai/DeepSeek-V4-Pro/mtp.h_proj", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.e_proj", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.h_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 29, + "n": 7168, + "k": 16384, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 29, + "n": 8080, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/head", + "deepseek-ai/DeepSeek-V4-Pro/head", + "sgl-project/DeepSeek-V4-Pro-FP8/head" + ] + }, + { + "type": "gemm", + "args": { + "m": 29, + "n": 8080, + "k": 10240, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 29, + "n": 8192, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 29, + "n": 8192, + "k": 10240, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 29, + "n": 10240, + "k": 1024, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 29, + "n": 10240, + "k": 2048, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 29, + "n": 10240, + "k": 4096, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 29, + "n": 10240, + "k": 16384, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 29, + "n": 12288, + "k": 1024, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 29, + "n": 14336, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 29, + "n": 16160, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/head", + "deepseek-ai/DeepSeek-V4-Pro/head", + "sgl-project/DeepSeek-V4-Pro-FP8/head" + ] + }, + { + "type": "gemm", + "args": { + "m": 29, + "n": 16384, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 29, + "n": 32320, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/head", + "deepseek-ai/DeepSeek-V4-Pro/head", + "sgl-project/DeepSeek-V4-Pro-FP8/head" + ] + }, + { + "type": "gemm", + "args": { + "m": 29, + "n": 65536, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 29, + "n": 129280, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/head", + "deepseek-ai/DeepSeek-V4-Pro/head", + "sgl-project/DeepSeek-V4-Pro-FP8/head" + ] + }, + { + "type": "gemm", + "args": { + "m": 29, + "n": 129280, + "k": 10240, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 30, + "n": 6144, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3" + ] + }, + { + "type": "gemm", + "args": { + "m": 30, + "n": 7168, + "k": 3072, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w2" + ] + }, + { + "type": "gemm", + "args": { + "m": 32, + "n": 24, + "k": 28672, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/hc_attn", + "deepseek-ai/DeepSeek-V4-Pro-0813/hc_ffn", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.hc_attn", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.hc_ffn", + "deepseek-ai/DeepSeek-V4-Pro/hc_attn", + "deepseek-ai/DeepSeek-V4-Pro/hc_ffn", + "deepseek-ai/DeepSeek-V4-Pro/mtp.hc_attn", + "deepseek-ai/DeepSeek-V4-Pro/mtp.hc_ffn", + "sgl-project/DeepSeek-V4-Pro-FP8/hc_attn", + "sgl-project/DeepSeek-V4-Pro-FP8/hc_ffn", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.hc_attn", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.hc_ffn" + ] + }, + { + "type": "gemm", + "args": { + "m": 32, + "n": 24, + "k": 40960, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 32, + "n": 24, + "k": 49152, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 32, + "n": 64, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.weights_proj", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.weights_proj", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.weights_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 32, + "n": 64, + "k": 10240, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 32, + "n": 64, + "k": 12288, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 32, + "n": 120, + "k": 12288, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 32, + "n": 192, + "k": 10240, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 32, + "n": 256, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.compressor.wkv", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.compressor.wkv", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.compressor.wgate", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.compressor.wkv" + ] + }, + { + "type": "gemm", + "args": { + "m": 32, + "n": 256, + "k": 10240, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 32, + "n": 256, + "k": 12288, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 32, + "n": 320, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 32, + "n": 384, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.gate", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.gate", + "deepseek-ai/DeepSeek-V4-Pro/ffn.gate", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.gate", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.gate", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.gate" + ] + }, + { + "type": "gemm", + "args": { + "m": 32, + "n": 384, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w3" + ] + }, + { + "type": "gemm", + "args": { + "m": 32, + "n": 512, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wkv", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wkv", + "deepseek-ai/DeepSeek-V4-Pro/attn.wkv", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wkv", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wkv", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wkv" + ] + }, + { + "type": "gemm", + "args": { + "m": 32, + "n": 512, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.compressor.wkv+wgate", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.compressor.wkv+wgate", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.compressor.wkv+wgate" + ] + }, + { + "type": "gemm", + "args": { + "m": 32, + "n": 512, + "k": 10240, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 32, + "n": 512, + "k": 10240, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 32, + "n": 512, + "k": 12288, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 32, + "n": 512, + "k": 12288, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 32, + "n": 768, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w3" + ] + }, + { + "type": "gemm", + "args": { + "m": 32, + "n": 896, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 32, + "n": 1024, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.compressor.wkv", + "deepseek-ai/DeepSeek-V4-Pro/attn.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro/attn.compressor.wkv", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.compressor.wgate", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.compressor.wkv" + ] + }, + { + "type": "gemm", + "args": { + "m": 32, + "n": 1024, + "k": 10240, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 32, + "n": 1024, + "k": 12288, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 32, + "n": 1536, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w3" + ] + }, + { + "type": "gemm", + "args": { + "m": 32, + "n": 1536, + "k": 10240, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 32, + "n": 1536, + "k": 12288, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 32, + "n": 1792, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 32, + "n": 4096, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 32, + "n": 5120, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 32, + "n": 6144, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3" + ] + }, + { + "type": "gemm", + "args": { + "m": 32, + "n": 7168, + "k": 192, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w2" + ] + }, + { + "type": "gemm", + "args": { + "m": 32, + "n": 7168, + "k": 384, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w2" + ] + }, + { + "type": "gemm", + "args": { + "m": 32, + "n": 7168, + "k": 448, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 32, + "n": 7168, + "k": 768, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w2" + ] + }, + { + "type": "gemm", + "args": { + "m": 32, + "n": 7168, + "k": 896, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 32, + "n": 7168, + "k": 1024, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 32, + "n": 7168, + "k": 2048, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 32, + "n": 7168, + "k": 3072, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w2" + ] + }, + { + "type": "gemm", + "args": { + "m": 32, + "n": 7168, + "k": 4096, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 32, + "n": 7168, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.e_proj", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.h_proj", + "deepseek-ai/DeepSeek-V4-Pro/mtp.e_proj", + "deepseek-ai/DeepSeek-V4-Pro/mtp.h_proj", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.e_proj", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.h_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 32, + "n": 7168, + "k": 16384, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 32, + "n": 8080, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/head", + "deepseek-ai/DeepSeek-V4-Pro/head", + "sgl-project/DeepSeek-V4-Pro-FP8/head" + ] + }, + { + "type": "gemm", + "args": { + "m": 32, + "n": 8080, + "k": 10240, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 32, + "n": 8192, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 32, + "n": 8192, + "k": 10240, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 32, + "n": 10240, + "k": 1024, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 32, + "n": 10240, + "k": 2048, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 32, + "n": 10240, + "k": 4096, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 32, + "n": 10240, + "k": 16384, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 32, + "n": 12288, + "k": 1024, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 32, + "n": 14336, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 32, + "n": 16160, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/head", + "deepseek-ai/DeepSeek-V4-Pro/head", + "sgl-project/DeepSeek-V4-Pro-FP8/head" + ] + }, + { + "type": "gemm", + "args": { + "m": 32, + "n": 16384, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 32, + "n": 32320, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/head", + "deepseek-ai/DeepSeek-V4-Pro/head", + "sgl-project/DeepSeek-V4-Pro-FP8/head" + ] + }, + { + "type": "gemm", + "args": { + "m": 32, + "n": 65536, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 32, + "n": 129280, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/head", + "deepseek-ai/DeepSeek-V4-Pro/head", + "sgl-project/DeepSeek-V4-Pro-FP8/head" + ] + }, + { + "type": "gemm", + "args": { + "m": 32, + "n": 129280, + "k": 10240, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 37, + "n": 24, + "k": 28672, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/hc_attn", + "deepseek-ai/DeepSeek-V4-Pro-0813/hc_ffn", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.hc_attn", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.hc_ffn", + "deepseek-ai/DeepSeek-V4-Pro/hc_attn", + "deepseek-ai/DeepSeek-V4-Pro/hc_ffn", + "deepseek-ai/DeepSeek-V4-Pro/mtp.hc_attn", + "deepseek-ai/DeepSeek-V4-Pro/mtp.hc_ffn", + "sgl-project/DeepSeek-V4-Pro-FP8/hc_attn", + "sgl-project/DeepSeek-V4-Pro-FP8/hc_ffn", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.hc_attn", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.hc_ffn" + ] + }, + { + "type": "gemm", + "args": { + "m": 37, + "n": 24, + "k": 40960, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 37, + "n": 64, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.weights_proj", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.weights_proj", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.weights_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 37, + "n": 64, + "k": 10240, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 37, + "n": 192, + "k": 10240, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 37, + "n": 256, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.compressor.wkv", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.compressor.wkv", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.compressor.wgate", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.compressor.wkv" + ] + }, + { + "type": "gemm", + "args": { + "m": 37, + "n": 256, + "k": 10240, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 37, + "n": 320, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 37, + "n": 384, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.gate", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.gate", + "deepseek-ai/DeepSeek-V4-Pro/ffn.gate", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.gate", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.gate", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.gate" + ] + }, + { + "type": "gemm", + "args": { + "m": 37, + "n": 384, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w3" + ] + }, + { + "type": "gemm", + "args": { + "m": 37, + "n": 512, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wkv", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wkv", + "deepseek-ai/DeepSeek-V4-Pro/attn.wkv", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wkv", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wkv", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wkv" + ] + }, + { + "type": "gemm", + "args": { + "m": 37, + "n": 512, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.compressor.wkv+wgate", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.compressor.wkv+wgate", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.compressor.wkv+wgate" + ] + }, + { + "type": "gemm", + "args": { + "m": 37, + "n": 512, + "k": 10240, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 37, + "n": 512, + "k": 10240, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 37, + "n": 768, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w3" + ] + }, + { + "type": "gemm", + "args": { + "m": 37, + "n": 896, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 37, + "n": 1024, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.compressor.wkv", + "deepseek-ai/DeepSeek-V4-Pro/attn.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro/attn.compressor.wkv", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.compressor.wgate", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.compressor.wkv" + ] + }, + { + "type": "gemm", + "args": { + "m": 37, + "n": 1024, + "k": 10240, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 37, + "n": 1536, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w3" + ] + }, + { + "type": "gemm", + "args": { + "m": 37, + "n": 1536, + "k": 10240, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 37, + "n": 1792, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 37, + "n": 4096, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 37, + "n": 7168, + "k": 192, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w2" + ] + }, + { + "type": "gemm", + "args": { + "m": 37, + "n": 7168, + "k": 384, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w2" + ] + }, + { + "type": "gemm", + "args": { + "m": 37, + "n": 7168, + "k": 448, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 37, + "n": 7168, + "k": 768, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w2" + ] + }, + { + "type": "gemm", + "args": { + "m": 37, + "n": 7168, + "k": 896, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 37, + "n": 7168, + "k": 1024, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 37, + "n": 7168, + "k": 2048, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 37, + "n": 7168, + "k": 4096, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 37, + "n": 8080, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/head", + "deepseek-ai/DeepSeek-V4-Pro/head", + "sgl-project/DeepSeek-V4-Pro-FP8/head" + ] + }, + { + "type": "gemm", + "args": { + "m": 37, + "n": 8080, + "k": 10240, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 37, + "n": 8192, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 37, + "n": 8192, + "k": 10240, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 37, + "n": 10240, + "k": 1024, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 37, + "n": 10240, + "k": 4096, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 37, + "n": 10240, + "k": 16384, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 37, + "n": 16160, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/head", + "deepseek-ai/DeepSeek-V4-Pro/head", + "sgl-project/DeepSeek-V4-Pro-FP8/head" + ] + }, + { + "type": "gemm", + "args": { + "m": 37, + "n": 16384, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 37, + "n": 65536, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 37, + "n": 129280, + "k": 10240, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 38, + "n": 6144, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3" + ] + }, + { + "type": "gemm", + "args": { + "m": 38, + "n": 7168, + "k": 3072, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w2" + ] + }, + { + "type": "gemm", + "args": { + "m": 38, + "n": 7168, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.e_proj", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.h_proj", + "deepseek-ai/DeepSeek-V4-Pro/mtp.e_proj", + "deepseek-ai/DeepSeek-V4-Pro/mtp.h_proj", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.e_proj", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.h_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 38, + "n": 7168, + "k": 16384, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 38, + "n": 14336, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 38, + "n": 129280, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/head", + "deepseek-ai/DeepSeek-V4-Pro/head", + "sgl-project/DeepSeek-V4-Pro-FP8/head" + ] + }, + { + "type": "gemm", + "args": { + "m": 40, + "n": 24, + "k": 28672, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/hc_attn", + "deepseek-ai/DeepSeek-V4-Pro-0813/hc_ffn", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.hc_attn", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.hc_ffn", + "deepseek-ai/DeepSeek-V4-Pro/hc_attn", + "deepseek-ai/DeepSeek-V4-Pro/hc_ffn", + "deepseek-ai/DeepSeek-V4-Pro/mtp.hc_attn", + "deepseek-ai/DeepSeek-V4-Pro/mtp.hc_ffn", + "sgl-project/DeepSeek-V4-Pro-FP8/hc_attn", + "sgl-project/DeepSeek-V4-Pro-FP8/hc_ffn", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.hc_attn", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.hc_ffn" + ] + }, + { + "type": "gemm", + "args": { + "m": 40, + "n": 24, + "k": 40960, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 40, + "n": 64, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.weights_proj", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.weights_proj", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.weights_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 40, + "n": 64, + "k": 10240, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 40, + "n": 192, + "k": 10240, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 40, + "n": 256, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.compressor.wkv", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.compressor.wkv", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.compressor.wgate", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.compressor.wkv" + ] + }, + { + "type": "gemm", + "args": { + "m": 40, + "n": 256, + "k": 10240, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 40, + "n": 320, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 40, + "n": 384, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.gate", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.gate", + "deepseek-ai/DeepSeek-V4-Pro/ffn.gate", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.gate", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.gate", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.gate" + ] + }, + { + "type": "gemm", + "args": { + "m": 40, + "n": 384, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w3" + ] + }, + { + "type": "gemm", + "args": { + "m": 40, + "n": 512, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wkv", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wkv", + "deepseek-ai/DeepSeek-V4-Pro/attn.wkv", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wkv", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wkv", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wkv" + ] + }, + { + "type": "gemm", + "args": { + "m": 40, + "n": 512, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.compressor.wkv+wgate", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.compressor.wkv+wgate", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.compressor.wkv+wgate" + ] + }, + { + "type": "gemm", + "args": { + "m": 40, + "n": 512, + "k": 10240, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 40, + "n": 512, + "k": 10240, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 40, + "n": 768, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w3" + ] + }, + { + "type": "gemm", + "args": { + "m": 40, + "n": 896, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 40, + "n": 1024, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.compressor.wkv", + "deepseek-ai/DeepSeek-V4-Pro/attn.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro/attn.compressor.wkv", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.compressor.wgate", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.compressor.wkv" + ] + }, + { + "type": "gemm", + "args": { + "m": 40, + "n": 1024, + "k": 10240, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 40, + "n": 1536, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w3" + ] + }, + { + "type": "gemm", + "args": { + "m": 40, + "n": 1536, + "k": 10240, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 40, + "n": 1792, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 40, + "n": 4096, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 40, + "n": 6144, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3" + ] + }, + { + "type": "gemm", + "args": { + "m": 40, + "n": 7168, + "k": 192, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w2" + ] + }, + { + "type": "gemm", + "args": { + "m": 40, + "n": 7168, + "k": 384, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w2" + ] + }, + { + "type": "gemm", + "args": { + "m": 40, + "n": 7168, + "k": 448, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 40, + "n": 7168, + "k": 768, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w2" + ] + }, + { + "type": "gemm", + "args": { + "m": 40, + "n": 7168, + "k": 896, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 40, + "n": 7168, + "k": 1024, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 40, + "n": 7168, + "k": 2048, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 40, + "n": 7168, + "k": 3072, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w2" + ] + }, + { + "type": "gemm", + "args": { + "m": 40, + "n": 7168, + "k": 4096, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 40, + "n": 7168, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.e_proj", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.h_proj", + "deepseek-ai/DeepSeek-V4-Pro/mtp.e_proj", + "deepseek-ai/DeepSeek-V4-Pro/mtp.h_proj", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.e_proj", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.h_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 40, + "n": 7168, + "k": 16384, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 40, + "n": 8080, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/head", + "deepseek-ai/DeepSeek-V4-Pro/head", + "sgl-project/DeepSeek-V4-Pro-FP8/head" + ] + }, + { + "type": "gemm", + "args": { + "m": 40, + "n": 8080, + "k": 10240, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 40, + "n": 8192, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 40, + "n": 8192, + "k": 10240, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 40, + "n": 10240, + "k": 1024, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 40, + "n": 10240, + "k": 4096, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 40, + "n": 10240, + "k": 16384, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 40, + "n": 14336, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 40, + "n": 16160, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/head", + "deepseek-ai/DeepSeek-V4-Pro/head", + "sgl-project/DeepSeek-V4-Pro-FP8/head" + ] + }, + { + "type": "gemm", + "args": { + "m": 40, + "n": 16384, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 40, + "n": 65536, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 40, + "n": 129280, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/head", + "deepseek-ai/DeepSeek-V4-Pro/head", + "sgl-project/DeepSeek-V4-Pro-FP8/head" + ] + }, + { + "type": "gemm", + "args": { + "m": 40, + "n": 129280, + "k": 10240, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 41, + "n": 6144, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3" + ] + }, + { + "type": "gemm", + "args": { + "m": 41, + "n": 7168, + "k": 3072, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w2" + ] + }, + { + "type": "gemm", + "args": { + "m": 41, + "n": 32320, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/head", + "deepseek-ai/DeepSeek-V4-Pro/head", + "sgl-project/DeepSeek-V4-Pro-FP8/head" + ] + }, + { + "type": "gemm", + "args": { + "m": 47, + "n": 24, + "k": 28672, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/hc_attn", + "deepseek-ai/DeepSeek-V4-Pro-0813/hc_ffn", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.hc_attn", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.hc_ffn", + "deepseek-ai/DeepSeek-V4-Pro/hc_attn", + "deepseek-ai/DeepSeek-V4-Pro/hc_ffn", + "deepseek-ai/DeepSeek-V4-Pro/mtp.hc_attn", + "deepseek-ai/DeepSeek-V4-Pro/mtp.hc_ffn", + "sgl-project/DeepSeek-V4-Pro-FP8/hc_attn", + "sgl-project/DeepSeek-V4-Pro-FP8/hc_ffn", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.hc_attn", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.hc_ffn" + ] + }, + { + "type": "gemm", + "args": { + "m": 47, + "n": 64, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.weights_proj", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.weights_proj", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.weights_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 47, + "n": 256, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.compressor.wkv", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.compressor.wkv", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.compressor.wgate", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.compressor.wkv" + ] + }, + { + "type": "gemm", + "args": { + "m": 47, + "n": 320, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 47, + "n": 384, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.gate", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.gate", + "deepseek-ai/DeepSeek-V4-Pro/ffn.gate", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.gate", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.gate", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.gate" + ] + }, + { + "type": "gemm", + "args": { + "m": 47, + "n": 384, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w3" + ] + }, + { + "type": "gemm", + "args": { + "m": 47, + "n": 512, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wkv", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wkv", + "deepseek-ai/DeepSeek-V4-Pro/attn.wkv", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wkv", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wkv", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wkv" + ] + }, + { + "type": "gemm", + "args": { + "m": 47, + "n": 512, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.compressor.wkv+wgate", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.compressor.wkv+wgate", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.compressor.wkv+wgate" + ] + }, + { + "type": "gemm", + "args": { + "m": 47, + "n": 768, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w3" + ] + }, + { + "type": "gemm", + "args": { + "m": 47, + "n": 896, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 47, + "n": 1024, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.compressor.wkv", + "deepseek-ai/DeepSeek-V4-Pro/attn.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro/attn.compressor.wkv", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.compressor.wgate", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.compressor.wkv" + ] + }, + { + "type": "gemm", + "args": { + "m": 47, + "n": 1536, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w3" + ] + }, + { + "type": "gemm", + "args": { + "m": 47, + "n": 1792, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 47, + "n": 4096, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 47, + "n": 7168, + "k": 192, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w2" + ] + }, + { + "type": "gemm", + "args": { + "m": 47, + "n": 7168, + "k": 384, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w2" + ] + }, + { + "type": "gemm", + "args": { + "m": 47, + "n": 7168, + "k": 448, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 47, + "n": 7168, + "k": 768, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w2" + ] + }, + { + "type": "gemm", + "args": { + "m": 47, + "n": 7168, + "k": 896, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 47, + "n": 7168, + "k": 1024, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 47, + "n": 7168, + "k": 2048, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 47, + "n": 7168, + "k": 4096, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 47, + "n": 8080, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/head", + "deepseek-ai/DeepSeek-V4-Pro/head", + "sgl-project/DeepSeek-V4-Pro-FP8/head" + ] + }, + { + "type": "gemm", + "args": { + "m": 47, + "n": 8192, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 47, + "n": 16160, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/head", + "deepseek-ai/DeepSeek-V4-Pro/head", + "sgl-project/DeepSeek-V4-Pro-FP8/head" + ] + }, + { + "type": "gemm", + "args": { + "m": 47, + "n": 16384, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 47, + "n": 65536, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 48, + "n": 24, + "k": 28672, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/hc_attn", + "deepseek-ai/DeepSeek-V4-Pro-0813/hc_ffn", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.hc_attn", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.hc_ffn", + "deepseek-ai/DeepSeek-V4-Pro/hc_attn", + "deepseek-ai/DeepSeek-V4-Pro/hc_ffn", + "deepseek-ai/DeepSeek-V4-Pro/mtp.hc_attn", + "deepseek-ai/DeepSeek-V4-Pro/mtp.hc_ffn", + "sgl-project/DeepSeek-V4-Pro-FP8/hc_attn", + "sgl-project/DeepSeek-V4-Pro-FP8/hc_ffn", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.hc_attn", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.hc_ffn" + ] + }, + { + "type": "gemm", + "args": { + "m": 48, + "n": 64, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.weights_proj", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.weights_proj", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.weights_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 48, + "n": 256, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.compressor.wkv", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.compressor.wkv", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.compressor.wgate", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.compressor.wkv" + ] + }, + { + "type": "gemm", + "args": { + "m": 48, + "n": 320, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 48, + "n": 384, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.gate", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.gate", + "deepseek-ai/DeepSeek-V4-Pro/ffn.gate", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.gate", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.gate", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.gate" + ] + }, + { + "type": "gemm", + "args": { + "m": 48, + "n": 384, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w3" + ] + }, + { + "type": "gemm", + "args": { + "m": 48, + "n": 512, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wkv", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wkv", + "deepseek-ai/DeepSeek-V4-Pro/attn.wkv", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wkv", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wkv", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wkv" + ] + }, + { + "type": "gemm", + "args": { + "m": 48, + "n": 512, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.compressor.wkv+wgate", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.compressor.wkv+wgate", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.compressor.wkv+wgate" + ] + }, + { + "type": "gemm", + "args": { + "m": 48, + "n": 768, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w3" + ] + }, + { + "type": "gemm", + "args": { + "m": 48, + "n": 896, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 48, + "n": 1024, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.compressor.wkv", + "deepseek-ai/DeepSeek-V4-Pro/attn.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro/attn.compressor.wkv", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.compressor.wgate", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.compressor.wkv" + ] + }, + { + "type": "gemm", + "args": { + "m": 48, + "n": 1536, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w3" + ] + }, + { + "type": "gemm", + "args": { + "m": 48, + "n": 4096, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 48, + "n": 6144, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3" + ] + }, + { + "type": "gemm", + "args": { + "m": 48, + "n": 7168, + "k": 192, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w2" + ] + }, + { + "type": "gemm", + "args": { + "m": 48, + "n": 7168, + "k": 384, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w2" + ] + }, + { + "type": "gemm", + "args": { + "m": 48, + "n": 7168, + "k": 448, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 48, + "n": 7168, + "k": 768, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w2" + ] + }, + { + "type": "gemm", + "args": { + "m": 48, + "n": 7168, + "k": 1024, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 48, + "n": 7168, + "k": 2048, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 48, + "n": 7168, + "k": 3072, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w2" + ] + }, + { + "type": "gemm", + "args": { + "m": 48, + "n": 7168, + "k": 4096, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 48, + "n": 7168, + "k": 16384, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 48, + "n": 8080, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/head", + "deepseek-ai/DeepSeek-V4-Pro/head", + "sgl-project/DeepSeek-V4-Pro-FP8/head" + ] + }, + { + "type": "gemm", + "args": { + "m": 48, + "n": 8192, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 48, + "n": 16160, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/head", + "deepseek-ai/DeepSeek-V4-Pro/head", + "sgl-project/DeepSeek-V4-Pro-FP8/head" + ] + }, + { + "type": "gemm", + "args": { + "m": 48, + "n": 16384, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 48, + "n": 32320, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/head", + "deepseek-ai/DeepSeek-V4-Pro/head", + "sgl-project/DeepSeek-V4-Pro-FP8/head" + ] + }, + { + "type": "gemm", + "args": { + "m": 48, + "n": 65536, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 49, + "n": 6144, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3" + ] + }, + { + "type": "gemm", + "args": { + "m": 49, + "n": 7168, + "k": 3072, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w2" + ] + }, + { + "type": "gemm", + "args": { + "m": 49, + "n": 7168, + "k": 16384, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 49, + "n": 129280, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/head", + "deepseek-ai/DeepSeek-V4-Pro/head", + "sgl-project/DeepSeek-V4-Pro-FP8/head" + ] + }, + { + "type": "gemm", + "args": { + "m": 53, + "n": 6144, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3" + ] + }, + { + "type": "gemm", + "args": { + "m": 53, + "n": 7168, + "k": 3072, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w2" + ] + }, + { + "type": "gemm", + "args": { + "m": 54, + "n": 32320, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/head", + "deepseek-ai/DeepSeek-V4-Pro/head", + "sgl-project/DeepSeek-V4-Pro-FP8/head" + ] + }, + { + "type": "gemm", + "args": { + "m": 55, + "n": 24, + "k": 40960, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 55, + "n": 64, + "k": 10240, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 55, + "n": 192, + "k": 10240, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 55, + "n": 256, + "k": 10240, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 55, + "n": 512, + "k": 10240, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 55, + "n": 512, + "k": 10240, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 55, + "n": 1024, + "k": 10240, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 55, + "n": 1024, + "k": 10240, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 55, + "n": 1536, + "k": 10240, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 55, + "n": 10240, + "k": 512, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 55, + "n": 10240, + "k": 2048, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 56, + "n": 24, + "k": 28672, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/hc_attn", + "deepseek-ai/DeepSeek-V4-Pro-0813/hc_ffn", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.hc_attn", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.hc_ffn", + "deepseek-ai/DeepSeek-V4-Pro/hc_attn", + "deepseek-ai/DeepSeek-V4-Pro/hc_ffn", + "deepseek-ai/DeepSeek-V4-Pro/mtp.hc_attn", + "deepseek-ai/DeepSeek-V4-Pro/mtp.hc_ffn", + "sgl-project/DeepSeek-V4-Pro-FP8/hc_attn", + "sgl-project/DeepSeek-V4-Pro-FP8/hc_ffn", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.hc_attn", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.hc_ffn" + ] + }, + { + "type": "gemm", + "args": { + "m": 56, + "n": 64, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.weights_proj", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.weights_proj", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.weights_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 56, + "n": 256, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.compressor.wkv", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.compressor.wkv", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.compressor.wgate", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.compressor.wkv" + ] + }, + { + "type": "gemm", + "args": { + "m": 56, + "n": 320, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 56, + "n": 384, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.gate", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.gate", + "deepseek-ai/DeepSeek-V4-Pro/ffn.gate", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.gate", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.gate", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.gate" + ] + }, + { + "type": "gemm", + "args": { + "m": 56, + "n": 384, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w3" + ] + }, + { + "type": "gemm", + "args": { + "m": 56, + "n": 512, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wkv", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wkv", + "deepseek-ai/DeepSeek-V4-Pro/attn.wkv", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wkv", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wkv", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wkv" + ] + }, + { + "type": "gemm", + "args": { + "m": 56, + "n": 512, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.compressor.wkv+wgate", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.compressor.wkv+wgate", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.compressor.wkv+wgate" + ] + }, + { + "type": "gemm", + "args": { + "m": 56, + "n": 896, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 56, + "n": 1024, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.compressor.wkv", + "deepseek-ai/DeepSeek-V4-Pro/attn.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro/attn.compressor.wkv", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.compressor.wgate", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.compressor.wkv" + ] + }, + { + "type": "gemm", + "args": { + "m": 56, + "n": 1536, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w3" + ] + }, + { + "type": "gemm", + "args": { + "m": 56, + "n": 4096, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 56, + "n": 7168, + "k": 192, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w2" + ] + }, + { + "type": "gemm", + "args": { + "m": 56, + "n": 7168, + "k": 448, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 56, + "n": 7168, + "k": 768, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w2" + ] + }, + { + "type": "gemm", + "args": { + "m": 56, + "n": 7168, + "k": 1024, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 56, + "n": 7168, + "k": 4096, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 56, + "n": 8080, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/head", + "deepseek-ai/DeepSeek-V4-Pro/head", + "sgl-project/DeepSeek-V4-Pro-FP8/head" + ] + }, + { + "type": "gemm", + "args": { + "m": 56, + "n": 8192, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 56, + "n": 16384, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 59, + "n": 24, + "k": 28672, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/hc_attn", + "deepseek-ai/DeepSeek-V4-Pro-0813/hc_ffn", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.hc_attn", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.hc_ffn", + "deepseek-ai/DeepSeek-V4-Pro/hc_attn", + "deepseek-ai/DeepSeek-V4-Pro/hc_ffn", + "deepseek-ai/DeepSeek-V4-Pro/mtp.hc_attn", + "deepseek-ai/DeepSeek-V4-Pro/mtp.hc_ffn", + "sgl-project/DeepSeek-V4-Pro-FP8/hc_attn", + "sgl-project/DeepSeek-V4-Pro-FP8/hc_ffn", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.hc_attn", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.hc_ffn" + ] + }, + { + "type": "gemm", + "args": { + "m": 59, + "n": 64, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.weights_proj", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.weights_proj", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.weights_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 59, + "n": 256, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.compressor.wkv", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.compressor.wkv", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.compressor.wgate", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.compressor.wkv" + ] + }, + { + "type": "gemm", + "args": { + "m": 59, + "n": 384, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.gate", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.gate", + "deepseek-ai/DeepSeek-V4-Pro/ffn.gate", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.gate", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.gate", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.gate" + ] + }, + { + "type": "gemm", + "args": { + "m": 59, + "n": 384, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w3" + ] + }, + { + "type": "gemm", + "args": { + "m": 59, + "n": 512, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wkv", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wkv", + "deepseek-ai/DeepSeek-V4-Pro/attn.wkv", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wkv", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wkv", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wkv" + ] + }, + { + "type": "gemm", + "args": { + "m": 59, + "n": 512, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.compressor.wkv+wgate", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.compressor.wkv+wgate", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.compressor.wkv+wgate" + ] + }, + { + "type": "gemm", + "args": { + "m": 59, + "n": 1024, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.compressor.wkv", + "deepseek-ai/DeepSeek-V4-Pro/attn.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro/attn.compressor.wkv", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.compressor.wgate", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.compressor.wkv" + ] + }, + { + "type": "gemm", + "args": { + "m": 59, + "n": 1536, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w3" + ] + }, + { + "type": "gemm", + "args": { + "m": 59, + "n": 4096, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 59, + "n": 7168, + "k": 192, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w2" + ] + }, + { + "type": "gemm", + "args": { + "m": 59, + "n": 7168, + "k": 768, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w2" + ] + }, + { + "type": "gemm", + "args": { + "m": 59, + "n": 7168, + "k": 1024, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 59, + "n": 7168, + "k": 4096, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 59, + "n": 8080, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/head", + "deepseek-ai/DeepSeek-V4-Pro/head", + "sgl-project/DeepSeek-V4-Pro-FP8/head" + ] + }, + { + "type": "gemm", + "args": { + "m": 59, + "n": 8192, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 59, + "n": 16384, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 59, + "n": 65536, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 61, + "n": 320, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 61, + "n": 896, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 61, + "n": 6144, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3" + ] + }, + { + "type": "gemm", + "args": { + "m": 61, + "n": 7168, + "k": 448, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 61, + "n": 7168, + "k": 3072, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w2" + ] + }, + { + "type": "gemm", + "args": { + "m": 61, + "n": 7168, + "k": 16384, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 64, + "n": 24, + "k": 28672, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/hc_attn", + "deepseek-ai/DeepSeek-V4-Pro-0813/hc_ffn", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.hc_attn", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.hc_ffn", + "deepseek-ai/DeepSeek-V4-Pro/hc_attn", + "deepseek-ai/DeepSeek-V4-Pro/hc_ffn", + "deepseek-ai/DeepSeek-V4-Pro/mtp.hc_attn", + "deepseek-ai/DeepSeek-V4-Pro/mtp.hc_ffn", + "sgl-project/DeepSeek-V4-Pro-FP8/hc_attn", + "sgl-project/DeepSeek-V4-Pro-FP8/hc_ffn", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.hc_attn", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.hc_ffn" + ] + }, + { + "type": "gemm", + "args": { + "m": 64, + "n": 24, + "k": 40960, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 64, + "n": 24, + "k": 49152, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 64, + "n": 64, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.weights_proj", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.weights_proj", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.weights_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 64, + "n": 64, + "k": 10240, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 64, + "n": 64, + "k": 12288, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 64, + "n": 120, + "k": 12288, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 64, + "n": 192, + "k": 10240, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 64, + "n": 256, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.compressor.wkv", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.compressor.wkv", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.compressor.wgate", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.compressor.wkv" + ] + }, + { + "type": "gemm", + "args": { + "m": 64, + "n": 256, + "k": 10240, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 64, + "n": 256, + "k": 12288, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 64, + "n": 320, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 64, + "n": 384, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.gate", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.gate", + "deepseek-ai/DeepSeek-V4-Pro/ffn.gate", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.gate", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.gate", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.gate" + ] + }, + { + "type": "gemm", + "args": { + "m": 64, + "n": 384, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w3" + ] + }, + { + "type": "gemm", + "args": { + "m": 64, + "n": 512, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wkv", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wkv", + "deepseek-ai/DeepSeek-V4-Pro/attn.wkv", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wkv", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wkv", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wkv" + ] + }, + { + "type": "gemm", + "args": { + "m": 64, + "n": 512, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.compressor.wkv+wgate", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.compressor.wkv+wgate", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.compressor.wkv+wgate" + ] + }, + { + "type": "gemm", + "args": { + "m": 64, + "n": 512, + "k": 10240, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 64, + "n": 512, + "k": 10240, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 64, + "n": 512, + "k": 12288, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 64, + "n": 512, + "k": 12288, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 64, + "n": 768, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w3" + ] + }, + { + "type": "gemm", + "args": { + "m": 64, + "n": 896, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 64, + "n": 1024, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.compressor.wkv", + "deepseek-ai/DeepSeek-V4-Pro/attn.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro/attn.compressor.wkv", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.compressor.wgate", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.compressor.wkv" + ] + }, + { + "type": "gemm", + "args": { + "m": 64, + "n": 1024, + "k": 10240, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 64, + "n": 1024, + "k": 10240, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 64, + "n": 1024, + "k": 12288, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 64, + "n": 1536, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w3" + ] + }, + { + "type": "gemm", + "args": { + "m": 64, + "n": 1536, + "k": 10240, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 64, + "n": 1536, + "k": 12288, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 64, + "n": 1792, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 64, + "n": 2048, + "k": 12288, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 64, + "n": 4096, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 64, + "n": 5120, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 64, + "n": 6144, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3" + ] + }, + { + "type": "gemm", + "args": { + "m": 64, + "n": 7168, + "k": 192, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w2" + ] + }, + { + "type": "gemm", + "args": { + "m": 64, + "n": 7168, + "k": 384, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w2" + ] + }, + { + "type": "gemm", + "args": { + "m": 64, + "n": 7168, + "k": 448, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 64, + "n": 7168, + "k": 768, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w2" + ] + }, + { + "type": "gemm", + "args": { + "m": 64, + "n": 7168, + "k": 896, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 64, + "n": 7168, + "k": 1024, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 64, + "n": 7168, + "k": 2048, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 64, + "n": 7168, + "k": 3072, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w2" + ] + }, + { + "type": "gemm", + "args": { + "m": 64, + "n": 7168, + "k": 4096, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 64, + "n": 7168, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.e_proj", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.h_proj", + "deepseek-ai/DeepSeek-V4-Pro/mtp.e_proj", + "deepseek-ai/DeepSeek-V4-Pro/mtp.h_proj", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.e_proj", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.h_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 64, + "n": 7168, + "k": 16384, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 64, + "n": 8080, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/head", + "deepseek-ai/DeepSeek-V4-Pro/head", + "sgl-project/DeepSeek-V4-Pro-FP8/head" + ] + }, + { + "type": "gemm", + "args": { + "m": 64, + "n": 8192, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 64, + "n": 10240, + "k": 512, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 64, + "n": 10240, + "k": 1024, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 64, + "n": 10240, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 64, + "n": 10240, + "k": 2048, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 64, + "n": 10240, + "k": 16384, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 64, + "n": 12288, + "k": 1024, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 64, + "n": 12288, + "k": 2048, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 64, + "n": 12288, + "k": 8192, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 64, + "n": 12288, + "k": 16384, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 64, + "n": 14336, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 64, + "n": 16384, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 64, + "n": 16384, + "k": 12288, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 64, + "n": 65536, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 64, + "n": 81920, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 65, + "n": 24, + "k": 49152, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 65, + "n": 64, + "k": 12288, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 65, + "n": 120, + "k": 12288, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 65, + "n": 256, + "k": 12288, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 65, + "n": 384, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w3" + ] + }, + { + "type": "gemm", + "args": { + "m": 65, + "n": 512, + "k": 12288, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 65, + "n": 512, + "k": 12288, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 65, + "n": 768, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w3" + ] + }, + { + "type": "gemm", + "args": { + "m": 65, + "n": 1024, + "k": 12288, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 65, + "n": 1536, + "k": 12288, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 65, + "n": 1792, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 65, + "n": 5120, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 65, + "n": 7168, + "k": 192, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w2" + ] + }, + { + "type": "gemm", + "args": { + "m": 65, + "n": 7168, + "k": 384, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w2" + ] + }, + { + "type": "gemm", + "args": { + "m": 65, + "n": 7168, + "k": 896, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 65, + "n": 7168, + "k": 2048, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 65, + "n": 10240, + "k": 1024, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 65, + "n": 10240, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 65, + "n": 12288, + "k": 1024, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 65, + "n": 12288, + "k": 2048, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 68, + "n": 32320, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/head", + "deepseek-ai/DeepSeek-V4-Pro/head", + "sgl-project/DeepSeek-V4-Pro-FP8/head" + ] + }, + { + "type": "gemm", + "args": { + "m": 69, + "n": 24, + "k": 40960, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 69, + "n": 64, + "k": 10240, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 69, + "n": 192, + "k": 10240, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 69, + "n": 256, + "k": 10240, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 69, + "n": 512, + "k": 10240, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 69, + "n": 512, + "k": 10240, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 69, + "n": 1024, + "k": 10240, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 69, + "n": 1536, + "k": 10240, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 69, + "n": 10240, + "k": 2048, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 70, + "n": 1024, + "k": 10240, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 70, + "n": 10240, + "k": 512, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 72, + "n": 24, + "k": 28672, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/hc_attn", + "deepseek-ai/DeepSeek-V4-Pro-0813/hc_ffn", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.hc_attn", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.hc_ffn", + "deepseek-ai/DeepSeek-V4-Pro/hc_attn", + "deepseek-ai/DeepSeek-V4-Pro/hc_ffn", + "deepseek-ai/DeepSeek-V4-Pro/mtp.hc_attn", + "deepseek-ai/DeepSeek-V4-Pro/mtp.hc_ffn", + "sgl-project/DeepSeek-V4-Pro-FP8/hc_attn", + "sgl-project/DeepSeek-V4-Pro-FP8/hc_ffn", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.hc_attn", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.hc_ffn" + ] + }, + { + "type": "gemm", + "args": { + "m": 72, + "n": 24, + "k": 40960, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 72, + "n": 24, + "k": 49152, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 72, + "n": 64, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.weights_proj", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.weights_proj", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.weights_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 72, + "n": 64, + "k": 10240, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 72, + "n": 64, + "k": 12288, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 72, + "n": 120, + "k": 12288, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 72, + "n": 192, + "k": 10240, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 72, + "n": 256, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.compressor.wkv", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.compressor.wkv", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.compressor.wgate", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.compressor.wkv" + ] + }, + { + "type": "gemm", + "args": { + "m": 72, + "n": 256, + "k": 10240, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 72, + "n": 256, + "k": 12288, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 72, + "n": 320, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 72, + "n": 384, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.gate", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.gate", + "deepseek-ai/DeepSeek-V4-Pro/ffn.gate", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.gate", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.gate", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.gate" + ] + }, + { + "type": "gemm", + "args": { + "m": 72, + "n": 384, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w3" + ] + }, + { + "type": "gemm", + "args": { + "m": 72, + "n": 512, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wkv", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wkv", + "deepseek-ai/DeepSeek-V4-Pro/attn.wkv", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wkv", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wkv", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wkv" + ] + }, + { + "type": "gemm", + "args": { + "m": 72, + "n": 512, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.compressor.wkv+wgate", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.compressor.wkv+wgate", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.compressor.wkv+wgate" + ] + }, + { + "type": "gemm", + "args": { + "m": 72, + "n": 512, + "k": 10240, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 72, + "n": 512, + "k": 10240, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 72, + "n": 512, + "k": 12288, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 72, + "n": 512, + "k": 12288, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 72, + "n": 768, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w3" + ] + }, + { + "type": "gemm", + "args": { + "m": 72, + "n": 896, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 72, + "n": 1024, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.compressor.wkv", + "deepseek-ai/DeepSeek-V4-Pro/attn.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro/attn.compressor.wkv", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.compressor.wgate", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.compressor.wkv" + ] + }, + { + "type": "gemm", + "args": { + "m": 72, + "n": 1024, + "k": 10240, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 72, + "n": 1024, + "k": 12288, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 72, + "n": 1536, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w3" + ] + }, + { + "type": "gemm", + "args": { + "m": 72, + "n": 1536, + "k": 10240, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 72, + "n": 1536, + "k": 12288, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 72, + "n": 1792, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 72, + "n": 4096, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 72, + "n": 5120, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 72, + "n": 7168, + "k": 192, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w2" + ] + }, + { + "type": "gemm", + "args": { + "m": 72, + "n": 7168, + "k": 384, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w2" + ] + }, + { + "type": "gemm", + "args": { + "m": 72, + "n": 7168, + "k": 448, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 72, + "n": 7168, + "k": 768, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w2" + ] + }, + { + "type": "gemm", + "args": { + "m": 72, + "n": 7168, + "k": 896, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 72, + "n": 7168, + "k": 1024, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 72, + "n": 7168, + "k": 2048, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 72, + "n": 7168, + "k": 4096, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 72, + "n": 8192, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 72, + "n": 10240, + "k": 1024, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 72, + "n": 10240, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 72, + "n": 10240, + "k": 2048, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 72, + "n": 12288, + "k": 1024, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 72, + "n": 12288, + "k": 2048, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 72, + "n": 16384, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 74, + "n": 24, + "k": 28672, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/hc_attn", + "deepseek-ai/DeepSeek-V4-Pro-0813/hc_ffn", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.hc_attn", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.hc_ffn", + "deepseek-ai/DeepSeek-V4-Pro/hc_attn", + "deepseek-ai/DeepSeek-V4-Pro/hc_ffn", + "deepseek-ai/DeepSeek-V4-Pro/mtp.hc_attn", + "deepseek-ai/DeepSeek-V4-Pro/mtp.hc_ffn", + "sgl-project/DeepSeek-V4-Pro-FP8/hc_attn", + "sgl-project/DeepSeek-V4-Pro-FP8/hc_ffn", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.hc_attn", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.hc_ffn" + ] + }, + { + "type": "gemm", + "args": { + "m": 74, + "n": 64, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.weights_proj", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.weights_proj", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.weights_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 74, + "n": 256, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.compressor.wkv", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.compressor.wkv", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.compressor.wgate", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.compressor.wkv" + ] + }, + { + "type": "gemm", + "args": { + "m": 74, + "n": 384, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.gate", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.gate", + "deepseek-ai/DeepSeek-V4-Pro/ffn.gate", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.gate", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.gate", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.gate" + ] + }, + { + "type": "gemm", + "args": { + "m": 74, + "n": 512, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wkv", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wkv", + "deepseek-ai/DeepSeek-V4-Pro/attn.wkv", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wkv", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wkv", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wkv" + ] + }, + { + "type": "gemm", + "args": { + "m": 74, + "n": 512, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.compressor.wkv+wgate", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.compressor.wkv+wgate", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.compressor.wkv+wgate" + ] + }, + { + "type": "gemm", + "args": { + "m": 74, + "n": 1024, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.compressor.wkv", + "deepseek-ai/DeepSeek-V4-Pro/attn.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro/attn.compressor.wkv", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.compressor.wgate", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.compressor.wkv" + ] + }, + { + "type": "gemm", + "args": { + "m": 74, + "n": 1536, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w3" + ] + }, + { + "type": "gemm", + "args": { + "m": 74, + "n": 4096, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 74, + "n": 7168, + "k": 768, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w2" + ] + }, + { + "type": "gemm", + "args": { + "m": 74, + "n": 7168, + "k": 1024, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 74, + "n": 7168, + "k": 4096, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 74, + "n": 8192, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 74, + "n": 16384, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 77, + "n": 320, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 77, + "n": 896, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 77, + "n": 7168, + "k": 448, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 80, + "n": 24, + "k": 28672, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/hc_attn", + "deepseek-ai/DeepSeek-V4-Pro-0813/hc_ffn", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.hc_attn", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.hc_ffn", + "deepseek-ai/DeepSeek-V4-Pro/hc_attn", + "deepseek-ai/DeepSeek-V4-Pro/hc_ffn", + "deepseek-ai/DeepSeek-V4-Pro/mtp.hc_attn", + "deepseek-ai/DeepSeek-V4-Pro/mtp.hc_ffn", + "sgl-project/DeepSeek-V4-Pro-FP8/hc_attn", + "sgl-project/DeepSeek-V4-Pro-FP8/hc_ffn", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.hc_attn", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.hc_ffn" + ] + }, + { + "type": "gemm", + "args": { + "m": 80, + "n": 24, + "k": 40960, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 80, + "n": 24, + "k": 49152, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 80, + "n": 64, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.weights_proj", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.weights_proj", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.weights_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 80, + "n": 64, + "k": 10240, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 80, + "n": 64, + "k": 12288, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 80, + "n": 120, + "k": 12288, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 80, + "n": 192, + "k": 10240, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 80, + "n": 256, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.compressor.wkv", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.compressor.wkv", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.compressor.wgate", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.compressor.wkv" + ] + }, + { + "type": "gemm", + "args": { + "m": 80, + "n": 256, + "k": 10240, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 80, + "n": 256, + "k": 12288, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 80, + "n": 320, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 80, + "n": 384, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.gate", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.gate", + "deepseek-ai/DeepSeek-V4-Pro/ffn.gate", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.gate", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.gate", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.gate" + ] + }, + { + "type": "gemm", + "args": { + "m": 80, + "n": 384, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w3" + ] + }, + { + "type": "gemm", + "args": { + "m": 80, + "n": 512, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wkv", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wkv", + "deepseek-ai/DeepSeek-V4-Pro/attn.wkv", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wkv", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wkv", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wkv" + ] + }, + { + "type": "gemm", + "args": { + "m": 80, + "n": 512, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.compressor.wkv+wgate", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.compressor.wkv+wgate", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.compressor.wkv+wgate" + ] + }, + { + "type": "gemm", + "args": { + "m": 80, + "n": 512, + "k": 10240, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 80, + "n": 512, + "k": 10240, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 80, + "n": 512, + "k": 12288, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 80, + "n": 512, + "k": 12288, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 80, + "n": 768, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w3" + ] + }, + { + "type": "gemm", + "args": { + "m": 80, + "n": 896, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 80, + "n": 1024, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.compressor.wkv", + "deepseek-ai/DeepSeek-V4-Pro/attn.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro/attn.compressor.wkv", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.compressor.wgate", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.compressor.wkv" + ] + }, + { + "type": "gemm", + "args": { + "m": 80, + "n": 1024, + "k": 10240, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 80, + "n": 1024, + "k": 12288, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 80, + "n": 1536, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w3" + ] + }, + { + "type": "gemm", + "args": { + "m": 80, + "n": 1536, + "k": 10240, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 80, + "n": 1536, + "k": 12288, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 80, + "n": 4096, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 80, + "n": 5120, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 80, + "n": 7168, + "k": 192, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w2" + ] + }, + { + "type": "gemm", + "args": { + "m": 80, + "n": 7168, + "k": 384, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w2" + ] + }, + { + "type": "gemm", + "args": { + "m": 80, + "n": 7168, + "k": 448, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 80, + "n": 7168, + "k": 768, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w2" + ] + }, + { + "type": "gemm", + "args": { + "m": 80, + "n": 7168, + "k": 1024, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 80, + "n": 7168, + "k": 2048, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 80, + "n": 7168, + "k": 4096, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 80, + "n": 8192, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 80, + "n": 10240, + "k": 1024, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 80, + "n": 10240, + "k": 2048, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 80, + "n": 12288, + "k": 1024, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 80, + "n": 16384, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 80, + "n": 32320, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/head", + "deepseek-ai/DeepSeek-V4-Pro/head", + "sgl-project/DeepSeek-V4-Pro-FP8/head" + ] + }, + { + "type": "gemm", + "args": { + "m": 81, + "n": 24, + "k": 49152, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 81, + "n": 64, + "k": 12288, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 81, + "n": 120, + "k": 12288, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 81, + "n": 256, + "k": 12288, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 81, + "n": 512, + "k": 12288, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 81, + "n": 512, + "k": 12288, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 81, + "n": 1024, + "k": 12288, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 81, + "n": 1536, + "k": 12288, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 81, + "n": 5120, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 81, + "n": 10240, + "k": 1024, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 81, + "n": 12288, + "k": 1024, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 83, + "n": 768, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w3" + ] + }, + { + "type": "gemm", + "args": { + "m": 83, + "n": 7168, + "k": 384, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w2" + ] + }, + { + "type": "gemm", + "args": { + "m": 83, + "n": 7168, + "k": 2048, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 84, + "n": 1792, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 84, + "n": 7168, + "k": 896, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 84, + "n": 10240, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 84, + "n": 12288, + "k": 2048, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 88, + "n": 24, + "k": 28672, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/hc_attn", + "deepseek-ai/DeepSeek-V4-Pro-0813/hc_ffn", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.hc_attn", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.hc_ffn", + "deepseek-ai/DeepSeek-V4-Pro/hc_attn", + "deepseek-ai/DeepSeek-V4-Pro/hc_ffn", + "deepseek-ai/DeepSeek-V4-Pro/mtp.hc_attn", + "deepseek-ai/DeepSeek-V4-Pro/mtp.hc_ffn", + "sgl-project/DeepSeek-V4-Pro-FP8/hc_attn", + "sgl-project/DeepSeek-V4-Pro-FP8/hc_ffn", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.hc_attn", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.hc_ffn" + ] + }, + { + "type": "gemm", + "args": { + "m": 88, + "n": 24, + "k": 40960, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 88, + "n": 64, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.weights_proj", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.weights_proj", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.weights_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 88, + "n": 64, + "k": 10240, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 88, + "n": 192, + "k": 10240, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 88, + "n": 256, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.compressor.wkv", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.compressor.wkv", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.compressor.wgate", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.compressor.wkv" + ] + }, + { + "type": "gemm", + "args": { + "m": 88, + "n": 256, + "k": 10240, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 88, + "n": 320, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 88, + "n": 384, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.gate", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.gate", + "deepseek-ai/DeepSeek-V4-Pro/ffn.gate", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.gate", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.gate", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.gate" + ] + }, + { + "type": "gemm", + "args": { + "m": 88, + "n": 512, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wkv", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wkv", + "deepseek-ai/DeepSeek-V4-Pro/attn.wkv", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wkv", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wkv", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wkv" + ] + }, + { + "type": "gemm", + "args": { + "m": 88, + "n": 512, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.compressor.wkv+wgate", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.compressor.wkv+wgate", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.compressor.wkv+wgate" + ] + }, + { + "type": "gemm", + "args": { + "m": 88, + "n": 512, + "k": 10240, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 88, + "n": 512, + "k": 10240, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 88, + "n": 896, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 88, + "n": 1024, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.compressor.wkv", + "deepseek-ai/DeepSeek-V4-Pro/attn.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro/attn.compressor.wkv", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.compressor.wgate", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.compressor.wkv" + ] + }, + { + "type": "gemm", + "args": { + "m": 88, + "n": 1024, + "k": 10240, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 88, + "n": 1536, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w3" + ] + }, + { + "type": "gemm", + "args": { + "m": 88, + "n": 1536, + "k": 10240, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 88, + "n": 4096, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 88, + "n": 7168, + "k": 448, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 88, + "n": 7168, + "k": 768, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w2" + ] + }, + { + "type": "gemm", + "args": { + "m": 88, + "n": 7168, + "k": 1024, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 88, + "n": 7168, + "k": 4096, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 88, + "n": 8192, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 88, + "n": 10240, + "k": 1024, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 88, + "n": 16384, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 88, + "n": 32320, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/head", + "deepseek-ai/DeepSeek-V4-Pro/head", + "sgl-project/DeepSeek-V4-Pro-FP8/head" + ] + }, + { + "type": "gemm", + "args": { + "m": 89, + "n": 32320, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/head", + "deepseek-ai/DeepSeek-V4-Pro/head", + "sgl-project/DeepSeek-V4-Pro-FP8/head" + ] + }, + { + "type": "gemm", + "args": { + "m": 93, + "n": 24, + "k": 28672, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/hc_attn", + "deepseek-ai/DeepSeek-V4-Pro-0813/hc_ffn", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.hc_attn", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.hc_ffn", + "deepseek-ai/DeepSeek-V4-Pro/hc_attn", + "deepseek-ai/DeepSeek-V4-Pro/hc_ffn", + "deepseek-ai/DeepSeek-V4-Pro/mtp.hc_attn", + "deepseek-ai/DeepSeek-V4-Pro/mtp.hc_ffn", + "sgl-project/DeepSeek-V4-Pro-FP8/hc_attn", + "sgl-project/DeepSeek-V4-Pro-FP8/hc_ffn", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.hc_attn", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.hc_ffn" + ] + }, + { + "type": "gemm", + "args": { + "m": 93, + "n": 64, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.weights_proj", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.weights_proj", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.weights_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 93, + "n": 256, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.compressor.wkv", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.compressor.wkv", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.compressor.wgate", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.compressor.wkv" + ] + }, + { + "type": "gemm", + "args": { + "m": 93, + "n": 384, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.gate", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.gate", + "deepseek-ai/DeepSeek-V4-Pro/ffn.gate", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.gate", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.gate", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.gate" + ] + }, + { + "type": "gemm", + "args": { + "m": 93, + "n": 512, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wkv", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wkv", + "deepseek-ai/DeepSeek-V4-Pro/attn.wkv", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wkv", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wkv", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wkv" + ] + }, + { + "type": "gemm", + "args": { + "m": 93, + "n": 512, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.compressor.wkv+wgate", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.compressor.wkv+wgate", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.compressor.wkv+wgate" + ] + }, + { + "type": "gemm", + "args": { + "m": 93, + "n": 1024, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.compressor.wkv", + "deepseek-ai/DeepSeek-V4-Pro/attn.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro/attn.compressor.wkv", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.compressor.wgate", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.compressor.wkv" + ] + }, + { + "type": "gemm", + "args": { + "m": 93, + "n": 1536, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w3" + ] + }, + { + "type": "gemm", + "args": { + "m": 93, + "n": 4096, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 93, + "n": 7168, + "k": 768, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w2" + ] + }, + { + "type": "gemm", + "args": { + "m": 93, + "n": 7168, + "k": 1024, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 93, + "n": 7168, + "k": 4096, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 93, + "n": 8192, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 93, + "n": 16384, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 96, + "n": 24, + "k": 28672, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/hc_attn", + "deepseek-ai/DeepSeek-V4-Pro-0813/hc_ffn", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.hc_attn", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.hc_ffn", + "deepseek-ai/DeepSeek-V4-Pro/hc_attn", + "deepseek-ai/DeepSeek-V4-Pro/hc_ffn", + "deepseek-ai/DeepSeek-V4-Pro/mtp.hc_attn", + "deepseek-ai/DeepSeek-V4-Pro/mtp.hc_ffn", + "sgl-project/DeepSeek-V4-Pro-FP8/hc_attn", + "sgl-project/DeepSeek-V4-Pro-FP8/hc_ffn", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.hc_attn", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.hc_ffn" + ] + }, + { + "type": "gemm", + "args": { + "m": 96, + "n": 64, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.weights_proj", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.weights_proj", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.weights_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 96, + "n": 256, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.compressor.wkv", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.compressor.wkv", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.compressor.wgate", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.compressor.wkv" + ] + }, + { + "type": "gemm", + "args": { + "m": 96, + "n": 384, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.gate", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.gate", + "deepseek-ai/DeepSeek-V4-Pro/ffn.gate", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.gate", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.gate", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.gate" + ] + }, + { + "type": "gemm", + "args": { + "m": 96, + "n": 512, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wkv", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wkv", + "deepseek-ai/DeepSeek-V4-Pro/attn.wkv", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wkv", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wkv", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wkv" + ] + }, + { + "type": "gemm", + "args": { + "m": 96, + "n": 512, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.compressor.wkv+wgate", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.compressor.wkv+wgate", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.compressor.wkv+wgate" + ] + }, + { + "type": "gemm", + "args": { + "m": 96, + "n": 1024, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.compressor.wkv", + "deepseek-ai/DeepSeek-V4-Pro/attn.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro/attn.compressor.wkv", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.compressor.wgate", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.compressor.wkv" + ] + }, + { + "type": "gemm", + "args": { + "m": 96, + "n": 1536, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w3" + ] + }, + { + "type": "gemm", + "args": { + "m": 96, + "n": 7168, + "k": 768, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w2" + ] + }, + { + "type": "gemm", + "args": { + "m": 96, + "n": 7168, + "k": 4096, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 96, + "n": 8192, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 96, + "n": 16384, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 97, + "n": 320, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 97, + "n": 896, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 97, + "n": 7168, + "k": 448, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 99, + "n": 24, + "k": 40960, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 99, + "n": 64, + "k": 10240, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 99, + "n": 192, + "k": 10240, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 99, + "n": 256, + "k": 10240, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 99, + "n": 512, + "k": 10240, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 99, + "n": 512, + "k": 10240, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 99, + "n": 1024, + "k": 10240, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 99, + "n": 1024, + "k": 10240, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 99, + "n": 1536, + "k": 10240, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 99, + "n": 10240, + "k": 512, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 99, + "n": 10240, + "k": 2048, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 104, + "n": 24, + "k": 28672, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/hc_attn", + "deepseek-ai/DeepSeek-V4-Pro-0813/hc_ffn", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.hc_attn", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.hc_ffn", + "deepseek-ai/DeepSeek-V4-Pro/hc_attn", + "deepseek-ai/DeepSeek-V4-Pro/hc_ffn", + "deepseek-ai/DeepSeek-V4-Pro/mtp.hc_attn", + "deepseek-ai/DeepSeek-V4-Pro/mtp.hc_ffn", + "sgl-project/DeepSeek-V4-Pro-FP8/hc_attn", + "sgl-project/DeepSeek-V4-Pro-FP8/hc_ffn", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.hc_attn", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.hc_ffn" + ] + }, + { + "type": "gemm", + "args": { + "m": 104, + "n": 64, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.weights_proj", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.weights_proj", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.weights_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 104, + "n": 256, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.compressor.wkv", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.compressor.wkv", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.compressor.wgate", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.compressor.wkv" + ] + }, + { + "type": "gemm", + "args": { + "m": 104, + "n": 384, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.gate", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.gate", + "deepseek-ai/DeepSeek-V4-Pro/ffn.gate", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.gate", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.gate", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.gate" + ] + }, + { + "type": "gemm", + "args": { + "m": 104, + "n": 512, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wkv", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wkv", + "deepseek-ai/DeepSeek-V4-Pro/attn.wkv", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wkv", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wkv", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wkv" + ] + }, + { + "type": "gemm", + "args": { + "m": 104, + "n": 512, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.compressor.wkv+wgate", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.compressor.wkv+wgate", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.compressor.wkv+wgate" + ] + }, + { + "type": "gemm", + "args": { + "m": 104, + "n": 1024, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.compressor.wkv", + "deepseek-ai/DeepSeek-V4-Pro/attn.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro/attn.compressor.wkv", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.compressor.wgate", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.compressor.wkv" + ] + }, + { + "type": "gemm", + "args": { + "m": 104, + "n": 1536, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w3" + ] + }, + { + "type": "gemm", + "args": { + "m": 104, + "n": 7168, + "k": 768, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w2" + ] + }, + { + "type": "gemm", + "args": { + "m": 104, + "n": 7168, + "k": 4096, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 104, + "n": 8192, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 104, + "n": 16384, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 111, + "n": 32320, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/head", + "deepseek-ai/DeepSeek-V4-Pro/head", + "sgl-project/DeepSeek-V4-Pro-FP8/head" + ] + }, + { + "type": "gemm", + "args": { + "m": 112, + "n": 24, + "k": 28672, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/hc_attn", + "deepseek-ai/DeepSeek-V4-Pro-0813/hc_ffn", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.hc_attn", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.hc_ffn", + "deepseek-ai/DeepSeek-V4-Pro/hc_attn", + "deepseek-ai/DeepSeek-V4-Pro/hc_ffn", + "deepseek-ai/DeepSeek-V4-Pro/mtp.hc_attn", + "deepseek-ai/DeepSeek-V4-Pro/mtp.hc_ffn", + "sgl-project/DeepSeek-V4-Pro-FP8/hc_attn", + "sgl-project/DeepSeek-V4-Pro-FP8/hc_ffn", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.hc_attn", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.hc_ffn" + ] + }, + { + "type": "gemm", + "args": { + "m": 112, + "n": 24, + "k": 40960, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 112, + "n": 64, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.weights_proj", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.weights_proj", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.weights_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 112, + "n": 64, + "k": 10240, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 112, + "n": 192, + "k": 10240, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 112, + "n": 256, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.compressor.wkv", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.compressor.wkv", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.compressor.wgate", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.compressor.wkv" + ] + }, + { + "type": "gemm", + "args": { + "m": 112, + "n": 256, + "k": 10240, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 112, + "n": 384, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.gate", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.gate", + "deepseek-ai/DeepSeek-V4-Pro/ffn.gate", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.gate", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.gate", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.gate" + ] + }, + { + "type": "gemm", + "args": { + "m": 112, + "n": 512, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wkv", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wkv", + "deepseek-ai/DeepSeek-V4-Pro/attn.wkv", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wkv", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wkv", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wkv" + ] + }, + { + "type": "gemm", + "args": { + "m": 112, + "n": 512, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.compressor.wkv+wgate", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.compressor.wkv+wgate", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.compressor.wkv+wgate" + ] + }, + { + "type": "gemm", + "args": { + "m": 112, + "n": 512, + "k": 10240, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 112, + "n": 512, + "k": 10240, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 112, + "n": 1024, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.compressor.wkv", + "deepseek-ai/DeepSeek-V4-Pro/attn.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro/attn.compressor.wkv", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.compressor.wgate", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.compressor.wkv" + ] + }, + { + "type": "gemm", + "args": { + "m": 112, + "n": 1024, + "k": 10240, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 112, + "n": 1024, + "k": 10240, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 112, + "n": 1536, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w3" + ] + }, + { + "type": "gemm", + "args": { + "m": 112, + "n": 1536, + "k": 10240, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 112, + "n": 4096, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 112, + "n": 7168, + "k": 768, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w2" + ] + }, + { + "type": "gemm", + "args": { + "m": 112, + "n": 7168, + "k": 4096, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 112, + "n": 8192, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 112, + "n": 10240, + "k": 512, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 112, + "n": 10240, + "k": 1024, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 112, + "n": 10240, + "k": 2048, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 112, + "n": 16384, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 117, + "n": 24, + "k": 28672, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/hc_attn", + "deepseek-ai/DeepSeek-V4-Pro-0813/hc_ffn", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.hc_attn", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.hc_ffn", + "deepseek-ai/DeepSeek-V4-Pro/hc_attn", + "deepseek-ai/DeepSeek-V4-Pro/hc_ffn", + "deepseek-ai/DeepSeek-V4-Pro/mtp.hc_attn", + "deepseek-ai/DeepSeek-V4-Pro/mtp.hc_ffn", + "sgl-project/DeepSeek-V4-Pro-FP8/hc_attn", + "sgl-project/DeepSeek-V4-Pro-FP8/hc_ffn", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.hc_attn", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.hc_ffn" + ] + }, + { + "type": "gemm", + "args": { + "m": 117, + "n": 64, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.weights_proj", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.weights_proj", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.weights_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 117, + "n": 256, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.compressor.wkv", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.compressor.wkv", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.compressor.wgate", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.compressor.wkv" + ] + }, + { + "type": "gemm", + "args": { + "m": 117, + "n": 384, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.gate", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.gate", + "deepseek-ai/DeepSeek-V4-Pro/ffn.gate", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.gate", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.gate", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.gate" + ] + }, + { + "type": "gemm", + "args": { + "m": 117, + "n": 512, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wkv", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wkv", + "deepseek-ai/DeepSeek-V4-Pro/attn.wkv", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wkv", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wkv", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wkv" + ] + }, + { + "type": "gemm", + "args": { + "m": 117, + "n": 512, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.compressor.wkv+wgate", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.compressor.wkv+wgate", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.compressor.wkv+wgate" + ] + }, + { + "type": "gemm", + "args": { + "m": 117, + "n": 1024, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.compressor.wkv", + "deepseek-ai/DeepSeek-V4-Pro/attn.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro/attn.compressor.wkv", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.compressor.wgate", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.compressor.wkv" + ] + }, + { + "type": "gemm", + "args": { + "m": 117, + "n": 1536, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w3" + ] + }, + { + "type": "gemm", + "args": { + "m": 117, + "n": 7168, + "k": 768, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w2" + ] + }, + { + "type": "gemm", + "args": { + "m": 117, + "n": 7168, + "k": 4096, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 117, + "n": 8192, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 117, + "n": 16384, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 120, + "n": 24, + "k": 28672, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/hc_attn", + "deepseek-ai/DeepSeek-V4-Pro-0813/hc_ffn", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.hc_attn", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.hc_ffn", + "deepseek-ai/DeepSeek-V4-Pro/hc_attn", + "deepseek-ai/DeepSeek-V4-Pro/hc_ffn", + "deepseek-ai/DeepSeek-V4-Pro/mtp.hc_attn", + "deepseek-ai/DeepSeek-V4-Pro/mtp.hc_ffn", + "sgl-project/DeepSeek-V4-Pro-FP8/hc_attn", + "sgl-project/DeepSeek-V4-Pro-FP8/hc_ffn", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.hc_attn", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.hc_ffn" + ] + }, + { + "type": "gemm", + "args": { + "m": 120, + "n": 64, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.weights_proj", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.weights_proj", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.weights_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 120, + "n": 256, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.compressor.wkv", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.compressor.wkv", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.compressor.wgate", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.compressor.wkv" + ] + }, + { + "type": "gemm", + "args": { + "m": 120, + "n": 384, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.gate", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.gate", + "deepseek-ai/DeepSeek-V4-Pro/ffn.gate", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.gate", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.gate", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.gate" + ] + }, + { + "type": "gemm", + "args": { + "m": 120, + "n": 512, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wkv", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wkv", + "deepseek-ai/DeepSeek-V4-Pro/attn.wkv", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wkv", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wkv", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wkv" + ] + }, + { + "type": "gemm", + "args": { + "m": 120, + "n": 512, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.compressor.wkv+wgate", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.compressor.wkv+wgate", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.compressor.wkv+wgate" + ] + }, + { + "type": "gemm", + "args": { + "m": 120, + "n": 768, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w3" + ] + }, + { + "type": "gemm", + "args": { + "m": 120, + "n": 1024, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.compressor.wkv", + "deepseek-ai/DeepSeek-V4-Pro/attn.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro/attn.compressor.wkv", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.compressor.wgate", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.compressor.wkv" + ] + }, + { + "type": "gemm", + "args": { + "m": 120, + "n": 1536, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w3" + ] + }, + { + "type": "gemm", + "args": { + "m": 120, + "n": 7168, + "k": 384, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w2" + ] + }, + { + "type": "gemm", + "args": { + "m": 120, + "n": 7168, + "k": 768, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w2" + ] + }, + { + "type": "gemm", + "args": { + "m": 120, + "n": 7168, + "k": 2048, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 120, + "n": 7168, + "k": 4096, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 120, + "n": 8192, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 120, + "n": 16384, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 120, + "n": 32320, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/head", + "deepseek-ai/DeepSeek-V4-Pro/head", + "sgl-project/DeepSeek-V4-Pro-FP8/head" + ] + }, + { + "type": "gemm", + "args": { + "m": 124, + "n": 768, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w3" + ] + }, + { + "type": "gemm", + "args": { + "m": 124, + "n": 7168, + "k": 384, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w2" + ] + }, + { + "type": "gemm", + "args": { + "m": 124, + "n": 7168, + "k": 2048, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 128, + "n": 24, + "k": 28672, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/hc_attn", + "deepseek-ai/DeepSeek-V4-Pro-0813/hc_ffn", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.hc_attn", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.hc_ffn", + "deepseek-ai/DeepSeek-V4-Pro/hc_attn", + "deepseek-ai/DeepSeek-V4-Pro/hc_ffn", + "deepseek-ai/DeepSeek-V4-Pro/mtp.hc_attn", + "deepseek-ai/DeepSeek-V4-Pro/mtp.hc_ffn", + "sgl-project/DeepSeek-V4-Pro-FP8/hc_attn", + "sgl-project/DeepSeek-V4-Pro-FP8/hc_ffn", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.hc_attn", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.hc_ffn" + ] + }, + { + "type": "gemm", + "args": { + "m": 128, + "n": 24, + "k": 40960, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 128, + "n": 24, + "k": 49152, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 128, + "n": 64, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.weights_proj", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.weights_proj", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.weights_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 128, + "n": 64, + "k": 10240, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 128, + "n": 64, + "k": 12288, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 128, + "n": 120, + "k": 12288, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 128, + "n": 192, + "k": 10240, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 128, + "n": 256, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.compressor.wkv", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.compressor.wkv", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.compressor.wgate", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.compressor.wkv" + ] + }, + { + "type": "gemm", + "args": { + "m": 128, + "n": 256, + "k": 10240, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 128, + "n": 256, + "k": 12288, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 128, + "n": 320, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 128, + "n": 384, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.gate", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.gate", + "deepseek-ai/DeepSeek-V4-Pro/ffn.gate", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.gate", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.gate", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.gate" + ] + }, + { + "type": "gemm", + "args": { + "m": 128, + "n": 384, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w3" + ] + }, + { + "type": "gemm", + "args": { + "m": 128, + "n": 512, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wkv", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wkv", + "deepseek-ai/DeepSeek-V4-Pro/attn.wkv", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wkv", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wkv", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wkv" + ] + }, + { + "type": "gemm", + "args": { + "m": 128, + "n": 512, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.compressor.wkv+wgate", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.compressor.wkv+wgate", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.compressor.wkv+wgate" + ] + }, + { + "type": "gemm", + "args": { + "m": 128, + "n": 512, + "k": 10240, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 128, + "n": 512, + "k": 10240, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 128, + "n": 512, + "k": 12288, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 128, + "n": 512, + "k": 12288, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 128, + "n": 768, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w3" + ] + }, + { + "type": "gemm", + "args": { + "m": 128, + "n": 1024, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.compressor.wkv", + "deepseek-ai/DeepSeek-V4-Pro/attn.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro/attn.compressor.wkv", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.compressor.wgate", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.compressor.wkv" + ] + }, + { + "type": "gemm", + "args": { + "m": 128, + "n": 1024, + "k": 10240, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 128, + "n": 1024, + "k": 10240, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 128, + "n": 1024, + "k": 12288, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 128, + "n": 1536, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w3" + ] + }, + { + "type": "gemm", + "args": { + "m": 128, + "n": 1536, + "k": 10240, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 128, + "n": 1536, + "k": 12288, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 128, + "n": 1792, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 128, + "n": 2048, + "k": 12288, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 128, + "n": 4096, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 128, + "n": 5120, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 128, + "n": 6144, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3" + ] + }, + { + "type": "gemm", + "args": { + "m": 128, + "n": 7168, + "k": 192, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w2" + ] + }, + { + "type": "gemm", + "args": { + "m": 128, + "n": 7168, + "k": 384, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w2" + ] + }, + { + "type": "gemm", + "args": { + "m": 128, + "n": 7168, + "k": 768, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w2" + ] + }, + { + "type": "gemm", + "args": { + "m": 128, + "n": 7168, + "k": 896, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 128, + "n": 7168, + "k": 1024, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 128, + "n": 7168, + "k": 2048, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 128, + "n": 7168, + "k": 3072, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w2" + ] + }, + { + "type": "gemm", + "args": { + "m": 128, + "n": 7168, + "k": 4096, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 128, + "n": 7168, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.e_proj", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.h_proj", + "deepseek-ai/DeepSeek-V4-Pro/mtp.e_proj", + "deepseek-ai/DeepSeek-V4-Pro/mtp.h_proj", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.e_proj", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.h_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 128, + "n": 7168, + "k": 16384, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 128, + "n": 8192, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 128, + "n": 10240, + "k": 512, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 128, + "n": 10240, + "k": 1024, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 128, + "n": 10240, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 128, + "n": 10240, + "k": 2048, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 128, + "n": 10240, + "k": 16384, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 128, + "n": 12288, + "k": 1024, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 128, + "n": 12288, + "k": 2048, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 128, + "n": 12288, + "k": 8192, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 128, + "n": 12288, + "k": 16384, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 128, + "n": 14336, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 128, + "n": 16384, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 128, + "n": 16384, + "k": 12288, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 128, + "n": 32320, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/head", + "deepseek-ai/DeepSeek-V4-Pro/head", + "sgl-project/DeepSeek-V4-Pro-FP8/head" + ] + }, + { + "type": "gemm", + "args": { + "m": 128, + "n": 65536, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 128, + "n": 81920, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 129, + "n": 24, + "k": 40960, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 129, + "n": 24, + "k": 49152, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 129, + "n": 64, + "k": 10240, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 129, + "n": 64, + "k": 12288, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 129, + "n": 120, + "k": 12288, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 129, + "n": 192, + "k": 10240, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 129, + "n": 256, + "k": 10240, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 129, + "n": 256, + "k": 12288, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 129, + "n": 320, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 129, + "n": 384, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w3" + ] + }, + { + "type": "gemm", + "args": { + "m": 129, + "n": 512, + "k": 10240, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 129, + "n": 512, + "k": 10240, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 129, + "n": 512, + "k": 12288, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 129, + "n": 512, + "k": 12288, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 129, + "n": 896, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 129, + "n": 1024, + "k": 10240, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 129, + "n": 1024, + "k": 10240, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 129, + "n": 1024, + "k": 12288, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 129, + "n": 1536, + "k": 10240, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 129, + "n": 1536, + "k": 12288, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 129, + "n": 1792, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 129, + "n": 4096, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 129, + "n": 5120, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 129, + "n": 7168, + "k": 192, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w2" + ] + }, + { + "type": "gemm", + "args": { + "m": 129, + "n": 7168, + "k": 448, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 129, + "n": 7168, + "k": 896, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 129, + "n": 7168, + "k": 1024, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 129, + "n": 10240, + "k": 512, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 129, + "n": 10240, + "k": 1024, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 129, + "n": 10240, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 129, + "n": 10240, + "k": 2048, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 129, + "n": 12288, + "k": 1024, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 129, + "n": 12288, + "k": 2048, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 136, + "n": 24, + "k": 40960, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 136, + "n": 24, + "k": 49152, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 136, + "n": 64, + "k": 10240, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 136, + "n": 64, + "k": 12288, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 136, + "n": 120, + "k": 12288, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 136, + "n": 192, + "k": 10240, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 136, + "n": 256, + "k": 10240, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 136, + "n": 256, + "k": 12288, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 136, + "n": 320, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 136, + "n": 384, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w3" + ] + }, + { + "type": "gemm", + "args": { + "m": 136, + "n": 512, + "k": 10240, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 136, + "n": 512, + "k": 10240, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 136, + "n": 512, + "k": 12288, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 136, + "n": 512, + "k": 12288, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 136, + "n": 896, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 136, + "n": 1024, + "k": 10240, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 136, + "n": 1024, + "k": 10240, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 136, + "n": 1024, + "k": 12288, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 136, + "n": 1536, + "k": 10240, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 136, + "n": 1536, + "k": 12288, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 136, + "n": 1792, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 136, + "n": 4096, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 136, + "n": 5120, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 136, + "n": 7168, + "k": 192, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w2" + ] + }, + { + "type": "gemm", + "args": { + "m": 136, + "n": 7168, + "k": 448, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 136, + "n": 7168, + "k": 896, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 136, + "n": 7168, + "k": 1024, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 136, + "n": 10240, + "k": 512, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 136, + "n": 10240, + "k": 1024, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 136, + "n": 10240, + "k": 2048, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 136, + "n": 12288, + "k": 1024, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 140, + "n": 32320, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/head", + "deepseek-ai/DeepSeek-V4-Pro/head", + "sgl-project/DeepSeek-V4-Pro-FP8/head" + ] + }, + { + "type": "gemm", + "args": { + "m": 141, + "n": 24, + "k": 40960, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 141, + "n": 64, + "k": 10240, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 141, + "n": 192, + "k": 10240, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 141, + "n": 256, + "k": 10240, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 141, + "n": 512, + "k": 10240, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 141, + "n": 512, + "k": 10240, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 141, + "n": 1024, + "k": 10240, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 141, + "n": 1536, + "k": 10240, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 141, + "n": 10240, + "k": 1024, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 144, + "n": 24, + "k": 28672, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/hc_attn", + "deepseek-ai/DeepSeek-V4-Pro-0813/hc_ffn", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.hc_attn", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.hc_ffn", + "deepseek-ai/DeepSeek-V4-Pro/hc_attn", + "deepseek-ai/DeepSeek-V4-Pro/hc_ffn", + "deepseek-ai/DeepSeek-V4-Pro/mtp.hc_attn", + "deepseek-ai/DeepSeek-V4-Pro/mtp.hc_ffn", + "sgl-project/DeepSeek-V4-Pro-FP8/hc_attn", + "sgl-project/DeepSeek-V4-Pro-FP8/hc_ffn", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.hc_attn", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.hc_ffn" + ] + }, + { + "type": "gemm", + "args": { + "m": 144, + "n": 24, + "k": 40960, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 144, + "n": 24, + "k": 49152, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 144, + "n": 64, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.weights_proj", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.weights_proj", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.weights_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 144, + "n": 64, + "k": 10240, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 144, + "n": 64, + "k": 12288, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 144, + "n": 120, + "k": 12288, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 144, + "n": 192, + "k": 10240, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 144, + "n": 256, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.compressor.wkv", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.compressor.wkv", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.compressor.wgate", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.compressor.wkv" + ] + }, + { + "type": "gemm", + "args": { + "m": 144, + "n": 256, + "k": 10240, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 144, + "n": 256, + "k": 12288, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 144, + "n": 320, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 144, + "n": 384, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.gate", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.gate", + "deepseek-ai/DeepSeek-V4-Pro/ffn.gate", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.gate", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.gate", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.gate" + ] + }, + { + "type": "gemm", + "args": { + "m": 144, + "n": 384, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w3" + ] + }, + { + "type": "gemm", + "args": { + "m": 144, + "n": 512, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wkv", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wkv", + "deepseek-ai/DeepSeek-V4-Pro/attn.wkv", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wkv", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wkv", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wkv" + ] + }, + { + "type": "gemm", + "args": { + "m": 144, + "n": 512, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.compressor.wkv+wgate", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.compressor.wkv+wgate", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.compressor.wkv+wgate" + ] + }, + { + "type": "gemm", + "args": { + "m": 144, + "n": 512, + "k": 10240, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 144, + "n": 512, + "k": 10240, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 144, + "n": 512, + "k": 12288, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 144, + "n": 512, + "k": 12288, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 144, + "n": 768, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w3" + ] + }, + { + "type": "gemm", + "args": { + "m": 144, + "n": 896, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 144, + "n": 1024, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.compressor.wkv", + "deepseek-ai/DeepSeek-V4-Pro/attn.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro/attn.compressor.wkv", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.compressor.wgate", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.compressor.wkv" + ] + }, + { + "type": "gemm", + "args": { + "m": 144, + "n": 1024, + "k": 10240, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 144, + "n": 1024, + "k": 12288, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 144, + "n": 1536, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w3" + ] + }, + { + "type": "gemm", + "args": { + "m": 144, + "n": 1536, + "k": 10240, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 144, + "n": 1536, + "k": 12288, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 144, + "n": 4096, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 144, + "n": 5120, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 144, + "n": 7168, + "k": 192, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w2" + ] + }, + { + "type": "gemm", + "args": { + "m": 144, + "n": 7168, + "k": 384, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w2" + ] + }, + { + "type": "gemm", + "args": { + "m": 144, + "n": 7168, + "k": 448, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 144, + "n": 7168, + "k": 768, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w2" + ] + }, + { + "type": "gemm", + "args": { + "m": 144, + "n": 7168, + "k": 1024, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 144, + "n": 7168, + "k": 2048, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 144, + "n": 7168, + "k": 4096, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 144, + "n": 8192, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 144, + "n": 10240, + "k": 1024, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 144, + "n": 10240, + "k": 2048, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 144, + "n": 12288, + "k": 1024, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 144, + "n": 16384, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 144, + "n": 32320, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/head", + "deepseek-ai/DeepSeek-V4-Pro/head", + "sgl-project/DeepSeek-V4-Pro-FP8/head" + ] + }, + { + "type": "gemm", + "args": { + "m": 147, + "n": 24, + "k": 28672, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/hc_attn", + "deepseek-ai/DeepSeek-V4-Pro-0813/hc_ffn", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.hc_attn", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.hc_ffn", + "deepseek-ai/DeepSeek-V4-Pro/hc_attn", + "deepseek-ai/DeepSeek-V4-Pro/hc_ffn", + "deepseek-ai/DeepSeek-V4-Pro/mtp.hc_attn", + "deepseek-ai/DeepSeek-V4-Pro/mtp.hc_ffn", + "sgl-project/DeepSeek-V4-Pro-FP8/hc_attn", + "sgl-project/DeepSeek-V4-Pro-FP8/hc_ffn", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.hc_attn", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.hc_ffn" + ] + }, + { + "type": "gemm", + "args": { + "m": 147, + "n": 64, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.weights_proj", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.weights_proj", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.weights_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 147, + "n": 256, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.compressor.wkv", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.compressor.wkv", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.compressor.wgate", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.compressor.wkv" + ] + }, + { + "type": "gemm", + "args": { + "m": 147, + "n": 384, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.gate", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.gate", + "deepseek-ai/DeepSeek-V4-Pro/ffn.gate", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.gate", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.gate", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.gate" + ] + }, + { + "type": "gemm", + "args": { + "m": 147, + "n": 512, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wkv", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wkv", + "deepseek-ai/DeepSeek-V4-Pro/attn.wkv", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wkv", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wkv", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wkv" + ] + }, + { + "type": "gemm", + "args": { + "m": 147, + "n": 512, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.compressor.wkv+wgate", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.compressor.wkv+wgate", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.compressor.wkv+wgate" + ] + }, + { + "type": "gemm", + "args": { + "m": 147, + "n": 1024, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.compressor.wkv", + "deepseek-ai/DeepSeek-V4-Pro/attn.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro/attn.compressor.wkv", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.compressor.wgate", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.compressor.wkv" + ] + }, + { + "type": "gemm", + "args": { + "m": 147, + "n": 1536, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w3" + ] + }, + { + "type": "gemm", + "args": { + "m": 147, + "n": 7168, + "k": 768, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w2" + ] + }, + { + "type": "gemm", + "args": { + "m": 147, + "n": 7168, + "k": 4096, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 147, + "n": 8192, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 147, + "n": 16384, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 151, + "n": 768, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w3" + ] + }, + { + "type": "gemm", + "args": { + "m": 151, + "n": 7168, + "k": 384, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w2" + ] + }, + { + "type": "gemm", + "args": { + "m": 151, + "n": 7168, + "k": 2048, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 152, + "n": 24, + "k": 28672, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/hc_attn", + "deepseek-ai/DeepSeek-V4-Pro-0813/hc_ffn", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.hc_attn", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.hc_ffn", + "deepseek-ai/DeepSeek-V4-Pro/hc_attn", + "deepseek-ai/DeepSeek-V4-Pro/hc_ffn", + "deepseek-ai/DeepSeek-V4-Pro/mtp.hc_attn", + "deepseek-ai/DeepSeek-V4-Pro/mtp.hc_ffn", + "sgl-project/DeepSeek-V4-Pro-FP8/hc_attn", + "sgl-project/DeepSeek-V4-Pro-FP8/hc_ffn", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.hc_attn", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.hc_ffn" + ] + }, + { + "type": "gemm", + "args": { + "m": 152, + "n": 64, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.weights_proj", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.weights_proj", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.weights_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 152, + "n": 256, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.compressor.wkv", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.compressor.wkv", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.compressor.wgate", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.compressor.wkv" + ] + }, + { + "type": "gemm", + "args": { + "m": 152, + "n": 384, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.gate", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.gate", + "deepseek-ai/DeepSeek-V4-Pro/ffn.gate", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.gate", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.gate", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.gate" + ] + }, + { + "type": "gemm", + "args": { + "m": 152, + "n": 512, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wkv", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wkv", + "deepseek-ai/DeepSeek-V4-Pro/attn.wkv", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wkv", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wkv", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wkv" + ] + }, + { + "type": "gemm", + "args": { + "m": 152, + "n": 512, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.compressor.wkv+wgate", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.compressor.wkv+wgate", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.compressor.wkv+wgate" + ] + }, + { + "type": "gemm", + "args": { + "m": 152, + "n": 1024, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.compressor.wkv", + "deepseek-ai/DeepSeek-V4-Pro/attn.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro/attn.compressor.wkv", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.compressor.wgate", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.compressor.wkv" + ] + }, + { + "type": "gemm", + "args": { + "m": 152, + "n": 1536, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w3" + ] + }, + { + "type": "gemm", + "args": { + "m": 152, + "n": 7168, + "k": 768, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w2" + ] + }, + { + "type": "gemm", + "args": { + "m": 152, + "n": 7168, + "k": 2048, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 152, + "n": 7168, + "k": 4096, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 152, + "n": 8192, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 152, + "n": 16384, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 152, + "n": 32320, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/head", + "deepseek-ai/DeepSeek-V4-Pro/head", + "sgl-project/DeepSeek-V4-Pro-FP8/head" + ] + }, + { + "type": "gemm", + "args": { + "m": 160, + "n": 24, + "k": 28672, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/hc_attn", + "deepseek-ai/DeepSeek-V4-Pro-0813/hc_ffn", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.hc_attn", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.hc_ffn", + "deepseek-ai/DeepSeek-V4-Pro/hc_attn", + "deepseek-ai/DeepSeek-V4-Pro/hc_ffn", + "deepseek-ai/DeepSeek-V4-Pro/mtp.hc_attn", + "deepseek-ai/DeepSeek-V4-Pro/mtp.hc_ffn", + "sgl-project/DeepSeek-V4-Pro-FP8/hc_attn", + "sgl-project/DeepSeek-V4-Pro-FP8/hc_ffn", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.hc_attn", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.hc_ffn" + ] + }, + { + "type": "gemm", + "args": { + "m": 160, + "n": 64, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.weights_proj", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.weights_proj", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.weights_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 160, + "n": 256, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.compressor.wkv", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.compressor.wkv", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.compressor.wgate", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.compressor.wkv" + ] + }, + { + "type": "gemm", + "args": { + "m": 160, + "n": 320, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 160, + "n": 384, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.gate", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.gate", + "deepseek-ai/DeepSeek-V4-Pro/ffn.gate", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.gate", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.gate", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.gate" + ] + }, + { + "type": "gemm", + "args": { + "m": 160, + "n": 512, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wkv", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wkv", + "deepseek-ai/DeepSeek-V4-Pro/attn.wkv", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wkv", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wkv", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wkv" + ] + }, + { + "type": "gemm", + "args": { + "m": 160, + "n": 512, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.compressor.wkv+wgate", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.compressor.wkv+wgate", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.compressor.wkv+wgate" + ] + }, + { + "type": "gemm", + "args": { + "m": 160, + "n": 896, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 160, + "n": 1024, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.compressor.wkv", + "deepseek-ai/DeepSeek-V4-Pro/attn.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro/attn.compressor.wkv", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.compressor.wgate", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.compressor.wkv" + ] + }, + { + "type": "gemm", + "args": { + "m": 160, + "n": 1536, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w3" + ] + }, + { + "type": "gemm", + "args": { + "m": 160, + "n": 4096, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 160, + "n": 7168, + "k": 448, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 160, + "n": 7168, + "k": 768, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w2" + ] + }, + { + "type": "gemm", + "args": { + "m": 160, + "n": 7168, + "k": 1024, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 160, + "n": 7168, + "k": 4096, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 160, + "n": 8192, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 160, + "n": 16384, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 160, + "n": 32320, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/head", + "deepseek-ai/DeepSeek-V4-Pro/head", + "sgl-project/DeepSeek-V4-Pro-FP8/head" + ] + }, + { + "type": "gemm", + "args": { + "m": 161, + "n": 320, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 161, + "n": 4096, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 161, + "n": 7168, + "k": 1024, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 164, + "n": 896, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 164, + "n": 7168, + "k": 448, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 170, + "n": 24, + "k": 49152, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 170, + "n": 64, + "k": 12288, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 170, + "n": 120, + "k": 12288, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 170, + "n": 256, + "k": 12288, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 170, + "n": 512, + "k": 12288, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 170, + "n": 512, + "k": 12288, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 170, + "n": 1024, + "k": 12288, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 170, + "n": 1536, + "k": 12288, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 170, + "n": 5120, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 170, + "n": 12288, + "k": 1024, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 176, + "n": 24, + "k": 28672, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/hc_attn", + "deepseek-ai/DeepSeek-V4-Pro-0813/hc_ffn", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.hc_attn", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.hc_ffn", + "deepseek-ai/DeepSeek-V4-Pro/hc_attn", + "deepseek-ai/DeepSeek-V4-Pro/hc_ffn", + "deepseek-ai/DeepSeek-V4-Pro/mtp.hc_attn", + "deepseek-ai/DeepSeek-V4-Pro/mtp.hc_ffn", + "sgl-project/DeepSeek-V4-Pro-FP8/hc_attn", + "sgl-project/DeepSeek-V4-Pro-FP8/hc_ffn", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.hc_attn", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.hc_ffn" + ] + }, + { + "type": "gemm", + "args": { + "m": 176, + "n": 64, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.weights_proj", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.weights_proj", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.weights_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 176, + "n": 256, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.compressor.wkv", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.compressor.wkv", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.compressor.wgate", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.compressor.wkv" + ] + }, + { + "type": "gemm", + "args": { + "m": 176, + "n": 384, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.gate", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.gate", + "deepseek-ai/DeepSeek-V4-Pro/ffn.gate", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.gate", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.gate", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.gate" + ] + }, + { + "type": "gemm", + "args": { + "m": 176, + "n": 512, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wkv", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wkv", + "deepseek-ai/DeepSeek-V4-Pro/attn.wkv", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wkv", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wkv", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wkv" + ] + }, + { + "type": "gemm", + "args": { + "m": 176, + "n": 512, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.compressor.wkv+wgate", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.compressor.wkv+wgate", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.compressor.wkv+wgate" + ] + }, + { + "type": "gemm", + "args": { + "m": 176, + "n": 1024, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.compressor.wkv", + "deepseek-ai/DeepSeek-V4-Pro/attn.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro/attn.compressor.wkv", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.compressor.wgate", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.compressor.wkv" + ] + }, + { + "type": "gemm", + "args": { + "m": 176, + "n": 1536, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w3" + ] + }, + { + "type": "gemm", + "args": { + "m": 176, + "n": 7168, + "k": 768, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w2" + ] + }, + { + "type": "gemm", + "args": { + "m": 176, + "n": 7168, + "k": 4096, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 176, + "n": 8192, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 176, + "n": 16384, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 176, + "n": 32320, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/head", + "deepseek-ai/DeepSeek-V4-Pro/head", + "sgl-project/DeepSeek-V4-Pro-FP8/head" + ] + }, + { + "type": "gemm", + "args": { + "m": 177, + "n": 32320, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/head", + "deepseek-ai/DeepSeek-V4-Pro/head", + "sgl-project/DeepSeek-V4-Pro-FP8/head" + ] + }, + { + "type": "gemm", + "args": { + "m": 184, + "n": 24, + "k": 28672, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/hc_attn", + "deepseek-ai/DeepSeek-V4-Pro-0813/hc_ffn", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.hc_attn", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.hc_ffn", + "deepseek-ai/DeepSeek-V4-Pro/hc_attn", + "deepseek-ai/DeepSeek-V4-Pro/hc_ffn", + "deepseek-ai/DeepSeek-V4-Pro/mtp.hc_attn", + "deepseek-ai/DeepSeek-V4-Pro/mtp.hc_ffn", + "sgl-project/DeepSeek-V4-Pro-FP8/hc_attn", + "sgl-project/DeepSeek-V4-Pro-FP8/hc_ffn", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.hc_attn", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.hc_ffn" + ] + }, + { + "type": "gemm", + "args": { + "m": 184, + "n": 64, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.weights_proj", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.weights_proj", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.weights_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 184, + "n": 256, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.compressor.wkv", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.compressor.wkv", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.compressor.wgate", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.compressor.wkv" + ] + }, + { + "type": "gemm", + "args": { + "m": 184, + "n": 384, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.gate", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.gate", + "deepseek-ai/DeepSeek-V4-Pro/ffn.gate", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.gate", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.gate", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.gate" + ] + }, + { + "type": "gemm", + "args": { + "m": 184, + "n": 512, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wkv", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wkv", + "deepseek-ai/DeepSeek-V4-Pro/attn.wkv", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wkv", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wkv", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wkv" + ] + }, + { + "type": "gemm", + "args": { + "m": 184, + "n": 512, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.compressor.wkv+wgate", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.compressor.wkv+wgate", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.compressor.wkv+wgate" + ] + }, + { + "type": "gemm", + "args": { + "m": 184, + "n": 1024, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.compressor.wkv", + "deepseek-ai/DeepSeek-V4-Pro/attn.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro/attn.compressor.wkv", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.compressor.wgate", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.compressor.wkv" + ] + }, + { + "type": "gemm", + "args": { + "m": 184, + "n": 1536, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w3" + ] + }, + { + "type": "gemm", + "args": { + "m": 184, + "n": 4096, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 184, + "n": 7168, + "k": 1024, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 184, + "n": 8192, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 185, + "n": 24, + "k": 28672, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/hc_attn", + "deepseek-ai/DeepSeek-V4-Pro-0813/hc_ffn", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.hc_attn", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.hc_ffn", + "deepseek-ai/DeepSeek-V4-Pro/hc_attn", + "deepseek-ai/DeepSeek-V4-Pro/hc_ffn", + "deepseek-ai/DeepSeek-V4-Pro/mtp.hc_attn", + "deepseek-ai/DeepSeek-V4-Pro/mtp.hc_ffn", + "sgl-project/DeepSeek-V4-Pro-FP8/hc_attn", + "sgl-project/DeepSeek-V4-Pro-FP8/hc_ffn", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.hc_attn", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.hc_ffn" + ] + }, + { + "type": "gemm", + "args": { + "m": 185, + "n": 64, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.weights_proj", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.weights_proj", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.weights_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 185, + "n": 256, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.compressor.wkv", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.compressor.wkv", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.compressor.wgate", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.compressor.wkv" + ] + }, + { + "type": "gemm", + "args": { + "m": 185, + "n": 384, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.gate", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.gate", + "deepseek-ai/DeepSeek-V4-Pro/ffn.gate", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.gate", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.gate", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.gate" + ] + }, + { + "type": "gemm", + "args": { + "m": 185, + "n": 512, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wkv", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wkv", + "deepseek-ai/DeepSeek-V4-Pro/attn.wkv", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wkv", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wkv", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wkv" + ] + }, + { + "type": "gemm", + "args": { + "m": 185, + "n": 512, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.compressor.wkv+wgate", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.compressor.wkv+wgate", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.compressor.wkv+wgate" + ] + }, + { + "type": "gemm", + "args": { + "m": 185, + "n": 1024, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.compressor.wkv", + "deepseek-ai/DeepSeek-V4-Pro/attn.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro/attn.compressor.wkv", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.compressor.wgate", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.compressor.wkv" + ] + }, + { + "type": "gemm", + "args": { + "m": 185, + "n": 1536, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w3" + ] + }, + { + "type": "gemm", + "args": { + "m": 185, + "n": 7168, + "k": 768, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w2" + ] + }, + { + "type": "gemm", + "args": { + "m": 185, + "n": 7168, + "k": 4096, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 185, + "n": 8192, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 185, + "n": 16384, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 192, + "n": 24, + "k": 28672, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/hc_attn", + "deepseek-ai/DeepSeek-V4-Pro-0813/hc_ffn", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.hc_attn", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.hc_ffn", + "deepseek-ai/DeepSeek-V4-Pro/hc_attn", + "deepseek-ai/DeepSeek-V4-Pro/hc_ffn", + "deepseek-ai/DeepSeek-V4-Pro/mtp.hc_attn", + "deepseek-ai/DeepSeek-V4-Pro/mtp.hc_ffn", + "sgl-project/DeepSeek-V4-Pro-FP8/hc_attn", + "sgl-project/DeepSeek-V4-Pro-FP8/hc_ffn", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.hc_attn", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.hc_ffn" + ] + }, + { + "type": "gemm", + "args": { + "m": 192, + "n": 24, + "k": 40960, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 192, + "n": 24, + "k": 49152, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 192, + "n": 64, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.weights_proj", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.weights_proj", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.weights_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 192, + "n": 64, + "k": 10240, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 192, + "n": 64, + "k": 12288, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 192, + "n": 120, + "k": 12288, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 192, + "n": 192, + "k": 10240, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 192, + "n": 256, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.compressor.wkv", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.compressor.wkv", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.compressor.wgate", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.compressor.wkv" + ] + }, + { + "type": "gemm", + "args": { + "m": 192, + "n": 256, + "k": 10240, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 192, + "n": 256, + "k": 12288, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 192, + "n": 320, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 192, + "n": 384, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.gate", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.gate", + "deepseek-ai/DeepSeek-V4-Pro/ffn.gate", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.gate", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.gate", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.gate" + ] + }, + { + "type": "gemm", + "args": { + "m": 192, + "n": 384, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w3" + ] + }, + { + "type": "gemm", + "args": { + "m": 192, + "n": 512, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wkv", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wkv", + "deepseek-ai/DeepSeek-V4-Pro/attn.wkv", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wkv", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wkv", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wkv" + ] + }, + { + "type": "gemm", + "args": { + "m": 192, + "n": 512, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.compressor.wkv+wgate", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.compressor.wkv+wgate", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.compressor.wkv+wgate" + ] + }, + { + "type": "gemm", + "args": { + "m": 192, + "n": 512, + "k": 10240, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 192, + "n": 512, + "k": 10240, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 192, + "n": 512, + "k": 12288, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 192, + "n": 512, + "k": 12288, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 192, + "n": 768, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w3" + ] + }, + { + "type": "gemm", + "args": { + "m": 192, + "n": 896, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 192, + "n": 1024, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.compressor.wkv", + "deepseek-ai/DeepSeek-V4-Pro/attn.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro/attn.compressor.wkv", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.compressor.wgate", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.compressor.wkv" + ] + }, + { + "type": "gemm", + "args": { + "m": 192, + "n": 1024, + "k": 10240, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 192, + "n": 1024, + "k": 10240, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 192, + "n": 1024, + "k": 12288, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 192, + "n": 1536, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w3" + ] + }, + { + "type": "gemm", + "args": { + "m": 192, + "n": 1536, + "k": 10240, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 192, + "n": 1536, + "k": 12288, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 192, + "n": 1792, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 192, + "n": 2048, + "k": 12288, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 192, + "n": 4096, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 192, + "n": 5120, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 192, + "n": 6144, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3" + ] + }, + { + "type": "gemm", + "args": { + "m": 192, + "n": 7168, + "k": 192, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w2" + ] + }, + { + "type": "gemm", + "args": { + "m": 192, + "n": 7168, + "k": 384, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w2" + ] + }, + { + "type": "gemm", + "args": { + "m": 192, + "n": 7168, + "k": 448, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 192, + "n": 7168, + "k": 768, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w2" + ] + }, + { + "type": "gemm", + "args": { + "m": 192, + "n": 7168, + "k": 896, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 192, + "n": 7168, + "k": 1024, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 192, + "n": 7168, + "k": 2048, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 192, + "n": 7168, + "k": 3072, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w2" + ] + }, + { + "type": "gemm", + "args": { + "m": 192, + "n": 7168, + "k": 4096, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 192, + "n": 7168, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.e_proj", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.h_proj", + "deepseek-ai/DeepSeek-V4-Pro/mtp.e_proj", + "deepseek-ai/DeepSeek-V4-Pro/mtp.h_proj", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.e_proj", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.h_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 192, + "n": 7168, + "k": 16384, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 192, + "n": 8192, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 192, + "n": 10240, + "k": 512, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 192, + "n": 10240, + "k": 1024, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 192, + "n": 10240, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 192, + "n": 10240, + "k": 2048, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 192, + "n": 10240, + "k": 16384, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 192, + "n": 12288, + "k": 1024, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 192, + "n": 12288, + "k": 2048, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 192, + "n": 12288, + "k": 8192, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 192, + "n": 12288, + "k": 16384, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 192, + "n": 14336, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 192, + "n": 16384, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 192, + "n": 16384, + "k": 12288, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 192, + "n": 32320, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/head", + "deepseek-ai/DeepSeek-V4-Pro/head", + "sgl-project/DeepSeek-V4-Pro-FP8/head" + ] + }, + { + "type": "gemm", + "args": { + "m": 192, + "n": 65536, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 192, + "n": 81920, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 193, + "n": 24, + "k": 40960, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 193, + "n": 64, + "k": 10240, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 193, + "n": 192, + "k": 10240, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 193, + "n": 256, + "k": 10240, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 193, + "n": 384, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w3" + ] + }, + { + "type": "gemm", + "args": { + "m": 193, + "n": 512, + "k": 10240, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 193, + "n": 512, + "k": 10240, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 193, + "n": 768, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w3" + ] + }, + { + "type": "gemm", + "args": { + "m": 193, + "n": 1024, + "k": 10240, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 193, + "n": 1024, + "k": 10240, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 193, + "n": 1536, + "k": 10240, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 193, + "n": 1792, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 193, + "n": 7168, + "k": 192, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w2" + ] + }, + { + "type": "gemm", + "args": { + "m": 193, + "n": 7168, + "k": 384, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w2" + ] + }, + { + "type": "gemm", + "args": { + "m": 193, + "n": 7168, + "k": 896, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 193, + "n": 7168, + "k": 2048, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 193, + "n": 10240, + "k": 512, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 193, + "n": 10240, + "k": 1024, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 193, + "n": 10240, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 193, + "n": 10240, + "k": 2048, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 193, + "n": 12288, + "k": 2048, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 200, + "n": 24, + "k": 40960, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 200, + "n": 24, + "k": 49152, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 200, + "n": 64, + "k": 10240, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 200, + "n": 64, + "k": 12288, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 200, + "n": 120, + "k": 12288, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 200, + "n": 192, + "k": 10240, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 200, + "n": 256, + "k": 10240, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 200, + "n": 256, + "k": 12288, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 200, + "n": 320, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 200, + "n": 384, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w3" + ] + }, + { + "type": "gemm", + "args": { + "m": 200, + "n": 512, + "k": 10240, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 200, + "n": 512, + "k": 10240, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 200, + "n": 512, + "k": 12288, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 200, + "n": 512, + "k": 12288, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 200, + "n": 768, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w3" + ] + }, + { + "type": "gemm", + "args": { + "m": 200, + "n": 896, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 200, + "n": 1024, + "k": 10240, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 200, + "n": 1024, + "k": 12288, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 200, + "n": 1536, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w3" + ] + }, + { + "type": "gemm", + "args": { + "m": 200, + "n": 1536, + "k": 10240, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 200, + "n": 1536, + "k": 12288, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 200, + "n": 1792, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 200, + "n": 5120, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 200, + "n": 7168, + "k": 192, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w2" + ] + }, + { + "type": "gemm", + "args": { + "m": 200, + "n": 7168, + "k": 384, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w2" + ] + }, + { + "type": "gemm", + "args": { + "m": 200, + "n": 7168, + "k": 448, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 200, + "n": 7168, + "k": 768, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w2" + ] + }, + { + "type": "gemm", + "args": { + "m": 200, + "n": 7168, + "k": 896, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 200, + "n": 7168, + "k": 2048, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 200, + "n": 7168, + "k": 4096, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 200, + "n": 10240, + "k": 1024, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 200, + "n": 10240, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 200, + "n": 10240, + "k": 2048, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 200, + "n": 12288, + "k": 1024, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 200, + "n": 12288, + "k": 2048, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 200, + "n": 16384, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 202, + "n": 320, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 202, + "n": 4096, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 202, + "n": 7168, + "k": 1024, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 206, + "n": 896, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 206, + "n": 7168, + "k": 448, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 208, + "n": 24, + "k": 28672, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/hc_attn", + "deepseek-ai/DeepSeek-V4-Pro-0813/hc_ffn", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.hc_attn", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.hc_ffn", + "deepseek-ai/DeepSeek-V4-Pro/hc_attn", + "deepseek-ai/DeepSeek-V4-Pro/hc_ffn", + "deepseek-ai/DeepSeek-V4-Pro/mtp.hc_attn", + "deepseek-ai/DeepSeek-V4-Pro/mtp.hc_ffn", + "sgl-project/DeepSeek-V4-Pro-FP8/hc_attn", + "sgl-project/DeepSeek-V4-Pro-FP8/hc_ffn", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.hc_attn", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.hc_ffn" + ] + }, + { + "type": "gemm", + "args": { + "m": 208, + "n": 24, + "k": 40960, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 208, + "n": 24, + "k": 49152, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 208, + "n": 64, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.weights_proj", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.weights_proj", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.weights_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 208, + "n": 64, + "k": 10240, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 208, + "n": 64, + "k": 12288, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 208, + "n": 120, + "k": 12288, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 208, + "n": 192, + "k": 10240, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 208, + "n": 256, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.compressor.wkv", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.compressor.wkv", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.compressor.wgate", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.compressor.wkv" + ] + }, + { + "type": "gemm", + "args": { + "m": 208, + "n": 256, + "k": 10240, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 208, + "n": 256, + "k": 12288, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 208, + "n": 320, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 208, + "n": 384, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.gate", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.gate", + "deepseek-ai/DeepSeek-V4-Pro/ffn.gate", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.gate", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.gate", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.gate" + ] + }, + { + "type": "gemm", + "args": { + "m": 208, + "n": 512, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wkv", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wkv", + "deepseek-ai/DeepSeek-V4-Pro/attn.wkv", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wkv", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wkv", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wkv" + ] + }, + { + "type": "gemm", + "args": { + "m": 208, + "n": 512, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.compressor.wkv+wgate", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.compressor.wkv+wgate", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.compressor.wkv+wgate" + ] + }, + { + "type": "gemm", + "args": { + "m": 208, + "n": 512, + "k": 10240, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 208, + "n": 512, + "k": 10240, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 208, + "n": 512, + "k": 12288, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 208, + "n": 512, + "k": 12288, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 208, + "n": 768, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w3" + ] + }, + { + "type": "gemm", + "args": { + "m": 208, + "n": 896, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 208, + "n": 1024, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.compressor.wkv", + "deepseek-ai/DeepSeek-V4-Pro/attn.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro/attn.compressor.wkv", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.compressor.wgate", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.compressor.wkv" + ] + }, + { + "type": "gemm", + "args": { + "m": 208, + "n": 1024, + "k": 10240, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 208, + "n": 1024, + "k": 12288, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 208, + "n": 1536, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w3" + ] + }, + { + "type": "gemm", + "args": { + "m": 208, + "n": 1536, + "k": 10240, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 208, + "n": 1536, + "k": 12288, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 208, + "n": 4096, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 208, + "n": 5120, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 208, + "n": 7168, + "k": 384, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w2" + ] + }, + { + "type": "gemm", + "args": { + "m": 208, + "n": 7168, + "k": 448, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 208, + "n": 7168, + "k": 768, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w2" + ] + }, + { + "type": "gemm", + "args": { + "m": 208, + "n": 7168, + "k": 1024, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 208, + "n": 7168, + "k": 2048, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 208, + "n": 7168, + "k": 4096, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 208, + "n": 8192, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 208, + "n": 10240, + "k": 1024, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 208, + "n": 10240, + "k": 2048, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 208, + "n": 12288, + "k": 1024, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 208, + "n": 16384, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 213, + "n": 24, + "k": 49152, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 213, + "n": 64, + "k": 12288, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 213, + "n": 120, + "k": 12288, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 213, + "n": 256, + "k": 12288, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 213, + "n": 512, + "k": 12288, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 213, + "n": 512, + "k": 12288, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 213, + "n": 1024, + "k": 12288, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 213, + "n": 1536, + "k": 12288, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 213, + "n": 5120, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 213, + "n": 12288, + "k": 1024, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 216, + "n": 320, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 216, + "n": 896, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 216, + "n": 1024, + "k": 10240, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 216, + "n": 4096, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 216, + "n": 7168, + "k": 448, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 216, + "n": 7168, + "k": 1024, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 216, + "n": 10240, + "k": 512, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 216, + "n": 32320, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/head", + "deepseek-ai/DeepSeek-V4-Pro/head", + "sgl-project/DeepSeek-V4-Pro-FP8/head" + ] + }, + { + "type": "gemm", + "args": { + "m": 223, + "n": 32320, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/head", + "deepseek-ai/DeepSeek-V4-Pro/head", + "sgl-project/DeepSeek-V4-Pro-FP8/head" + ] + }, + { + "type": "gemm", + "args": { + "m": 224, + "n": 24, + "k": 28672, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/hc_attn", + "deepseek-ai/DeepSeek-V4-Pro-0813/hc_ffn", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.hc_attn", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.hc_ffn", + "deepseek-ai/DeepSeek-V4-Pro/hc_attn", + "deepseek-ai/DeepSeek-V4-Pro/hc_ffn", + "deepseek-ai/DeepSeek-V4-Pro/mtp.hc_attn", + "deepseek-ai/DeepSeek-V4-Pro/mtp.hc_ffn", + "sgl-project/DeepSeek-V4-Pro-FP8/hc_attn", + "sgl-project/DeepSeek-V4-Pro-FP8/hc_ffn", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.hc_attn", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.hc_ffn" + ] + }, + { + "type": "gemm", + "args": { + "m": 224, + "n": 64, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.weights_proj", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.weights_proj", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.weights_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 224, + "n": 256, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.compressor.wkv", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.compressor.wkv", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.compressor.wgate", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.compressor.wkv" + ] + }, + { + "type": "gemm", + "args": { + "m": 224, + "n": 384, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.gate", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.gate", + "deepseek-ai/DeepSeek-V4-Pro/ffn.gate", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.gate", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.gate", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.gate" + ] + }, + { + "type": "gemm", + "args": { + "m": 224, + "n": 512, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wkv", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wkv", + "deepseek-ai/DeepSeek-V4-Pro/attn.wkv", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wkv", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wkv", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wkv" + ] + }, + { + "type": "gemm", + "args": { + "m": 224, + "n": 512, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.compressor.wkv+wgate", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.compressor.wkv+wgate", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.compressor.wkv+wgate" + ] + }, + { + "type": "gemm", + "args": { + "m": 224, + "n": 1024, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.compressor.wkv", + "deepseek-ai/DeepSeek-V4-Pro/attn.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro/attn.compressor.wkv", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.compressor.wgate", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.compressor.wkv" + ] + }, + { + "type": "gemm", + "args": { + "m": 224, + "n": 1536, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w3" + ] + }, + { + "type": "gemm", + "args": { + "m": 224, + "n": 7168, + "k": 768, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w2" + ] + }, + { + "type": "gemm", + "args": { + "m": 224, + "n": 7168, + "k": 4096, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 224, + "n": 8192, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 224, + "n": 16384, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 231, + "n": 24, + "k": 28672, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/hc_attn", + "deepseek-ai/DeepSeek-V4-Pro-0813/hc_ffn", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.hc_attn", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.hc_ffn", + "deepseek-ai/DeepSeek-V4-Pro/hc_attn", + "deepseek-ai/DeepSeek-V4-Pro/hc_ffn", + "deepseek-ai/DeepSeek-V4-Pro/mtp.hc_attn", + "deepseek-ai/DeepSeek-V4-Pro/mtp.hc_ffn", + "sgl-project/DeepSeek-V4-Pro-FP8/hc_attn", + "sgl-project/DeepSeek-V4-Pro-FP8/hc_ffn", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.hc_attn", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.hc_ffn" + ] + }, + { + "type": "gemm", + "args": { + "m": 231, + "n": 64, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.weights_proj", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.weights_proj", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.weights_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 231, + "n": 256, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.compressor.wkv", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.compressor.wkv", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.compressor.wgate", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.compressor.wkv" + ] + }, + { + "type": "gemm", + "args": { + "m": 231, + "n": 384, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.gate", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.gate", + "deepseek-ai/DeepSeek-V4-Pro/ffn.gate", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.gate", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.gate", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.gate" + ] + }, + { + "type": "gemm", + "args": { + "m": 231, + "n": 512, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wkv", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wkv", + "deepseek-ai/DeepSeek-V4-Pro/attn.wkv", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wkv", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wkv", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wkv" + ] + }, + { + "type": "gemm", + "args": { + "m": 231, + "n": 512, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.compressor.wkv+wgate", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.compressor.wkv+wgate", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.compressor.wkv+wgate" + ] + }, + { + "type": "gemm", + "args": { + "m": 231, + "n": 1024, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.compressor.wkv", + "deepseek-ai/DeepSeek-V4-Pro/attn.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro/attn.compressor.wkv", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.compressor.wgate", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.compressor.wkv" + ] + }, + { + "type": "gemm", + "args": { + "m": 231, + "n": 1536, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w3" + ] + }, + { + "type": "gemm", + "args": { + "m": 231, + "n": 8192, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 232, + "n": 24, + "k": 28672, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/hc_attn", + "deepseek-ai/DeepSeek-V4-Pro-0813/hc_ffn", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.hc_attn", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.hc_ffn", + "deepseek-ai/DeepSeek-V4-Pro/hc_attn", + "deepseek-ai/DeepSeek-V4-Pro/hc_ffn", + "deepseek-ai/DeepSeek-V4-Pro/mtp.hc_attn", + "deepseek-ai/DeepSeek-V4-Pro/mtp.hc_ffn", + "sgl-project/DeepSeek-V4-Pro-FP8/hc_attn", + "sgl-project/DeepSeek-V4-Pro-FP8/hc_ffn", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.hc_attn", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.hc_ffn" + ] + }, + { + "type": "gemm", + "args": { + "m": 232, + "n": 64, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.weights_proj", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.weights_proj", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.weights_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 232, + "n": 256, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.compressor.wkv", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.compressor.wkv", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.compressor.wgate", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.compressor.wkv" + ] + }, + { + "type": "gemm", + "args": { + "m": 232, + "n": 384, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.gate", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.gate", + "deepseek-ai/DeepSeek-V4-Pro/ffn.gate", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.gate", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.gate", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.gate" + ] + }, + { + "type": "gemm", + "args": { + "m": 232, + "n": 512, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wkv", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wkv", + "deepseek-ai/DeepSeek-V4-Pro/attn.wkv", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wkv", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wkv", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wkv" + ] + }, + { + "type": "gemm", + "args": { + "m": 232, + "n": 512, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.compressor.wkv+wgate", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.compressor.wkv+wgate", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.compressor.wkv+wgate" + ] + }, + { + "type": "gemm", + "args": { + "m": 232, + "n": 1024, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.compressor.wkv", + "deepseek-ai/DeepSeek-V4-Pro/attn.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro/attn.compressor.wkv", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.compressor.wgate", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.compressor.wkv" + ] + }, + { + "type": "gemm", + "args": { + "m": 232, + "n": 1536, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w3" + ] + }, + { + "type": "gemm", + "args": { + "m": 232, + "n": 7168, + "k": 768, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w2" + ] + }, + { + "type": "gemm", + "args": { + "m": 232, + "n": 7168, + "k": 4096, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 232, + "n": 8192, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 232, + "n": 16384, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 233, + "n": 1536, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w3" + ] + }, + { + "type": "gemm", + "args": { + "m": 233, + "n": 7168, + "k": 768, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w2" + ] + }, + { + "type": "gemm", + "args": { + "m": 233, + "n": 7168, + "k": 4096, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 233, + "n": 16384, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 240, + "n": 24, + "k": 28672, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/hc_attn", + "deepseek-ai/DeepSeek-V4-Pro-0813/hc_ffn", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.hc_attn", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.hc_ffn", + "deepseek-ai/DeepSeek-V4-Pro/hc_attn", + "deepseek-ai/DeepSeek-V4-Pro/hc_ffn", + "deepseek-ai/DeepSeek-V4-Pro/mtp.hc_attn", + "deepseek-ai/DeepSeek-V4-Pro/mtp.hc_ffn", + "sgl-project/DeepSeek-V4-Pro-FP8/hc_attn", + "sgl-project/DeepSeek-V4-Pro-FP8/hc_ffn", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.hc_attn", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.hc_ffn" + ] + }, + { + "type": "gemm", + "args": { + "m": 240, + "n": 64, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.weights_proj", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.weights_proj", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.weights_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 240, + "n": 256, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.compressor.wkv", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.compressor.wkv", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.compressor.wgate", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.compressor.wkv" + ] + }, + { + "type": "gemm", + "args": { + "m": 240, + "n": 384, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.gate", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.gate", + "deepseek-ai/DeepSeek-V4-Pro/ffn.gate", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.gate", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.gate", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.gate" + ] + }, + { + "type": "gemm", + "args": { + "m": 240, + "n": 512, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wkv", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wkv", + "deepseek-ai/DeepSeek-V4-Pro/attn.wkv", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wkv", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wkv", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wkv" + ] + }, + { + "type": "gemm", + "args": { + "m": 240, + "n": 512, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.compressor.wkv+wgate", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.compressor.wkv+wgate", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.compressor.wkv+wgate" + ] + }, + { + "type": "gemm", + "args": { + "m": 240, + "n": 1024, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.compressor.wkv", + "deepseek-ai/DeepSeek-V4-Pro/attn.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro/attn.compressor.wkv", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.compressor.wgate", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.compressor.wkv" + ] + }, + { + "type": "gemm", + "args": { + "m": 240, + "n": 1536, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w3" + ] + }, + { + "type": "gemm", + "args": { + "m": 240, + "n": 7168, + "k": 768, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w2" + ] + }, + { + "type": "gemm", + "args": { + "m": 240, + "n": 7168, + "k": 4096, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 240, + "n": 8192, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 240, + "n": 16384, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 256, + "n": 24, + "k": 28672, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/hc_attn", + "deepseek-ai/DeepSeek-V4-Pro-0813/hc_ffn", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.hc_attn", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.hc_ffn", + "deepseek-ai/DeepSeek-V4-Pro/hc_attn", + "deepseek-ai/DeepSeek-V4-Pro/hc_ffn", + "deepseek-ai/DeepSeek-V4-Pro/mtp.hc_attn", + "deepseek-ai/DeepSeek-V4-Pro/mtp.hc_ffn", + "sgl-project/DeepSeek-V4-Pro-FP8/hc_attn", + "sgl-project/DeepSeek-V4-Pro-FP8/hc_ffn", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.hc_attn", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.hc_ffn" + ] + }, + { + "type": "gemm", + "args": { + "m": 256, + "n": 24, + "k": 40960, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 256, + "n": 24, + "k": 49152, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 256, + "n": 64, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.weights_proj", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.weights_proj", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.weights_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 256, + "n": 64, + "k": 10240, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 256, + "n": 64, + "k": 12288, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 256, + "n": 120, + "k": 12288, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 256, + "n": 192, + "k": 10240, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 256, + "n": 256, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.compressor.wkv", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.compressor.wkv", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.compressor.wgate", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.compressor.wkv" + ] + }, + { + "type": "gemm", + "args": { + "m": 256, + "n": 256, + "k": 10240, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 256, + "n": 256, + "k": 12288, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 256, + "n": 320, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 256, + "n": 384, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.gate", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.gate", + "deepseek-ai/DeepSeek-V4-Pro/ffn.gate", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.gate", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.gate", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.gate" + ] + }, + { + "type": "gemm", + "args": { + "m": 256, + "n": 384, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w3" + ] + }, + { + "type": "gemm", + "args": { + "m": 256, + "n": 512, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wkv", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wkv", + "deepseek-ai/DeepSeek-V4-Pro/attn.wkv", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wkv", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wkv", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wkv" + ] + }, + { + "type": "gemm", + "args": { + "m": 256, + "n": 512, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.compressor.wkv+wgate", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.compressor.wkv+wgate", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.compressor.wkv+wgate" + ] + }, + { + "type": "gemm", + "args": { + "m": 256, + "n": 512, + "k": 10240, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 256, + "n": 512, + "k": 10240, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 256, + "n": 512, + "k": 12288, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 256, + "n": 512, + "k": 12288, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 256, + "n": 768, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w3" + ] + }, + { + "type": "gemm", + "args": { + "m": 256, + "n": 1024, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.compressor.wkv", + "deepseek-ai/DeepSeek-V4-Pro/attn.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro/attn.compressor.wkv", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.compressor.wgate", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.compressor.wkv" + ] + }, + { + "type": "gemm", + "args": { + "m": 256, + "n": 1024, + "k": 10240, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 256, + "n": 1024, + "k": 10240, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 256, + "n": 1024, + "k": 12288, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 256, + "n": 1536, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w3" + ] + }, + { + "type": "gemm", + "args": { + "m": 256, + "n": 1536, + "k": 10240, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 256, + "n": 1536, + "k": 12288, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 256, + "n": 1792, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 256, + "n": 2048, + "k": 12288, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 256, + "n": 4096, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 256, + "n": 5120, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 256, + "n": 6144, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3" + ] + }, + { + "type": "gemm", + "args": { + "m": 256, + "n": 7168, + "k": 192, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w2" + ] + }, + { + "type": "gemm", + "args": { + "m": 256, + "n": 7168, + "k": 384, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w2" + ] + }, + { + "type": "gemm", + "args": { + "m": 256, + "n": 7168, + "k": 768, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w2" + ] + }, + { + "type": "gemm", + "args": { + "m": 256, + "n": 7168, + "k": 896, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 256, + "n": 7168, + "k": 1024, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 256, + "n": 7168, + "k": 2048, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 256, + "n": 7168, + "k": 3072, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w2" + ] + }, + { + "type": "gemm", + "args": { + "m": 256, + "n": 7168, + "k": 4096, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 256, + "n": 7168, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.e_proj", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.h_proj", + "deepseek-ai/DeepSeek-V4-Pro/mtp.e_proj", + "deepseek-ai/DeepSeek-V4-Pro/mtp.h_proj", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.e_proj", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.h_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 256, + "n": 7168, + "k": 16384, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 256, + "n": 8192, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 256, + "n": 10240, + "k": 512, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 256, + "n": 10240, + "k": 1024, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 256, + "n": 10240, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 256, + "n": 10240, + "k": 2048, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 256, + "n": 10240, + "k": 16384, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 256, + "n": 12288, + "k": 1024, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 256, + "n": 12288, + "k": 2048, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 256, + "n": 12288, + "k": 8192, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 256, + "n": 12288, + "k": 16384, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 256, + "n": 14336, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 256, + "n": 16384, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 256, + "n": 16384, + "k": 12288, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 256, + "n": 65536, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 256, + "n": 81920, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 257, + "n": 24, + "k": 40960, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 257, + "n": 64, + "k": 10240, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 257, + "n": 192, + "k": 10240, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 257, + "n": 256, + "k": 10240, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 257, + "n": 320, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 257, + "n": 384, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w3" + ] + }, + { + "type": "gemm", + "args": { + "m": 257, + "n": 512, + "k": 10240, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 257, + "n": 512, + "k": 10240, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 257, + "n": 768, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w3" + ] + }, + { + "type": "gemm", + "args": { + "m": 257, + "n": 1024, + "k": 10240, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 257, + "n": 1024, + "k": 10240, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 257, + "n": 1536, + "k": 10240, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 257, + "n": 1792, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 257, + "n": 4096, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 257, + "n": 7168, + "k": 192, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w2" + ] + }, + { + "type": "gemm", + "args": { + "m": 257, + "n": 7168, + "k": 384, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w2" + ] + }, + { + "type": "gemm", + "args": { + "m": 257, + "n": 7168, + "k": 896, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 257, + "n": 7168, + "k": 1024, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 257, + "n": 7168, + "k": 2048, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 257, + "n": 10240, + "k": 512, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 257, + "n": 10240, + "k": 1024, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 257, + "n": 10240, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 257, + "n": 10240, + "k": 2048, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 257, + "n": 12288, + "k": 2048, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 258, + "n": 896, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 258, + "n": 7168, + "k": 448, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 264, + "n": 24, + "k": 40960, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 264, + "n": 24, + "k": 49152, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 264, + "n": 64, + "k": 10240, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 264, + "n": 64, + "k": 12288, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 264, + "n": 120, + "k": 12288, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 264, + "n": 192, + "k": 10240, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 264, + "n": 256, + "k": 10240, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 264, + "n": 256, + "k": 12288, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 264, + "n": 320, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 264, + "n": 384, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w3" + ] + }, + { + "type": "gemm", + "args": { + "m": 264, + "n": 512, + "k": 10240, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 264, + "n": 512, + "k": 10240, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 264, + "n": 512, + "k": 12288, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 264, + "n": 512, + "k": 12288, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 264, + "n": 768, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w3" + ] + }, + { + "type": "gemm", + "args": { + "m": 264, + "n": 896, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 264, + "n": 1024, + "k": 10240, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 264, + "n": 1024, + "k": 12288, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 264, + "n": 1536, + "k": 10240, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 264, + "n": 1536, + "k": 12288, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 264, + "n": 1792, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 264, + "n": 4096, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 264, + "n": 5120, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 264, + "n": 7168, + "k": 192, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w2" + ] + }, + { + "type": "gemm", + "args": { + "m": 264, + "n": 7168, + "k": 384, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w2" + ] + }, + { + "type": "gemm", + "args": { + "m": 264, + "n": 7168, + "k": 448, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 264, + "n": 7168, + "k": 896, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 264, + "n": 7168, + "k": 1024, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 264, + "n": 7168, + "k": 2048, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 264, + "n": 10240, + "k": 1024, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 264, + "n": 10240, + "k": 2048, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 264, + "n": 12288, + "k": 1024, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 267, + "n": 24, + "k": 49152, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 267, + "n": 64, + "k": 12288, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 267, + "n": 120, + "k": 12288, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 267, + "n": 256, + "k": 12288, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 267, + "n": 512, + "k": 12288, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 267, + "n": 512, + "k": 12288, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 267, + "n": 1024, + "k": 12288, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 267, + "n": 1536, + "k": 12288, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 267, + "n": 5120, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 267, + "n": 12288, + "k": 1024, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 272, + "n": 24, + "k": 40960, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 272, + "n": 24, + "k": 49152, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 272, + "n": 64, + "k": 10240, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 272, + "n": 64, + "k": 12288, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 272, + "n": 120, + "k": 12288, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 272, + "n": 192, + "k": 10240, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 272, + "n": 256, + "k": 10240, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 272, + "n": 256, + "k": 12288, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 272, + "n": 320, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 272, + "n": 384, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w3" + ] + }, + { + "type": "gemm", + "args": { + "m": 272, + "n": 512, + "k": 10240, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 272, + "n": 512, + "k": 10240, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 272, + "n": 512, + "k": 12288, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 272, + "n": 512, + "k": 12288, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 272, + "n": 768, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w3" + ] + }, + { + "type": "gemm", + "args": { + "m": 272, + "n": 896, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 272, + "n": 1024, + "k": 10240, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 272, + "n": 1024, + "k": 12288, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 272, + "n": 1536, + "k": 10240, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 272, + "n": 1536, + "k": 12288, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 272, + "n": 1792, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 272, + "n": 4096, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 272, + "n": 5120, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 272, + "n": 7168, + "k": 192, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w2" + ] + }, + { + "type": "gemm", + "args": { + "m": 272, + "n": 7168, + "k": 384, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w2" + ] + }, + { + "type": "gemm", + "args": { + "m": 272, + "n": 7168, + "k": 448, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 272, + "n": 7168, + "k": 896, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 272, + "n": 7168, + "k": 1024, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 272, + "n": 7168, + "k": 2048, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 272, + "n": 10240, + "k": 1024, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 272, + "n": 10240, + "k": 2048, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 272, + "n": 12288, + "k": 1024, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 280, + "n": 24, + "k": 49152, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 280, + "n": 64, + "k": 12288, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 280, + "n": 120, + "k": 12288, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 280, + "n": 256, + "k": 12288, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 280, + "n": 512, + "k": 12288, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 280, + "n": 512, + "k": 12288, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 280, + "n": 1024, + "k": 12288, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 280, + "n": 1536, + "k": 12288, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 280, + "n": 5120, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 280, + "n": 12288, + "k": 1024, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 288, + "n": 24, + "k": 28672, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/hc_attn", + "deepseek-ai/DeepSeek-V4-Pro-0813/hc_ffn", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.hc_attn", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.hc_ffn", + "deepseek-ai/DeepSeek-V4-Pro/hc_attn", + "deepseek-ai/DeepSeek-V4-Pro/hc_ffn", + "deepseek-ai/DeepSeek-V4-Pro/mtp.hc_attn", + "deepseek-ai/DeepSeek-V4-Pro/mtp.hc_ffn", + "sgl-project/DeepSeek-V4-Pro-FP8/hc_attn", + "sgl-project/DeepSeek-V4-Pro-FP8/hc_ffn", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.hc_attn", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.hc_ffn" + ] + }, + { + "type": "gemm", + "args": { + "m": 288, + "n": 64, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.weights_proj", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.weights_proj", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.weights_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 288, + "n": 256, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.compressor.wkv", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.compressor.wkv", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.compressor.wgate", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.compressor.wkv" + ] + }, + { + "type": "gemm", + "args": { + "m": 288, + "n": 320, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 288, + "n": 384, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.gate", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.gate", + "deepseek-ai/DeepSeek-V4-Pro/ffn.gate", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.gate", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.gate", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.gate" + ] + }, + { + "type": "gemm", + "args": { + "m": 288, + "n": 512, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wkv", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wkv", + "deepseek-ai/DeepSeek-V4-Pro/attn.wkv", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wkv", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wkv", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wkv" + ] + }, + { + "type": "gemm", + "args": { + "m": 288, + "n": 512, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.compressor.wkv+wgate", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.compressor.wkv+wgate", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.compressor.wkv+wgate" + ] + }, + { + "type": "gemm", + "args": { + "m": 288, + "n": 896, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 288, + "n": 1024, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.compressor.wkv", + "deepseek-ai/DeepSeek-V4-Pro/attn.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro/attn.compressor.wkv", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.compressor.wgate", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.compressor.wkv" + ] + }, + { + "type": "gemm", + "args": { + "m": 288, + "n": 1536, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w3" + ] + }, + { + "type": "gemm", + "args": { + "m": 288, + "n": 4096, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 288, + "n": 7168, + "k": 448, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 288, + "n": 7168, + "k": 768, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w2" + ] + }, + { + "type": "gemm", + "args": { + "m": 288, + "n": 7168, + "k": 1024, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 288, + "n": 7168, + "k": 4096, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 288, + "n": 8192, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 288, + "n": 16384, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 289, + "n": 24, + "k": 28672, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/hc_attn", + "deepseek-ai/DeepSeek-V4-Pro-0813/hc_ffn", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.hc_attn", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.hc_ffn", + "deepseek-ai/DeepSeek-V4-Pro/hc_attn", + "deepseek-ai/DeepSeek-V4-Pro/hc_ffn", + "deepseek-ai/DeepSeek-V4-Pro/mtp.hc_attn", + "deepseek-ai/DeepSeek-V4-Pro/mtp.hc_ffn", + "sgl-project/DeepSeek-V4-Pro-FP8/hc_attn", + "sgl-project/DeepSeek-V4-Pro-FP8/hc_ffn", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.hc_attn", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.hc_ffn" + ] + }, + { + "type": "gemm", + "args": { + "m": 289, + "n": 64, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.weights_proj", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.weights_proj", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.weights_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 289, + "n": 256, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.compressor.wkv", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.compressor.wkv", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.compressor.wgate", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.compressor.wkv" + ] + }, + { + "type": "gemm", + "args": { + "m": 289, + "n": 384, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.gate", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.gate", + "deepseek-ai/DeepSeek-V4-Pro/ffn.gate", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.gate", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.gate", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.gate" + ] + }, + { + "type": "gemm", + "args": { + "m": 289, + "n": 512, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wkv", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wkv", + "deepseek-ai/DeepSeek-V4-Pro/attn.wkv", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wkv", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wkv", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wkv" + ] + }, + { + "type": "gemm", + "args": { + "m": 289, + "n": 512, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.compressor.wkv+wgate", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.compressor.wkv+wgate", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.compressor.wkv+wgate" + ] + }, + { + "type": "gemm", + "args": { + "m": 289, + "n": 1024, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.compressor.wkv", + "deepseek-ai/DeepSeek-V4-Pro/attn.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro/attn.compressor.wkv", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.compressor.wgate", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.compressor.wkv" + ] + }, + { + "type": "gemm", + "args": { + "m": 289, + "n": 1536, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w3" + ] + }, + { + "type": "gemm", + "args": { + "m": 289, + "n": 8192, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 291, + "n": 1536, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w3" + ] + }, + { + "type": "gemm", + "args": { + "m": 291, + "n": 7168, + "k": 768, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w2" + ] + }, + { + "type": "gemm", + "args": { + "m": 291, + "n": 7168, + "k": 4096, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 291, + "n": 16384, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 296, + "n": 24, + "k": 28672, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/hc_attn", + "deepseek-ai/DeepSeek-V4-Pro-0813/hc_ffn", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.hc_attn", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.hc_ffn", + "deepseek-ai/DeepSeek-V4-Pro/hc_attn", + "deepseek-ai/DeepSeek-V4-Pro/hc_ffn", + "deepseek-ai/DeepSeek-V4-Pro/mtp.hc_attn", + "deepseek-ai/DeepSeek-V4-Pro/mtp.hc_ffn", + "sgl-project/DeepSeek-V4-Pro-FP8/hc_attn", + "sgl-project/DeepSeek-V4-Pro-FP8/hc_ffn", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.hc_attn", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.hc_ffn" + ] + }, + { + "type": "gemm", + "args": { + "m": 296, + "n": 64, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.weights_proj", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.weights_proj", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.weights_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 296, + "n": 256, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.compressor.wkv", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.compressor.wkv", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.compressor.wgate", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.compressor.wkv" + ] + }, + { + "type": "gemm", + "args": { + "m": 296, + "n": 384, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.gate", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.gate", + "deepseek-ai/DeepSeek-V4-Pro/ffn.gate", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.gate", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.gate", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.gate" + ] + }, + { + "type": "gemm", + "args": { + "m": 296, + "n": 512, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wkv", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wkv", + "deepseek-ai/DeepSeek-V4-Pro/attn.wkv", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wkv", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wkv", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wkv" + ] + }, + { + "type": "gemm", + "args": { + "m": 296, + "n": 512, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.compressor.wkv+wgate", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.compressor.wkv+wgate", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.compressor.wkv+wgate" + ] + }, + { + "type": "gemm", + "args": { + "m": 296, + "n": 1024, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.compressor.wkv", + "deepseek-ai/DeepSeek-V4-Pro/attn.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro/attn.compressor.wkv", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.compressor.wgate", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.compressor.wkv" + ] + }, + { + "type": "gemm", + "args": { + "m": 296, + "n": 1536, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w3" + ] + }, + { + "type": "gemm", + "args": { + "m": 296, + "n": 7168, + "k": 768, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w2" + ] + }, + { + "type": "gemm", + "args": { + "m": 296, + "n": 7168, + "k": 4096, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 296, + "n": 8192, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 296, + "n": 16384, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 304, + "n": 24, + "k": 28672, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/hc_attn", + "deepseek-ai/DeepSeek-V4-Pro-0813/hc_ffn", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.hc_attn", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.hc_ffn", + "deepseek-ai/DeepSeek-V4-Pro/hc_attn", + "deepseek-ai/DeepSeek-V4-Pro/hc_ffn", + "deepseek-ai/DeepSeek-V4-Pro/mtp.hc_attn", + "deepseek-ai/DeepSeek-V4-Pro/mtp.hc_ffn", + "sgl-project/DeepSeek-V4-Pro-FP8/hc_attn", + "sgl-project/DeepSeek-V4-Pro-FP8/hc_ffn", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.hc_attn", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.hc_ffn" + ] + }, + { + "type": "gemm", + "args": { + "m": 304, + "n": 64, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.weights_proj", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.weights_proj", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.weights_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 304, + "n": 256, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.compressor.wkv", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.compressor.wkv", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.compressor.wgate", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.compressor.wkv" + ] + }, + { + "type": "gemm", + "args": { + "m": 304, + "n": 384, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.gate", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.gate", + "deepseek-ai/DeepSeek-V4-Pro/ffn.gate", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.gate", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.gate", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.gate" + ] + }, + { + "type": "gemm", + "args": { + "m": 304, + "n": 512, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wkv", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wkv", + "deepseek-ai/DeepSeek-V4-Pro/attn.wkv", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wkv", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wkv", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wkv" + ] + }, + { + "type": "gemm", + "args": { + "m": 304, + "n": 512, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.compressor.wkv+wgate", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.compressor.wkv+wgate", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.compressor.wkv+wgate" + ] + }, + { + "type": "gemm", + "args": { + "m": 304, + "n": 1024, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.compressor.wkv", + "deepseek-ai/DeepSeek-V4-Pro/attn.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro/attn.compressor.wkv", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.compressor.wgate", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.compressor.wkv" + ] + }, + { + "type": "gemm", + "args": { + "m": 304, + "n": 1536, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w3" + ] + }, + { + "type": "gemm", + "args": { + "m": 304, + "n": 7168, + "k": 768, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w2" + ] + }, + { + "type": "gemm", + "args": { + "m": 304, + "n": 7168, + "k": 4096, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 304, + "n": 8192, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 304, + "n": 16384, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 320, + "n": 24, + "k": 28672, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/hc_attn", + "deepseek-ai/DeepSeek-V4-Pro-0813/hc_ffn", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.hc_attn", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.hc_ffn", + "deepseek-ai/DeepSeek-V4-Pro/hc_attn", + "deepseek-ai/DeepSeek-V4-Pro/hc_ffn", + "deepseek-ai/DeepSeek-V4-Pro/mtp.hc_attn", + "deepseek-ai/DeepSeek-V4-Pro/mtp.hc_ffn", + "sgl-project/DeepSeek-V4-Pro-FP8/hc_attn", + "sgl-project/DeepSeek-V4-Pro-FP8/hc_ffn", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.hc_attn", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.hc_ffn" + ] + }, + { + "type": "gemm", + "args": { + "m": 320, + "n": 24, + "k": 40960, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 320, + "n": 24, + "k": 49152, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 320, + "n": 64, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.weights_proj", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.weights_proj", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.weights_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 320, + "n": 64, + "k": 10240, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 320, + "n": 64, + "k": 12288, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 320, + "n": 120, + "k": 12288, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 320, + "n": 192, + "k": 10240, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 320, + "n": 256, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.compressor.wkv", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.compressor.wkv", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.compressor.wgate", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.compressor.wkv" + ] + }, + { + "type": "gemm", + "args": { + "m": 320, + "n": 256, + "k": 10240, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 320, + "n": 256, + "k": 12288, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 320, + "n": 320, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 320, + "n": 384, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.gate", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.gate", + "deepseek-ai/DeepSeek-V4-Pro/ffn.gate", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.gate", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.gate", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.gate" + ] + }, + { + "type": "gemm", + "args": { + "m": 320, + "n": 384, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w3" + ] + }, + { + "type": "gemm", + "args": { + "m": 320, + "n": 512, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wkv", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wkv", + "deepseek-ai/DeepSeek-V4-Pro/attn.wkv", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wkv", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wkv", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wkv" + ] + }, + { + "type": "gemm", + "args": { + "m": 320, + "n": 512, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.compressor.wkv+wgate", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.compressor.wkv+wgate", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.compressor.wkv+wgate" + ] + }, + { + "type": "gemm", + "args": { + "m": 320, + "n": 512, + "k": 10240, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 320, + "n": 512, + "k": 10240, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 320, + "n": 512, + "k": 12288, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 320, + "n": 512, + "k": 12288, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 320, + "n": 1024, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.compressor.wkv", + "deepseek-ai/DeepSeek-V4-Pro/attn.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro/attn.compressor.wkv", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.compressor.wgate", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.compressor.wkv" + ] + }, + { + "type": "gemm", + "args": { + "m": 320, + "n": 1024, + "k": 10240, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 320, + "n": 1024, + "k": 10240, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 320, + "n": 1024, + "k": 12288, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 320, + "n": 1536, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w3" + ] + }, + { + "type": "gemm", + "args": { + "m": 320, + "n": 1536, + "k": 10240, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 320, + "n": 1536, + "k": 12288, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 320, + "n": 1792, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 320, + "n": 2048, + "k": 12288, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 320, + "n": 4096, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 320, + "n": 5120, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 320, + "n": 6144, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3" + ] + }, + { + "type": "gemm", + "args": { + "m": 320, + "n": 7168, + "k": 192, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w2" + ] + }, + { + "type": "gemm", + "args": { + "m": 320, + "n": 7168, + "k": 768, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w2" + ] + }, + { + "type": "gemm", + "args": { + "m": 320, + "n": 7168, + "k": 896, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 320, + "n": 7168, + "k": 1024, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 320, + "n": 7168, + "k": 2048, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 320, + "n": 7168, + "k": 3072, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w2" + ] + }, + { + "type": "gemm", + "args": { + "m": 320, + "n": 7168, + "k": 4096, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 320, + "n": 7168, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.e_proj", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.h_proj", + "deepseek-ai/DeepSeek-V4-Pro/mtp.e_proj", + "deepseek-ai/DeepSeek-V4-Pro/mtp.h_proj", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.e_proj", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.h_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 320, + "n": 7168, + "k": 16384, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 320, + "n": 8192, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 320, + "n": 10240, + "k": 512, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 320, + "n": 10240, + "k": 1024, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 320, + "n": 10240, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 320, + "n": 10240, + "k": 2048, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 320, + "n": 10240, + "k": 16384, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 320, + "n": 12288, + "k": 1024, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 320, + "n": 12288, + "k": 2048, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 320, + "n": 12288, + "k": 8192, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 320, + "n": 12288, + "k": 16384, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 320, + "n": 14336, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 320, + "n": 16384, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 320, + "n": 16384, + "k": 12288, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 320, + "n": 65536, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 320, + "n": 81920, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 321, + "n": 24, + "k": 40960, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 321, + "n": 64, + "k": 10240, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 321, + "n": 192, + "k": 10240, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 321, + "n": 256, + "k": 10240, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 321, + "n": 320, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 321, + "n": 384, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w3" + ] + }, + { + "type": "gemm", + "args": { + "m": 321, + "n": 512, + "k": 10240, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 321, + "n": 512, + "k": 10240, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 321, + "n": 768, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w3" + ] + }, + { + "type": "gemm", + "args": { + "m": 321, + "n": 1024, + "k": 10240, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 321, + "n": 1024, + "k": 10240, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 321, + "n": 1536, + "k": 10240, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 321, + "n": 1792, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 321, + "n": 4096, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 321, + "n": 7168, + "k": 192, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w2" + ] + }, + { + "type": "gemm", + "args": { + "m": 321, + "n": 7168, + "k": 384, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w2" + ] + }, + { + "type": "gemm", + "args": { + "m": 321, + "n": 7168, + "k": 896, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 321, + "n": 7168, + "k": 1024, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 321, + "n": 7168, + "k": 2048, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 321, + "n": 10240, + "k": 512, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 321, + "n": 10240, + "k": 1024, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 321, + "n": 10240, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 321, + "n": 10240, + "k": 2048, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 321, + "n": 12288, + "k": 2048, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 323, + "n": 896, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 323, + "n": 7168, + "k": 448, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 328, + "n": 24, + "k": 40960, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 328, + "n": 64, + "k": 10240, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 328, + "n": 192, + "k": 10240, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 328, + "n": 256, + "k": 10240, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 328, + "n": 320, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 328, + "n": 384, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w3" + ] + }, + { + "type": "gemm", + "args": { + "m": 328, + "n": 512, + "k": 10240, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 328, + "n": 512, + "k": 10240, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 328, + "n": 768, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w3" + ] + }, + { + "type": "gemm", + "args": { + "m": 328, + "n": 896, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 328, + "n": 1024, + "k": 10240, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 328, + "n": 1536, + "k": 10240, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 328, + "n": 1792, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 328, + "n": 4096, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 328, + "n": 7168, + "k": 192, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w2" + ] + }, + { + "type": "gemm", + "args": { + "m": 328, + "n": 7168, + "k": 384, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w2" + ] + }, + { + "type": "gemm", + "args": { + "m": 328, + "n": 7168, + "k": 448, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 328, + "n": 7168, + "k": 896, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 328, + "n": 7168, + "k": 1024, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 328, + "n": 7168, + "k": 2048, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 328, + "n": 10240, + "k": 1024, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 328, + "n": 10240, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 328, + "n": 10240, + "k": 2048, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 328, + "n": 12288, + "k": 2048, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 334, + "n": 24, + "k": 49152, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 334, + "n": 64, + "k": 12288, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 334, + "n": 120, + "k": 12288, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 334, + "n": 256, + "k": 12288, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 334, + "n": 512, + "k": 12288, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 334, + "n": 512, + "k": 12288, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 334, + "n": 1024, + "k": 12288, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 334, + "n": 1536, + "k": 12288, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 334, + "n": 5120, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 334, + "n": 12288, + "k": 1024, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 336, + "n": 24, + "k": 40960, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 336, + "n": 24, + "k": 49152, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 336, + "n": 64, + "k": 10240, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 336, + "n": 64, + "k": 12288, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 336, + "n": 120, + "k": 12288, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 336, + "n": 192, + "k": 10240, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 336, + "n": 256, + "k": 10240, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 336, + "n": 256, + "k": 12288, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 336, + "n": 320, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 336, + "n": 384, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w3" + ] + }, + { + "type": "gemm", + "args": { + "m": 336, + "n": 512, + "k": 10240, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 336, + "n": 512, + "k": 10240, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 336, + "n": 512, + "k": 12288, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 336, + "n": 512, + "k": 12288, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 336, + "n": 768, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w3" + ] + }, + { + "type": "gemm", + "args": { + "m": 336, + "n": 896, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 336, + "n": 1024, + "k": 10240, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 336, + "n": 1024, + "k": 12288, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 336, + "n": 1536, + "k": 10240, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 336, + "n": 1536, + "k": 12288, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 336, + "n": 4096, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 336, + "n": 5120, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 336, + "n": 7168, + "k": 192, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w2" + ] + }, + { + "type": "gemm", + "args": { + "m": 336, + "n": 7168, + "k": 384, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w2" + ] + }, + { + "type": "gemm", + "args": { + "m": 336, + "n": 7168, + "k": 448, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 336, + "n": 7168, + "k": 1024, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 336, + "n": 7168, + "k": 2048, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 336, + "n": 10240, + "k": 1024, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 336, + "n": 12288, + "k": 1024, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 344, + "n": 24, + "k": 49152, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 344, + "n": 64, + "k": 12288, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 344, + "n": 120, + "k": 12288, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 344, + "n": 256, + "k": 12288, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 344, + "n": 512, + "k": 12288, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 344, + "n": 512, + "k": 12288, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 344, + "n": 1024, + "k": 12288, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 344, + "n": 1536, + "k": 12288, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 344, + "n": 5120, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 344, + "n": 12288, + "k": 1024, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 352, + "n": 24, + "k": 28672, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/hc_attn", + "deepseek-ai/DeepSeek-V4-Pro-0813/hc_ffn", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.hc_attn", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.hc_ffn", + "deepseek-ai/DeepSeek-V4-Pro/hc_attn", + "deepseek-ai/DeepSeek-V4-Pro/hc_ffn", + "deepseek-ai/DeepSeek-V4-Pro/mtp.hc_attn", + "deepseek-ai/DeepSeek-V4-Pro/mtp.hc_ffn", + "sgl-project/DeepSeek-V4-Pro-FP8/hc_attn", + "sgl-project/DeepSeek-V4-Pro-FP8/hc_ffn", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.hc_attn", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.hc_ffn" + ] + }, + { + "type": "gemm", + "args": { + "m": 352, + "n": 64, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.weights_proj", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.weights_proj", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.weights_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 352, + "n": 256, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.compressor.wkv", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.compressor.wkv", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.compressor.wgate", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.compressor.wkv" + ] + }, + { + "type": "gemm", + "args": { + "m": 352, + "n": 384, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.gate", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.gate", + "deepseek-ai/DeepSeek-V4-Pro/ffn.gate", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.gate", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.gate", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.gate" + ] + }, + { + "type": "gemm", + "args": { + "m": 352, + "n": 512, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wkv", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wkv", + "deepseek-ai/DeepSeek-V4-Pro/attn.wkv", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wkv", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wkv", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wkv" + ] + }, + { + "type": "gemm", + "args": { + "m": 352, + "n": 512, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.compressor.wkv+wgate", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.compressor.wkv+wgate", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.compressor.wkv+wgate" + ] + }, + { + "type": "gemm", + "args": { + "m": 352, + "n": 1024, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.compressor.wkv", + "deepseek-ai/DeepSeek-V4-Pro/attn.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro/attn.compressor.wkv", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.compressor.wgate", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.compressor.wkv" + ] + }, + { + "type": "gemm", + "args": { + "m": 352, + "n": 1536, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w3" + ] + }, + { + "type": "gemm", + "args": { + "m": 352, + "n": 7168, + "k": 768, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w2" + ] + }, + { + "type": "gemm", + "args": { + "m": 352, + "n": 7168, + "k": 4096, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 352, + "n": 8192, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 352, + "n": 16384, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 362, + "n": 24, + "k": 28672, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/hc_attn", + "deepseek-ai/DeepSeek-V4-Pro-0813/hc_ffn", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.hc_attn", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.hc_ffn", + "deepseek-ai/DeepSeek-V4-Pro/hc_attn", + "deepseek-ai/DeepSeek-V4-Pro/hc_ffn", + "deepseek-ai/DeepSeek-V4-Pro/mtp.hc_attn", + "deepseek-ai/DeepSeek-V4-Pro/mtp.hc_ffn", + "sgl-project/DeepSeek-V4-Pro-FP8/hc_attn", + "sgl-project/DeepSeek-V4-Pro-FP8/hc_ffn", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.hc_attn", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.hc_ffn" + ] + }, + { + "type": "gemm", + "args": { + "m": 362, + "n": 64, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.weights_proj", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.weights_proj", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.weights_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 362, + "n": 256, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.compressor.wkv", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.compressor.wkv", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.compressor.wgate", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.compressor.wkv" + ] + }, + { + "type": "gemm", + "args": { + "m": 362, + "n": 384, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.gate", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.gate", + "deepseek-ai/DeepSeek-V4-Pro/ffn.gate", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.gate", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.gate", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.gate" + ] + }, + { + "type": "gemm", + "args": { + "m": 362, + "n": 512, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wkv", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wkv", + "deepseek-ai/DeepSeek-V4-Pro/attn.wkv", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wkv", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wkv", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wkv" + ] + }, + { + "type": "gemm", + "args": { + "m": 362, + "n": 512, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.compressor.wkv+wgate", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.compressor.wkv+wgate", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.compressor.wkv+wgate" + ] + }, + { + "type": "gemm", + "args": { + "m": 362, + "n": 1024, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.compressor.wkv", + "deepseek-ai/DeepSeek-V4-Pro/attn.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro/attn.compressor.wkv", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.compressor.wgate", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.compressor.wkv" + ] + }, + { + "type": "gemm", + "args": { + "m": 362, + "n": 1536, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w3" + ] + }, + { + "type": "gemm", + "args": { + "m": 362, + "n": 8192, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 364, + "n": 1536, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w3" + ] + }, + { + "type": "gemm", + "args": { + "m": 364, + "n": 7168, + "k": 768, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w2" + ] + }, + { + "type": "gemm", + "args": { + "m": 364, + "n": 7168, + "k": 4096, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 364, + "n": 16384, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 368, + "n": 24, + "k": 28672, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/hc_attn", + "deepseek-ai/DeepSeek-V4-Pro-0813/hc_ffn", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.hc_attn", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.hc_ffn", + "deepseek-ai/DeepSeek-V4-Pro/hc_attn", + "deepseek-ai/DeepSeek-V4-Pro/hc_ffn", + "deepseek-ai/DeepSeek-V4-Pro/mtp.hc_attn", + "deepseek-ai/DeepSeek-V4-Pro/mtp.hc_ffn", + "sgl-project/DeepSeek-V4-Pro-FP8/hc_attn", + "sgl-project/DeepSeek-V4-Pro-FP8/hc_ffn", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.hc_attn", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.hc_ffn" + ] + }, + { + "type": "gemm", + "args": { + "m": 368, + "n": 64, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.weights_proj", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.weights_proj", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.weights_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 368, + "n": 256, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.compressor.wkv", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.compressor.wkv", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.compressor.wgate", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.compressor.wkv" + ] + }, + { + "type": "gemm", + "args": { + "m": 368, + "n": 384, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.gate", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.gate", + "deepseek-ai/DeepSeek-V4-Pro/ffn.gate", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.gate", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.gate", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.gate" + ] + }, + { + "type": "gemm", + "args": { + "m": 368, + "n": 512, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wkv", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wkv", + "deepseek-ai/DeepSeek-V4-Pro/attn.wkv", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wkv", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wkv", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wkv" + ] + }, + { + "type": "gemm", + "args": { + "m": 368, + "n": 512, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.compressor.wkv+wgate", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.compressor.wkv+wgate", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.compressor.wkv+wgate" + ] + }, + { + "type": "gemm", + "args": { + "m": 368, + "n": 1024, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.compressor.wkv", + "deepseek-ai/DeepSeek-V4-Pro/attn.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro/attn.compressor.wkv", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.compressor.wgate", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.compressor.wkv" + ] + }, + { + "type": "gemm", + "args": { + "m": 368, + "n": 1536, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w3" + ] + }, + { + "type": "gemm", + "args": { + "m": 368, + "n": 7168, + "k": 768, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w2" + ] + }, + { + "type": "gemm", + "args": { + "m": 368, + "n": 7168, + "k": 4096, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 368, + "n": 8192, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 368, + "n": 16384, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 384, + "n": 24, + "k": 28672, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/hc_attn", + "deepseek-ai/DeepSeek-V4-Pro-0813/hc_ffn", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.hc_attn", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.hc_ffn", + "deepseek-ai/DeepSeek-V4-Pro/hc_attn", + "deepseek-ai/DeepSeek-V4-Pro/hc_ffn", + "deepseek-ai/DeepSeek-V4-Pro/mtp.hc_attn", + "deepseek-ai/DeepSeek-V4-Pro/mtp.hc_ffn", + "sgl-project/DeepSeek-V4-Pro-FP8/hc_attn", + "sgl-project/DeepSeek-V4-Pro-FP8/hc_ffn", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.hc_attn", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.hc_ffn" + ] + }, + { + "type": "gemm", + "args": { + "m": 384, + "n": 24, + "k": 40960, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 384, + "n": 24, + "k": 49152, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 384, + "n": 64, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.weights_proj", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.weights_proj", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.weights_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 384, + "n": 64, + "k": 10240, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 384, + "n": 64, + "k": 12288, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 384, + "n": 120, + "k": 12288, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 384, + "n": 192, + "k": 10240, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 384, + "n": 256, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.compressor.wkv", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.compressor.wkv", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.compressor.wgate", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.compressor.wkv" + ] + }, + { + "type": "gemm", + "args": { + "m": 384, + "n": 256, + "k": 10240, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 384, + "n": 256, + "k": 12288, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 384, + "n": 320, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 384, + "n": 384, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.gate", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.gate", + "deepseek-ai/DeepSeek-V4-Pro/ffn.gate", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.gate", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.gate", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.gate" + ] + }, + { + "type": "gemm", + "args": { + "m": 384, + "n": 384, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w3" + ] + }, + { + "type": "gemm", + "args": { + "m": 384, + "n": 512, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wkv", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wkv", + "deepseek-ai/DeepSeek-V4-Pro/attn.wkv", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wkv", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wkv", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wkv" + ] + }, + { + "type": "gemm", + "args": { + "m": 384, + "n": 512, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.compressor.wkv+wgate", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.compressor.wkv+wgate", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.compressor.wkv+wgate" + ] + }, + { + "type": "gemm", + "args": { + "m": 384, + "n": 512, + "k": 10240, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 384, + "n": 512, + "k": 10240, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 384, + "n": 512, + "k": 12288, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 384, + "n": 512, + "k": 12288, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 384, + "n": 1024, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.compressor.wkv", + "deepseek-ai/DeepSeek-V4-Pro/attn.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro/attn.compressor.wkv", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.compressor.wgate", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.compressor.wkv" + ] + }, + { + "type": "gemm", + "args": { + "m": 384, + "n": 1024, + "k": 10240, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 384, + "n": 1024, + "k": 10240, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 384, + "n": 1024, + "k": 12288, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 384, + "n": 1536, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w3" + ] + }, + { + "type": "gemm", + "args": { + "m": 384, + "n": 1536, + "k": 10240, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 384, + "n": 1536, + "k": 12288, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 384, + "n": 1792, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 384, + "n": 2048, + "k": 12288, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 384, + "n": 4096, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 384, + "n": 5120, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 384, + "n": 6144, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3" + ] + }, + { + "type": "gemm", + "args": { + "m": 384, + "n": 7168, + "k": 192, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w2" + ] + }, + { + "type": "gemm", + "args": { + "m": 384, + "n": 7168, + "k": 768, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w2" + ] + }, + { + "type": "gemm", + "args": { + "m": 384, + "n": 7168, + "k": 896, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 384, + "n": 7168, + "k": 1024, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 384, + "n": 7168, + "k": 2048, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 384, + "n": 7168, + "k": 3072, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w2" + ] + }, + { + "type": "gemm", + "args": { + "m": 384, + "n": 7168, + "k": 4096, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 384, + "n": 7168, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.e_proj", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.h_proj", + "deepseek-ai/DeepSeek-V4-Pro/mtp.e_proj", + "deepseek-ai/DeepSeek-V4-Pro/mtp.h_proj", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.e_proj", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.h_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 384, + "n": 7168, + "k": 16384, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 384, + "n": 8192, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 384, + "n": 10240, + "k": 512, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 384, + "n": 10240, + "k": 1024, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 384, + "n": 10240, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 384, + "n": 10240, + "k": 2048, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 384, + "n": 10240, + "k": 16384, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 384, + "n": 12288, + "k": 1024, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 384, + "n": 12288, + "k": 2048, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 384, + "n": 12288, + "k": 8192, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 384, + "n": 12288, + "k": 16384, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 384, + "n": 14336, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 384, + "n": 16384, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 384, + "n": 16384, + "k": 12288, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 384, + "n": 65536, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 384, + "n": 81920, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 392, + "n": 24, + "k": 28672, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/hc_attn", + "deepseek-ai/DeepSeek-V4-Pro-0813/hc_ffn", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.hc_attn", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.hc_ffn", + "deepseek-ai/DeepSeek-V4-Pro/hc_attn", + "deepseek-ai/DeepSeek-V4-Pro/hc_ffn", + "deepseek-ai/DeepSeek-V4-Pro/mtp.hc_attn", + "deepseek-ai/DeepSeek-V4-Pro/mtp.hc_ffn", + "sgl-project/DeepSeek-V4-Pro-FP8/hc_attn", + "sgl-project/DeepSeek-V4-Pro-FP8/hc_ffn", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.hc_attn", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.hc_ffn" + ] + }, + { + "type": "gemm", + "args": { + "m": 392, + "n": 64, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.weights_proj", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.weights_proj", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.weights_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 392, + "n": 256, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.compressor.wkv", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.compressor.wkv", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.compressor.wgate", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.compressor.wkv" + ] + }, + { + "type": "gemm", + "args": { + "m": 392, + "n": 384, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.gate", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.gate", + "deepseek-ai/DeepSeek-V4-Pro/ffn.gate", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.gate", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.gate", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.gate" + ] + }, + { + "type": "gemm", + "args": { + "m": 392, + "n": 512, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wkv", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wkv", + "deepseek-ai/DeepSeek-V4-Pro/attn.wkv", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wkv", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wkv", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wkv" + ] + }, + { + "type": "gemm", + "args": { + "m": 392, + "n": 512, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.compressor.wkv+wgate", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.compressor.wkv+wgate", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.compressor.wkv+wgate" + ] + }, + { + "type": "gemm", + "args": { + "m": 392, + "n": 1024, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.compressor.wkv", + "deepseek-ai/DeepSeek-V4-Pro/attn.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro/attn.compressor.wkv", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.compressor.wgate", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.compressor.wkv" + ] + }, + { + "type": "gemm", + "args": { + "m": 392, + "n": 1536, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w3" + ] + }, + { + "type": "gemm", + "args": { + "m": 392, + "n": 7168, + "k": 768, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w2" + ] + }, + { + "type": "gemm", + "args": { + "m": 392, + "n": 7168, + "k": 4096, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 392, + "n": 8192, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 392, + "n": 16384, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 400, + "n": 10240, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 400, + "n": 12288, + "k": 2048, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 402, + "n": 24, + "k": 40960, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 402, + "n": 64, + "k": 10240, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 402, + "n": 192, + "k": 10240, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 402, + "n": 256, + "k": 10240, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 402, + "n": 320, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 402, + "n": 384, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w3" + ] + }, + { + "type": "gemm", + "args": { + "m": 402, + "n": 512, + "k": 10240, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 402, + "n": 512, + "k": 10240, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 402, + "n": 1024, + "k": 10240, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 402, + "n": 1536, + "k": 10240, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 402, + "n": 4096, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 402, + "n": 7168, + "k": 192, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w2" + ] + }, + { + "type": "gemm", + "args": { + "m": 402, + "n": 7168, + "k": 1024, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 402, + "n": 10240, + "k": 1024, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 404, + "n": 768, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w3" + ] + }, + { + "type": "gemm", + "args": { + "m": 404, + "n": 896, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 404, + "n": 7168, + "k": 384, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w2" + ] + }, + { + "type": "gemm", + "args": { + "m": 404, + "n": 7168, + "k": 448, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 404, + "n": 7168, + "k": 2048, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 406, + "n": 1792, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 406, + "n": 7168, + "k": 896, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 407, + "n": 10240, + "k": 2048, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 408, + "n": 24, + "k": 40960, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 408, + "n": 64, + "k": 10240, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 408, + "n": 192, + "k": 10240, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 408, + "n": 256, + "k": 10240, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 408, + "n": 320, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 408, + "n": 512, + "k": 10240, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 408, + "n": 512, + "k": 10240, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 408, + "n": 768, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w3" + ] + }, + { + "type": "gemm", + "args": { + "m": 408, + "n": 896, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 408, + "n": 1024, + "k": 10240, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 408, + "n": 1536, + "k": 10240, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 408, + "n": 4096, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 408, + "n": 7168, + "k": 384, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w2" + ] + }, + { + "type": "gemm", + "args": { + "m": 408, + "n": 7168, + "k": 448, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 408, + "n": 7168, + "k": 1024, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 408, + "n": 7168, + "k": 2048, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 408, + "n": 10240, + "k": 1024, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 416, + "n": 24, + "k": 28672, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/hc_attn", + "deepseek-ai/DeepSeek-V4-Pro-0813/hc_ffn", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.hc_attn", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.hc_ffn", + "deepseek-ai/DeepSeek-V4-Pro/hc_attn", + "deepseek-ai/DeepSeek-V4-Pro/hc_ffn", + "deepseek-ai/DeepSeek-V4-Pro/mtp.hc_attn", + "deepseek-ai/DeepSeek-V4-Pro/mtp.hc_ffn", + "sgl-project/DeepSeek-V4-Pro-FP8/hc_attn", + "sgl-project/DeepSeek-V4-Pro-FP8/hc_ffn", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.hc_attn", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.hc_ffn" + ] + }, + { + "type": "gemm", + "args": { + "m": 416, + "n": 64, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.weights_proj", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.weights_proj", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.weights_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 416, + "n": 256, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.compressor.wkv", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.compressor.wkv", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.compressor.wgate", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.compressor.wkv" + ] + }, + { + "type": "gemm", + "args": { + "m": 416, + "n": 384, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.gate", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.gate", + "deepseek-ai/DeepSeek-V4-Pro/ffn.gate", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.gate", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.gate", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.gate" + ] + }, + { + "type": "gemm", + "args": { + "m": 416, + "n": 512, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wkv", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wkv", + "deepseek-ai/DeepSeek-V4-Pro/attn.wkv", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wkv", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wkv", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wkv" + ] + }, + { + "type": "gemm", + "args": { + "m": 416, + "n": 512, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.compressor.wkv+wgate", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.compressor.wkv+wgate", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.compressor.wkv+wgate" + ] + }, + { + "type": "gemm", + "args": { + "m": 416, + "n": 1024, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.compressor.wkv", + "deepseek-ai/DeepSeek-V4-Pro/attn.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro/attn.compressor.wkv", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.compressor.wgate", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.compressor.wkv" + ] + }, + { + "type": "gemm", + "args": { + "m": 416, + "n": 1536, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w3" + ] + }, + { + "type": "gemm", + "args": { + "m": 416, + "n": 4096, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 416, + "n": 7168, + "k": 768, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w2" + ] + }, + { + "type": "gemm", + "args": { + "m": 416, + "n": 7168, + "k": 1024, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 416, + "n": 7168, + "k": 4096, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 416, + "n": 8192, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 416, + "n": 16384, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 448, + "n": 24, + "k": 28672, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/hc_attn", + "deepseek-ai/DeepSeek-V4-Pro-0813/hc_ffn", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.hc_attn", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.hc_ffn", + "deepseek-ai/DeepSeek-V4-Pro/hc_attn", + "deepseek-ai/DeepSeek-V4-Pro/hc_ffn", + "deepseek-ai/DeepSeek-V4-Pro/mtp.hc_attn", + "deepseek-ai/DeepSeek-V4-Pro/mtp.hc_ffn", + "sgl-project/DeepSeek-V4-Pro-FP8/hc_attn", + "sgl-project/DeepSeek-V4-Pro-FP8/hc_ffn", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.hc_attn", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.hc_ffn" + ] + }, + { + "type": "gemm", + "args": { + "m": 448, + "n": 24, + "k": 40960, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 448, + "n": 24, + "k": 49152, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 448, + "n": 64, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.weights_proj", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.weights_proj", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.weights_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 448, + "n": 64, + "k": 10240, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 448, + "n": 64, + "k": 12288, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 448, + "n": 120, + "k": 12288, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 448, + "n": 192, + "k": 10240, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 448, + "n": 256, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.compressor.wkv", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.compressor.wkv", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.compressor.wgate", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.compressor.wkv" + ] + }, + { + "type": "gemm", + "args": { + "m": 448, + "n": 256, + "k": 10240, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 448, + "n": 256, + "k": 12288, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 448, + "n": 320, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 448, + "n": 384, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.gate", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.gate", + "deepseek-ai/DeepSeek-V4-Pro/ffn.gate", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.gate", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.gate", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.gate" + ] + }, + { + "type": "gemm", + "args": { + "m": 448, + "n": 384, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w3" + ] + }, + { + "type": "gemm", + "args": { + "m": 448, + "n": 512, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wkv", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wkv", + "deepseek-ai/DeepSeek-V4-Pro/attn.wkv", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wkv", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wkv", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wkv" + ] + }, + { + "type": "gemm", + "args": { + "m": 448, + "n": 512, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.compressor.wkv+wgate", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.compressor.wkv+wgate", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.compressor.wkv+wgate" + ] + }, + { + "type": "gemm", + "args": { + "m": 448, + "n": 512, + "k": 10240, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 448, + "n": 512, + "k": 10240, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 448, + "n": 512, + "k": 12288, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 448, + "n": 512, + "k": 12288, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 448, + "n": 768, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w3" + ] + }, + { + "type": "gemm", + "args": { + "m": 448, + "n": 1024, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.compressor.wkv", + "deepseek-ai/DeepSeek-V4-Pro/attn.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro/attn.compressor.wkv", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.compressor.wgate", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.compressor.wkv" + ] + }, + { + "type": "gemm", + "args": { + "m": 448, + "n": 1024, + "k": 10240, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 448, + "n": 1024, + "k": 10240, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 448, + "n": 1024, + "k": 12288, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 448, + "n": 1536, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w3" + ] + }, + { + "type": "gemm", + "args": { + "m": 448, + "n": 1536, + "k": 10240, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 448, + "n": 1536, + "k": 12288, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 448, + "n": 1792, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 448, + "n": 2048, + "k": 12288, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 448, + "n": 4096, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 448, + "n": 5120, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 448, + "n": 6144, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3" + ] + }, + { + "type": "gemm", + "args": { + "m": 448, + "n": 7168, + "k": 192, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w2" + ] + }, + { + "type": "gemm", + "args": { + "m": 448, + "n": 7168, + "k": 384, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w2" + ] + }, + { + "type": "gemm", + "args": { + "m": 448, + "n": 7168, + "k": 768, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w2" + ] + }, + { + "type": "gemm", + "args": { + "m": 448, + "n": 7168, + "k": 896, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 448, + "n": 7168, + "k": 1024, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 448, + "n": 7168, + "k": 2048, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 448, + "n": 7168, + "k": 3072, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w2" + ] + }, + { + "type": "gemm", + "args": { + "m": 448, + "n": 7168, + "k": 4096, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 448, + "n": 7168, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.e_proj", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.h_proj", + "deepseek-ai/DeepSeek-V4-Pro/mtp.e_proj", + "deepseek-ai/DeepSeek-V4-Pro/mtp.h_proj", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.e_proj", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.h_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 448, + "n": 7168, + "k": 16384, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 448, + "n": 8192, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 448, + "n": 8192, + "k": 10240, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 448, + "n": 10240, + "k": 512, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 448, + "n": 10240, + "k": 1024, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 448, + "n": 10240, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 448, + "n": 10240, + "k": 2048, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 448, + "n": 10240, + "k": 4096, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 448, + "n": 10240, + "k": 16384, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 448, + "n": 12288, + "k": 1024, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 448, + "n": 12288, + "k": 2048, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 448, + "n": 12288, + "k": 8192, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 448, + "n": 12288, + "k": 16384, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 448, + "n": 14336, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 448, + "n": 16384, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 448, + "n": 16384, + "k": 12288, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 448, + "n": 65536, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 448, + "n": 81920, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 449, + "n": 24, + "k": 49152, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 449, + "n": 64, + "k": 12288, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 449, + "n": 120, + "k": 12288, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 449, + "n": 256, + "k": 12288, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 449, + "n": 512, + "k": 12288, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 449, + "n": 512, + "k": 12288, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 449, + "n": 1024, + "k": 10240, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 449, + "n": 1024, + "k": 12288, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 449, + "n": 1536, + "k": 12288, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 449, + "n": 5120, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 449, + "n": 10240, + "k": 512, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 449, + "n": 10240, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 449, + "n": 12288, + "k": 1024, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 449, + "n": 12288, + "k": 2048, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 453, + "n": 24, + "k": 28672, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/hc_attn", + "deepseek-ai/DeepSeek-V4-Pro-0813/hc_ffn", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.hc_attn", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.hc_ffn", + "deepseek-ai/DeepSeek-V4-Pro/hc_attn", + "deepseek-ai/DeepSeek-V4-Pro/hc_ffn", + "deepseek-ai/DeepSeek-V4-Pro/mtp.hc_attn", + "deepseek-ai/DeepSeek-V4-Pro/mtp.hc_ffn", + "sgl-project/DeepSeek-V4-Pro-FP8/hc_attn", + "sgl-project/DeepSeek-V4-Pro-FP8/hc_ffn", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.hc_attn", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.hc_ffn" + ] + }, + { + "type": "gemm", + "args": { + "m": 453, + "n": 64, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.weights_proj", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.weights_proj", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.weights_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 453, + "n": 256, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.compressor.wkv", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.compressor.wkv", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.compressor.wgate", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.compressor.wkv" + ] + }, + { + "type": "gemm", + "args": { + "m": 453, + "n": 384, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.gate", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.gate", + "deepseek-ai/DeepSeek-V4-Pro/ffn.gate", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.gate", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.gate", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.gate" + ] + }, + { + "type": "gemm", + "args": { + "m": 453, + "n": 512, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wkv", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wkv", + "deepseek-ai/DeepSeek-V4-Pro/attn.wkv", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wkv", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wkv", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wkv" + ] + }, + { + "type": "gemm", + "args": { + "m": 453, + "n": 512, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.compressor.wkv+wgate", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.compressor.wkv+wgate", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.compressor.wkv+wgate" + ] + }, + { + "type": "gemm", + "args": { + "m": 453, + "n": 1024, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.compressor.wkv", + "deepseek-ai/DeepSeek-V4-Pro/attn.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro/attn.compressor.wkv", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.compressor.wgate", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.compressor.wkv" + ] + }, + { + "type": "gemm", + "args": { + "m": 453, + "n": 1536, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w3" + ] + }, + { + "type": "gemm", + "args": { + "m": 453, + "n": 8192, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 456, + "n": 24, + "k": 28672, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/hc_attn", + "deepseek-ai/DeepSeek-V4-Pro-0813/hc_ffn", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.hc_attn", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.hc_ffn", + "deepseek-ai/DeepSeek-V4-Pro/hc_attn", + "deepseek-ai/DeepSeek-V4-Pro/hc_ffn", + "deepseek-ai/DeepSeek-V4-Pro/mtp.hc_attn", + "deepseek-ai/DeepSeek-V4-Pro/mtp.hc_ffn", + "sgl-project/DeepSeek-V4-Pro-FP8/hc_attn", + "sgl-project/DeepSeek-V4-Pro-FP8/hc_ffn", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.hc_attn", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.hc_ffn" + ] + }, + { + "type": "gemm", + "args": { + "m": 456, + "n": 24, + "k": 49152, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 456, + "n": 64, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.weights_proj", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.weights_proj", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.weights_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 456, + "n": 64, + "k": 12288, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 456, + "n": 120, + "k": 12288, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 456, + "n": 256, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.compressor.wkv", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.compressor.wkv", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.compressor.wgate", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.compressor.wkv" + ] + }, + { + "type": "gemm", + "args": { + "m": 456, + "n": 256, + "k": 12288, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 456, + "n": 384, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.gate", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.gate", + "deepseek-ai/DeepSeek-V4-Pro/ffn.gate", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.gate", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.gate", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.gate" + ] + }, + { + "type": "gemm", + "args": { + "m": 456, + "n": 384, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w3" + ] + }, + { + "type": "gemm", + "args": { + "m": 456, + "n": 512, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wkv", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wkv", + "deepseek-ai/DeepSeek-V4-Pro/attn.wkv", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wkv", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wkv", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wkv" + ] + }, + { + "type": "gemm", + "args": { + "m": 456, + "n": 512, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.compressor.wkv+wgate", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.compressor.wkv+wgate", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.compressor.wkv+wgate" + ] + }, + { + "type": "gemm", + "args": { + "m": 456, + "n": 512, + "k": 12288, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 456, + "n": 512, + "k": 12288, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 456, + "n": 1024, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.compressor.wkv", + "deepseek-ai/DeepSeek-V4-Pro/attn.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro/attn.compressor.wkv", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.compressor.wgate", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.compressor.wkv" + ] + }, + { + "type": "gemm", + "args": { + "m": 456, + "n": 1024, + "k": 12288, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 456, + "n": 1536, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w3" + ] + }, + { + "type": "gemm", + "args": { + "m": 456, + "n": 1536, + "k": 12288, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 456, + "n": 1792, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 456, + "n": 5120, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 456, + "n": 7168, + "k": 192, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w2" + ] + }, + { + "type": "gemm", + "args": { + "m": 456, + "n": 7168, + "k": 768, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w2" + ] + }, + { + "type": "gemm", + "args": { + "m": 456, + "n": 7168, + "k": 896, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 456, + "n": 7168, + "k": 4096, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 456, + "n": 8192, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 456, + "n": 10240, + "k": 2048, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 456, + "n": 12288, + "k": 1024, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 456, + "n": 16384, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 457, + "n": 1536, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w3" + ] + }, + { + "type": "gemm", + "args": { + "m": 457, + "n": 7168, + "k": 768, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w2" + ] + }, + { + "type": "gemm", + "args": { + "m": 457, + "n": 7168, + "k": 4096, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 457, + "n": 16384, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 464, + "n": 24, + "k": 28672, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/hc_attn", + "deepseek-ai/DeepSeek-V4-Pro-0813/hc_ffn", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.hc_attn", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.hc_ffn", + "deepseek-ai/DeepSeek-V4-Pro/hc_attn", + "deepseek-ai/DeepSeek-V4-Pro/hc_ffn", + "deepseek-ai/DeepSeek-V4-Pro/mtp.hc_attn", + "deepseek-ai/DeepSeek-V4-Pro/mtp.hc_ffn", + "sgl-project/DeepSeek-V4-Pro-FP8/hc_attn", + "sgl-project/DeepSeek-V4-Pro-FP8/hc_ffn", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.hc_attn", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.hc_ffn" + ] + }, + { + "type": "gemm", + "args": { + "m": 464, + "n": 24, + "k": 40960, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 464, + "n": 24, + "k": 49152, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 464, + "n": 64, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.weights_proj", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.weights_proj", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.weights_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 464, + "n": 64, + "k": 10240, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 464, + "n": 64, + "k": 12288, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 464, + "n": 120, + "k": 12288, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 464, + "n": 192, + "k": 10240, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 464, + "n": 256, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.compressor.wkv", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.compressor.wkv", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.compressor.wgate", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.compressor.wkv" + ] + }, + { + "type": "gemm", + "args": { + "m": 464, + "n": 256, + "k": 10240, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 464, + "n": 256, + "k": 12288, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 464, + "n": 320, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 464, + "n": 384, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.gate", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.gate", + "deepseek-ai/DeepSeek-V4-Pro/ffn.gate", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.gate", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.gate", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.gate" + ] + }, + { + "type": "gemm", + "args": { + "m": 464, + "n": 512, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wkv", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wkv", + "deepseek-ai/DeepSeek-V4-Pro/attn.wkv", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wkv", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wkv", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wkv" + ] + }, + { + "type": "gemm", + "args": { + "m": 464, + "n": 512, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.compressor.wkv+wgate", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.compressor.wkv+wgate", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.compressor.wkv+wgate" + ] + }, + { + "type": "gemm", + "args": { + "m": 464, + "n": 512, + "k": 10240, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 464, + "n": 512, + "k": 10240, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 464, + "n": 512, + "k": 12288, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 464, + "n": 512, + "k": 12288, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 464, + "n": 768, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w3" + ] + }, + { + "type": "gemm", + "args": { + "m": 464, + "n": 896, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 464, + "n": 1024, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.compressor.wkv", + "deepseek-ai/DeepSeek-V4-Pro/attn.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro/attn.compressor.wkv", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.compressor.wgate", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.compressor.wkv" + ] + }, + { + "type": "gemm", + "args": { + "m": 464, + "n": 1024, + "k": 10240, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 464, + "n": 1024, + "k": 12288, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 464, + "n": 1536, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w3" + ] + }, + { + "type": "gemm", + "args": { + "m": 464, + "n": 1536, + "k": 10240, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 464, + "n": 1536, + "k": 12288, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 464, + "n": 1792, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 464, + "n": 4096, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 464, + "n": 5120, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 464, + "n": 7168, + "k": 384, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w2" + ] + }, + { + "type": "gemm", + "args": { + "m": 464, + "n": 7168, + "k": 448, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 464, + "n": 7168, + "k": 768, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w2" + ] + }, + { + "type": "gemm", + "args": { + "m": 464, + "n": 7168, + "k": 896, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 464, + "n": 7168, + "k": 1024, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 464, + "n": 7168, + "k": 2048, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 464, + "n": 7168, + "k": 4096, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 464, + "n": 8192, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 464, + "n": 10240, + "k": 1024, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 464, + "n": 12288, + "k": 1024, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 464, + "n": 16384, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 480, + "n": 24, + "k": 28672, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/hc_attn", + "deepseek-ai/DeepSeek-V4-Pro-0813/hc_ffn", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.hc_attn", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.hc_ffn", + "deepseek-ai/DeepSeek-V4-Pro/hc_attn", + "deepseek-ai/DeepSeek-V4-Pro/hc_ffn", + "deepseek-ai/DeepSeek-V4-Pro/mtp.hc_attn", + "deepseek-ai/DeepSeek-V4-Pro/mtp.hc_ffn", + "sgl-project/DeepSeek-V4-Pro-FP8/hc_attn", + "sgl-project/DeepSeek-V4-Pro-FP8/hc_ffn", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.hc_attn", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.hc_ffn" + ] + }, + { + "type": "gemm", + "args": { + "m": 480, + "n": 24, + "k": 40960, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 480, + "n": 64, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.weights_proj", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.weights_proj", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.weights_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 480, + "n": 64, + "k": 10240, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 480, + "n": 192, + "k": 10240, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 480, + "n": 256, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.compressor.wkv", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.compressor.wkv", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.compressor.wgate", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.compressor.wkv" + ] + }, + { + "type": "gemm", + "args": { + "m": 480, + "n": 256, + "k": 10240, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 480, + "n": 384, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.gate", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.gate", + "deepseek-ai/DeepSeek-V4-Pro/ffn.gate", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.gate", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.gate", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.gate" + ] + }, + { + "type": "gemm", + "args": { + "m": 480, + "n": 512, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wkv", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wkv", + "deepseek-ai/DeepSeek-V4-Pro/attn.wkv", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wkv", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wkv", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wkv" + ] + }, + { + "type": "gemm", + "args": { + "m": 480, + "n": 512, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.compressor.wkv+wgate", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.compressor.wkv+wgate", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.compressor.wkv+wgate" + ] + }, + { + "type": "gemm", + "args": { + "m": 480, + "n": 512, + "k": 10240, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 480, + "n": 512, + "k": 10240, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 480, + "n": 1024, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.compressor.wkv", + "deepseek-ai/DeepSeek-V4-Pro/attn.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro/attn.compressor.wkv", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.compressor.wgate", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.compressor.wkv" + ] + }, + { + "type": "gemm", + "args": { + "m": 480, + "n": 1024, + "k": 10240, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 480, + "n": 1536, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w3" + ] + }, + { + "type": "gemm", + "args": { + "m": 480, + "n": 1536, + "k": 10240, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 480, + "n": 7168, + "k": 768, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w2" + ] + }, + { + "type": "gemm", + "args": { + "m": 480, + "n": 7168, + "k": 4096, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 480, + "n": 8192, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 480, + "n": 10240, + "k": 1024, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 480, + "n": 16384, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 504, + "n": 320, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 504, + "n": 4096, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 504, + "n": 7168, + "k": 1024, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 508, + "n": 24, + "k": 40960, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 508, + "n": 64, + "k": 10240, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 508, + "n": 192, + "k": 10240, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 508, + "n": 256, + "k": 10240, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 508, + "n": 512, + "k": 10240, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 508, + "n": 512, + "k": 10240, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 508, + "n": 1024, + "k": 10240, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 508, + "n": 1536, + "k": 10240, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 508, + "n": 4096, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 508, + "n": 10240, + "k": 1024, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 510, + "n": 320, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 510, + "n": 896, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 510, + "n": 7168, + "k": 448, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 510, + "n": 7168, + "k": 1024, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 512, + "n": 24, + "k": 28672, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/hc_attn", + "deepseek-ai/DeepSeek-V4-Pro-0813/hc_ffn", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.hc_attn", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.hc_ffn", + "deepseek-ai/DeepSeek-V4-Pro/hc_attn", + "deepseek-ai/DeepSeek-V4-Pro/hc_ffn", + "deepseek-ai/DeepSeek-V4-Pro/mtp.hc_attn", + "deepseek-ai/DeepSeek-V4-Pro/mtp.hc_ffn", + "sgl-project/DeepSeek-V4-Pro-FP8/hc_attn", + "sgl-project/DeepSeek-V4-Pro-FP8/hc_ffn", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.hc_attn", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.hc_ffn" + ] + }, + { + "type": "gemm", + "args": { + "m": 512, + "n": 24, + "k": 40960, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 512, + "n": 24, + "k": 49152, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 512, + "n": 64, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.weights_proj", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.weights_proj", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.weights_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 512, + "n": 64, + "k": 10240, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 512, + "n": 64, + "k": 12288, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 512, + "n": 120, + "k": 12288, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 512, + "n": 192, + "k": 10240, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 512, + "n": 256, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.compressor.wkv", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.compressor.wkv", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.compressor.wgate", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.compressor.wkv" + ] + }, + { + "type": "gemm", + "args": { + "m": 512, + "n": 256, + "k": 10240, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 512, + "n": 256, + "k": 12288, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 512, + "n": 320, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 512, + "n": 384, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.gate", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.gate", + "deepseek-ai/DeepSeek-V4-Pro/ffn.gate", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.gate", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.gate", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.gate" + ] + }, + { + "type": "gemm", + "args": { + "m": 512, + "n": 384, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w3" + ] + }, + { + "type": "gemm", + "args": { + "m": 512, + "n": 512, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wkv", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wkv", + "deepseek-ai/DeepSeek-V4-Pro/attn.wkv", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wkv", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wkv", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wkv" + ] + }, + { + "type": "gemm", + "args": { + "m": 512, + "n": 512, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.compressor.wkv+wgate", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.compressor.wkv+wgate", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.compressor.wkv+wgate" + ] + }, + { + "type": "gemm", + "args": { + "m": 512, + "n": 512, + "k": 10240, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 512, + "n": 512, + "k": 10240, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 512, + "n": 512, + "k": 12288, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 512, + "n": 512, + "k": 12288, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 512, + "n": 768, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w3" + ] + }, + { + "type": "gemm", + "args": { + "m": 512, + "n": 1024, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.compressor.wkv", + "deepseek-ai/DeepSeek-V4-Pro/attn.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro/attn.compressor.wkv", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.compressor.wgate", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.compressor.wkv" + ] + }, + { + "type": "gemm", + "args": { + "m": 512, + "n": 1024, + "k": 10240, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 512, + "n": 1024, + "k": 10240, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 512, + "n": 1024, + "k": 12288, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 512, + "n": 1536, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w3" + ] + }, + { + "type": "gemm", + "args": { + "m": 512, + "n": 1536, + "k": 10240, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 512, + "n": 1536, + "k": 12288, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 512, + "n": 1792, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 512, + "n": 2048, + "k": 12288, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 512, + "n": 4096, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 512, + "n": 5120, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 512, + "n": 6144, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3" + ] + }, + { + "type": "gemm", + "args": { + "m": 512, + "n": 7168, + "k": 192, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w2" + ] + }, + { + "type": "gemm", + "args": { + "m": 512, + "n": 7168, + "k": 384, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w2" + ] + }, + { + "type": "gemm", + "args": { + "m": 512, + "n": 7168, + "k": 768, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w2" + ] + }, + { + "type": "gemm", + "args": { + "m": 512, + "n": 7168, + "k": 896, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 512, + "n": 7168, + "k": 1024, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 512, + "n": 7168, + "k": 2048, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 512, + "n": 7168, + "k": 3072, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w2" + ] + }, + { + "type": "gemm", + "args": { + "m": 512, + "n": 7168, + "k": 4096, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 512, + "n": 7168, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.e_proj", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.h_proj", + "deepseek-ai/DeepSeek-V4-Pro/mtp.e_proj", + "deepseek-ai/DeepSeek-V4-Pro/mtp.h_proj", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.e_proj", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.h_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 512, + "n": 7168, + "k": 16384, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 512, + "n": 8192, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 512, + "n": 10240, + "k": 512, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 512, + "n": 10240, + "k": 1024, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 512, + "n": 10240, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 512, + "n": 10240, + "k": 2048, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 512, + "n": 10240, + "k": 16384, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 512, + "n": 12288, + "k": 1024, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 512, + "n": 12288, + "k": 2048, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 512, + "n": 12288, + "k": 8192, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 512, + "n": 12288, + "k": 16384, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 512, + "n": 14336, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 512, + "n": 16384, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 512, + "n": 16384, + "k": 12288, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 512, + "n": 65536, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 512, + "n": 81920, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 513, + "n": 384, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w3" + ] + }, + { + "type": "gemm", + "args": { + "m": 513, + "n": 768, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w3" + ] + }, + { + "type": "gemm", + "args": { + "m": 513, + "n": 1792, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 513, + "n": 7168, + "k": 192, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w2" + ] + }, + { + "type": "gemm", + "args": { + "m": 513, + "n": 7168, + "k": 384, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w2" + ] + }, + { + "type": "gemm", + "args": { + "m": 513, + "n": 7168, + "k": 896, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 513, + "n": 7168, + "k": 2048, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 513, + "n": 10240, + "k": 2048, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 520, + "n": 24, + "k": 40960, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 520, + "n": 64, + "k": 10240, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 520, + "n": 192, + "k": 10240, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 520, + "n": 256, + "k": 10240, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 520, + "n": 384, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w3" + ] + }, + { + "type": "gemm", + "args": { + "m": 520, + "n": 512, + "k": 10240, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 520, + "n": 512, + "k": 10240, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 520, + "n": 768, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w3" + ] + }, + { + "type": "gemm", + "args": { + "m": 520, + "n": 896, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 520, + "n": 1024, + "k": 10240, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 520, + "n": 1536, + "k": 10240, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 520, + "n": 1792, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 520, + "n": 7168, + "k": 192, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w2" + ] + }, + { + "type": "gemm", + "args": { + "m": 520, + "n": 7168, + "k": 384, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w2" + ] + }, + { + "type": "gemm", + "args": { + "m": 520, + "n": 7168, + "k": 448, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 520, + "n": 7168, + "k": 896, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 520, + "n": 7168, + "k": 2048, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 520, + "n": 10240, + "k": 1024, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 520, + "n": 10240, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 520, + "n": 10240, + "k": 2048, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 520, + "n": 12288, + "k": 2048, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 528, + "n": 24, + "k": 40960, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 528, + "n": 64, + "k": 10240, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 528, + "n": 192, + "k": 10240, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 528, + "n": 256, + "k": 10240, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 528, + "n": 320, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 528, + "n": 384, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w3" + ] + }, + { + "type": "gemm", + "args": { + "m": 528, + "n": 512, + "k": 10240, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 528, + "n": 512, + "k": 10240, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 528, + "n": 768, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w3" + ] + }, + { + "type": "gemm", + "args": { + "m": 528, + "n": 896, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 528, + "n": 1024, + "k": 10240, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 528, + "n": 1536, + "k": 10240, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 528, + "n": 4096, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 528, + "n": 7168, + "k": 192, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w2" + ] + }, + { + "type": "gemm", + "args": { + "m": 528, + "n": 7168, + "k": 384, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w2" + ] + }, + { + "type": "gemm", + "args": { + "m": 528, + "n": 7168, + "k": 448, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 528, + "n": 7168, + "k": 1024, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 528, + "n": 7168, + "k": 2048, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 528, + "n": 10240, + "k": 1024, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 528, + "n": 10240, + "k": 2048, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 544, + "n": 320, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 544, + "n": 768, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w3" + ] + }, + { + "type": "gemm", + "args": { + "m": 544, + "n": 896, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 544, + "n": 4096, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 544, + "n": 7168, + "k": 384, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w2" + ] + }, + { + "type": "gemm", + "args": { + "m": 544, + "n": 7168, + "k": 448, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 544, + "n": 7168, + "k": 1024, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 544, + "n": 7168, + "k": 2048, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 567, + "n": 24, + "k": 28672, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/hc_attn", + "deepseek-ai/DeepSeek-V4-Pro-0813/hc_ffn", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.hc_attn", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.hc_ffn", + "deepseek-ai/DeepSeek-V4-Pro/hc_attn", + "deepseek-ai/DeepSeek-V4-Pro/hc_ffn", + "deepseek-ai/DeepSeek-V4-Pro/mtp.hc_attn", + "deepseek-ai/DeepSeek-V4-Pro/mtp.hc_ffn", + "sgl-project/DeepSeek-V4-Pro-FP8/hc_attn", + "sgl-project/DeepSeek-V4-Pro-FP8/hc_ffn", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.hc_attn", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.hc_ffn" + ] + }, + { + "type": "gemm", + "args": { + "m": 567, + "n": 64, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.weights_proj", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.weights_proj", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.weights_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 567, + "n": 256, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.compressor.wkv", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.compressor.wkv", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.compressor.wgate", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.compressor.wkv" + ] + }, + { + "type": "gemm", + "args": { + "m": 567, + "n": 512, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wkv", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wkv", + "deepseek-ai/DeepSeek-V4-Pro/attn.wkv", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wkv", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wkv", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wkv" + ] + }, + { + "type": "gemm", + "args": { + "m": 567, + "n": 512, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.compressor.wkv+wgate", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.compressor.wkv+wgate", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.compressor.wkv+wgate" + ] + }, + { + "type": "gemm", + "args": { + "m": 567, + "n": 1024, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.compressor.wkv", + "deepseek-ai/DeepSeek-V4-Pro/attn.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro/attn.compressor.wkv", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.compressor.wgate", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.compressor.wkv" + ] + }, + { + "type": "gemm", + "args": { + "m": 567, + "n": 1536, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w3" + ] + }, + { + "type": "gemm", + "args": { + "m": 567, + "n": 8192, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 568, + "n": 24, + "k": 28672, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/hc_attn", + "deepseek-ai/DeepSeek-V4-Pro-0813/hc_ffn", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.hc_attn", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.hc_ffn", + "deepseek-ai/DeepSeek-V4-Pro/hc_attn", + "deepseek-ai/DeepSeek-V4-Pro/hc_ffn", + "deepseek-ai/DeepSeek-V4-Pro/mtp.hc_attn", + "deepseek-ai/DeepSeek-V4-Pro/mtp.hc_ffn", + "sgl-project/DeepSeek-V4-Pro-FP8/hc_attn", + "sgl-project/DeepSeek-V4-Pro-FP8/hc_ffn", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.hc_attn", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.hc_ffn" + ] + }, + { + "type": "gemm", + "args": { + "m": 568, + "n": 24, + "k": 49152, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 568, + "n": 64, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.weights_proj", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.weights_proj", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.weights_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 568, + "n": 64, + "k": 12288, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 568, + "n": 120, + "k": 12288, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 568, + "n": 256, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.compressor.wkv", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.compressor.wkv", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.compressor.wgate", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.compressor.wkv" + ] + }, + { + "type": "gemm", + "args": { + "m": 568, + "n": 256, + "k": 12288, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 568, + "n": 384, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.gate", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.gate", + "deepseek-ai/DeepSeek-V4-Pro/ffn.gate", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.gate", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.gate", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.gate" + ] + }, + { + "type": "gemm", + "args": { + "m": 568, + "n": 512, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wkv", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wkv", + "deepseek-ai/DeepSeek-V4-Pro/attn.wkv", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wkv", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wkv", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wkv" + ] + }, + { + "type": "gemm", + "args": { + "m": 568, + "n": 512, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.compressor.wkv+wgate", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.compressor.wkv+wgate", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.compressor.wkv+wgate" + ] + }, + { + "type": "gemm", + "args": { + "m": 568, + "n": 512, + "k": 12288, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 568, + "n": 512, + "k": 12288, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 568, + "n": 1024, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.compressor.wkv", + "deepseek-ai/DeepSeek-V4-Pro/attn.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro/attn.compressor.wkv", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.compressor.wgate", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.compressor.wkv" + ] + }, + { + "type": "gemm", + "args": { + "m": 568, + "n": 1024, + "k": 12288, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 568, + "n": 1536, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w3" + ] + }, + { + "type": "gemm", + "args": { + "m": 568, + "n": 1536, + "k": 12288, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 568, + "n": 5120, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 568, + "n": 8192, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 568, + "n": 12288, + "k": 1024, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 569, + "n": 384, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.gate", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.gate", + "deepseek-ai/DeepSeek-V4-Pro/ffn.gate", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.gate", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.gate", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.gate" + ] + }, + { + "type": "gemm", + "args": { + "m": 571, + "n": 1536, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w3" + ] + }, + { + "type": "gemm", + "args": { + "m": 571, + "n": 7168, + "k": 768, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w2" + ] + }, + { + "type": "gemm", + "args": { + "m": 571, + "n": 7168, + "k": 4096, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 571, + "n": 16384, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 576, + "n": 24, + "k": 28672, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/hc_attn", + "deepseek-ai/DeepSeek-V4-Pro-0813/hc_ffn", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.hc_attn", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.hc_ffn", + "deepseek-ai/DeepSeek-V4-Pro/hc_attn", + "deepseek-ai/DeepSeek-V4-Pro/hc_ffn", + "deepseek-ai/DeepSeek-V4-Pro/mtp.hc_attn", + "deepseek-ai/DeepSeek-V4-Pro/mtp.hc_ffn", + "sgl-project/DeepSeek-V4-Pro-FP8/hc_attn", + "sgl-project/DeepSeek-V4-Pro-FP8/hc_ffn", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.hc_attn", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.hc_ffn" + ] + }, + { + "type": "gemm", + "args": { + "m": 576, + "n": 24, + "k": 40960, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 576, + "n": 24, + "k": 49152, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 576, + "n": 64, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.weights_proj", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.weights_proj", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.weights_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 576, + "n": 64, + "k": 10240, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 576, + "n": 64, + "k": 12288, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 576, + "n": 120, + "k": 12288, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 576, + "n": 192, + "k": 10240, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 576, + "n": 256, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.compressor.wkv", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.compressor.wkv", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.compressor.wgate", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.compressor.wkv" + ] + }, + { + "type": "gemm", + "args": { + "m": 576, + "n": 256, + "k": 10240, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 576, + "n": 256, + "k": 12288, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 576, + "n": 320, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 576, + "n": 384, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.gate", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.gate", + "deepseek-ai/DeepSeek-V4-Pro/ffn.gate", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.gate", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.gate", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.gate" + ] + }, + { + "type": "gemm", + "args": { + "m": 576, + "n": 384, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w3" + ] + }, + { + "type": "gemm", + "args": { + "m": 576, + "n": 512, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wkv", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wkv", + "deepseek-ai/DeepSeek-V4-Pro/attn.wkv", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wkv", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wkv", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wkv" + ] + }, + { + "type": "gemm", + "args": { + "m": 576, + "n": 512, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.compressor.wkv+wgate", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.compressor.wkv+wgate", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.compressor.wkv+wgate" + ] + }, + { + "type": "gemm", + "args": { + "m": 576, + "n": 512, + "k": 10240, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 576, + "n": 512, + "k": 10240, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 576, + "n": 512, + "k": 12288, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 576, + "n": 512, + "k": 12288, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 576, + "n": 1024, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.compressor.wkv", + "deepseek-ai/DeepSeek-V4-Pro/attn.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro/attn.compressor.wkv", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.compressor.wgate", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.compressor.wkv" + ] + }, + { + "type": "gemm", + "args": { + "m": 576, + "n": 1024, + "k": 10240, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 576, + "n": 1024, + "k": 10240, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 576, + "n": 1024, + "k": 12288, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 576, + "n": 1536, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w3" + ] + }, + { + "type": "gemm", + "args": { + "m": 576, + "n": 1536, + "k": 10240, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 576, + "n": 1536, + "k": 12288, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 576, + "n": 1792, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 576, + "n": 2048, + "k": 12288, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 576, + "n": 4096, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 576, + "n": 5120, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 576, + "n": 6144, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3" + ] + }, + { + "type": "gemm", + "args": { + "m": 576, + "n": 7168, + "k": 192, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w2" + ] + }, + { + "type": "gemm", + "args": { + "m": 576, + "n": 7168, + "k": 768, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w2" + ] + }, + { + "type": "gemm", + "args": { + "m": 576, + "n": 7168, + "k": 896, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 576, + "n": 7168, + "k": 1024, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 576, + "n": 7168, + "k": 2048, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 576, + "n": 7168, + "k": 3072, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w2" + ] + }, + { + "type": "gemm", + "args": { + "m": 576, + "n": 7168, + "k": 4096, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 576, + "n": 7168, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.e_proj", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.h_proj", + "deepseek-ai/DeepSeek-V4-Pro/mtp.e_proj", + "deepseek-ai/DeepSeek-V4-Pro/mtp.h_proj", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.e_proj", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.h_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 576, + "n": 7168, + "k": 16384, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 576, + "n": 8192, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 576, + "n": 10240, + "k": 512, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 576, + "n": 10240, + "k": 1024, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 576, + "n": 10240, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 576, + "n": 10240, + "k": 2048, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 576, + "n": 10240, + "k": 16384, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 576, + "n": 12288, + "k": 1024, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 576, + "n": 12288, + "k": 2048, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 576, + "n": 12288, + "k": 8192, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 576, + "n": 12288, + "k": 16384, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 576, + "n": 14336, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 576, + "n": 16384, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 576, + "n": 16384, + "k": 12288, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 576, + "n": 65536, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 576, + "n": 81920, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 577, + "n": 24, + "k": 49152, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 577, + "n": 64, + "k": 12288, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 577, + "n": 120, + "k": 12288, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 577, + "n": 256, + "k": 12288, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 577, + "n": 512, + "k": 12288, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 577, + "n": 512, + "k": 12288, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 577, + "n": 1024, + "k": 10240, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 577, + "n": 1024, + "k": 12288, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 577, + "n": 1536, + "k": 12288, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 577, + "n": 5120, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 577, + "n": 10240, + "k": 512, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 577, + "n": 10240, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 577, + "n": 12288, + "k": 1024, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 577, + "n": 12288, + "k": 2048, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 584, + "n": 1536, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w3" + ] + }, + { + "type": "gemm", + "args": { + "m": 584, + "n": 7168, + "k": 768, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w2" + ] + }, + { + "type": "gemm", + "args": { + "m": 584, + "n": 7168, + "k": 4096, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 584, + "n": 16384, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 592, + "n": 24, + "k": 28672, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/hc_attn", + "deepseek-ai/DeepSeek-V4-Pro-0813/hc_ffn", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.hc_attn", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.hc_ffn", + "deepseek-ai/DeepSeek-V4-Pro/hc_attn", + "deepseek-ai/DeepSeek-V4-Pro/hc_ffn", + "deepseek-ai/DeepSeek-V4-Pro/mtp.hc_attn", + "deepseek-ai/DeepSeek-V4-Pro/mtp.hc_ffn", + "sgl-project/DeepSeek-V4-Pro-FP8/hc_attn", + "sgl-project/DeepSeek-V4-Pro-FP8/hc_ffn", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.hc_attn", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.hc_ffn" + ] + }, + { + "type": "gemm", + "args": { + "m": 592, + "n": 24, + "k": 49152, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 592, + "n": 64, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.weights_proj", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.weights_proj", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.weights_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 592, + "n": 64, + "k": 12288, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 592, + "n": 120, + "k": 12288, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 592, + "n": 256, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.compressor.wkv", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.compressor.wkv", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.compressor.wgate", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.compressor.wkv" + ] + }, + { + "type": "gemm", + "args": { + "m": 592, + "n": 256, + "k": 12288, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 592, + "n": 384, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.gate", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.gate", + "deepseek-ai/DeepSeek-V4-Pro/ffn.gate", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.gate", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.gate", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.gate" + ] + }, + { + "type": "gemm", + "args": { + "m": 592, + "n": 512, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wkv", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wkv", + "deepseek-ai/DeepSeek-V4-Pro/attn.wkv", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wkv", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wkv", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wkv" + ] + }, + { + "type": "gemm", + "args": { + "m": 592, + "n": 512, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.compressor.wkv+wgate", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.compressor.wkv+wgate", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.compressor.wkv+wgate" + ] + }, + { + "type": "gemm", + "args": { + "m": 592, + "n": 512, + "k": 12288, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 592, + "n": 512, + "k": 12288, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 592, + "n": 1024, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.compressor.wkv", + "deepseek-ai/DeepSeek-V4-Pro/attn.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro/attn.compressor.wkv", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.compressor.wgate", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.compressor.wkv" + ] + }, + { + "type": "gemm", + "args": { + "m": 592, + "n": 1024, + "k": 12288, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 592, + "n": 1536, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w3" + ] + }, + { + "type": "gemm", + "args": { + "m": 592, + "n": 1536, + "k": 12288, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 592, + "n": 5120, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 592, + "n": 7168, + "k": 768, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w2" + ] + }, + { + "type": "gemm", + "args": { + "m": 592, + "n": 7168, + "k": 4096, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 592, + "n": 8192, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 592, + "n": 12288, + "k": 1024, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 592, + "n": 16384, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 608, + "n": 24, + "k": 28672, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/hc_attn", + "deepseek-ai/DeepSeek-V4-Pro-0813/hc_ffn", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.hc_attn", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.hc_ffn", + "deepseek-ai/DeepSeek-V4-Pro/hc_attn", + "deepseek-ai/DeepSeek-V4-Pro/hc_ffn", + "deepseek-ai/DeepSeek-V4-Pro/mtp.hc_attn", + "deepseek-ai/DeepSeek-V4-Pro/mtp.hc_ffn", + "sgl-project/DeepSeek-V4-Pro-FP8/hc_attn", + "sgl-project/DeepSeek-V4-Pro-FP8/hc_ffn", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.hc_attn", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.hc_ffn" + ] + }, + { + "type": "gemm", + "args": { + "m": 608, + "n": 64, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.weights_proj", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.weights_proj", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.weights_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 608, + "n": 256, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.compressor.wkv", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.compressor.wkv", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.compressor.wgate", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.compressor.wkv" + ] + }, + { + "type": "gemm", + "args": { + "m": 608, + "n": 384, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.gate", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.gate", + "deepseek-ai/DeepSeek-V4-Pro/ffn.gate", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.gate", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.gate", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.gate" + ] + }, + { + "type": "gemm", + "args": { + "m": 608, + "n": 512, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wkv", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wkv", + "deepseek-ai/DeepSeek-V4-Pro/attn.wkv", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wkv", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wkv", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wkv" + ] + }, + { + "type": "gemm", + "args": { + "m": 608, + "n": 512, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.compressor.wkv+wgate", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.compressor.wkv+wgate", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.compressor.wkv+wgate" + ] + }, + { + "type": "gemm", + "args": { + "m": 608, + "n": 1024, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.compressor.wkv", + "deepseek-ai/DeepSeek-V4-Pro/attn.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro/attn.compressor.wkv", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.compressor.wgate", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.compressor.wkv" + ] + }, + { + "type": "gemm", + "args": { + "m": 608, + "n": 1536, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w3" + ] + }, + { + "type": "gemm", + "args": { + "m": 608, + "n": 7168, + "k": 768, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w2" + ] + }, + { + "type": "gemm", + "args": { + "m": 608, + "n": 7168, + "k": 4096, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 608, + "n": 8192, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 608, + "n": 16384, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 636, + "n": 320, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 636, + "n": 4096, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 636, + "n": 7168, + "k": 1024, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 637, + "n": 24, + "k": 40960, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 637, + "n": 64, + "k": 10240, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 637, + "n": 192, + "k": 10240, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 637, + "n": 256, + "k": 10240, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 637, + "n": 512, + "k": 10240, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 637, + "n": 512, + "k": 10240, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 637, + "n": 1024, + "k": 10240, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 637, + "n": 1536, + "k": 10240, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 637, + "n": 10240, + "k": 1024, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 640, + "n": 24, + "k": 28672, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/hc_attn", + "deepseek-ai/DeepSeek-V4-Pro-0813/hc_ffn", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.hc_attn", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.hc_ffn", + "deepseek-ai/DeepSeek-V4-Pro/hc_attn", + "deepseek-ai/DeepSeek-V4-Pro/hc_ffn", + "deepseek-ai/DeepSeek-V4-Pro/mtp.hc_attn", + "deepseek-ai/DeepSeek-V4-Pro/mtp.hc_ffn", + "sgl-project/DeepSeek-V4-Pro-FP8/hc_attn", + "sgl-project/DeepSeek-V4-Pro-FP8/hc_ffn", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.hc_attn", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.hc_ffn" + ] + }, + { + "type": "gemm", + "args": { + "m": 640, + "n": 24, + "k": 40960, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 640, + "n": 24, + "k": 49152, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 640, + "n": 64, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.weights_proj", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.weights_proj", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.weights_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 640, + "n": 64, + "k": 10240, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 640, + "n": 64, + "k": 12288, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 640, + "n": 120, + "k": 12288, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 640, + "n": 192, + "k": 10240, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 640, + "n": 256, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.compressor.wkv", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.compressor.wkv", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.compressor.wgate", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.compressor.wkv" + ] + }, + { + "type": "gemm", + "args": { + "m": 640, + "n": 256, + "k": 10240, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 640, + "n": 256, + "k": 12288, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 640, + "n": 320, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 640, + "n": 384, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.gate", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.gate", + "deepseek-ai/DeepSeek-V4-Pro/ffn.gate", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.gate", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.gate", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.gate" + ] + }, + { + "type": "gemm", + "args": { + "m": 640, + "n": 384, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w3" + ] + }, + { + "type": "gemm", + "args": { + "m": 640, + "n": 512, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wkv", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wkv", + "deepseek-ai/DeepSeek-V4-Pro/attn.wkv", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wkv", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wkv", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wkv" + ] + }, + { + "type": "gemm", + "args": { + "m": 640, + "n": 512, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.compressor.wkv+wgate", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.compressor.wkv+wgate", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.compressor.wkv+wgate" + ] + }, + { + "type": "gemm", + "args": { + "m": 640, + "n": 512, + "k": 10240, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 640, + "n": 512, + "k": 10240, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 640, + "n": 512, + "k": 12288, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 640, + "n": 512, + "k": 12288, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 640, + "n": 768, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w3" + ] + }, + { + "type": "gemm", + "args": { + "m": 640, + "n": 1024, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.compressor.wkv", + "deepseek-ai/DeepSeek-V4-Pro/attn.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro/attn.compressor.wkv", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.compressor.wgate", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.compressor.wkv" + ] + }, + { + "type": "gemm", + "args": { + "m": 640, + "n": 1024, + "k": 10240, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 640, + "n": 1024, + "k": 10240, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 640, + "n": 1024, + "k": 12288, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 640, + "n": 1536, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w3" + ] + }, + { + "type": "gemm", + "args": { + "m": 640, + "n": 1536, + "k": 10240, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 640, + "n": 1536, + "k": 12288, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 640, + "n": 1792, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 640, + "n": 2048, + "k": 12288, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 640, + "n": 4096, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 640, + "n": 5120, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 640, + "n": 6144, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3" + ] + }, + { + "type": "gemm", + "args": { + "m": 640, + "n": 7168, + "k": 192, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w2" + ] + }, + { + "type": "gemm", + "args": { + "m": 640, + "n": 7168, + "k": 384, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w2" + ] + }, + { + "type": "gemm", + "args": { + "m": 640, + "n": 7168, + "k": 768, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w2" + ] + }, + { + "type": "gemm", + "args": { + "m": 640, + "n": 7168, + "k": 896, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 640, + "n": 7168, + "k": 1024, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 640, + "n": 7168, + "k": 2048, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 640, + "n": 7168, + "k": 3072, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w2" + ] + }, + { + "type": "gemm", + "args": { + "m": 640, + "n": 7168, + "k": 4096, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 640, + "n": 7168, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.e_proj", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.h_proj", + "deepseek-ai/DeepSeek-V4-Pro/mtp.e_proj", + "deepseek-ai/DeepSeek-V4-Pro/mtp.h_proj", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.e_proj", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.h_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 640, + "n": 7168, + "k": 16384, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 640, + "n": 8192, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 640, + "n": 10240, + "k": 512, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 640, + "n": 10240, + "k": 1024, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 640, + "n": 10240, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 640, + "n": 10240, + "k": 2048, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 640, + "n": 10240, + "k": 16384, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 640, + "n": 12288, + "k": 1024, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 640, + "n": 12288, + "k": 2048, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 640, + "n": 12288, + "k": 8192, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 640, + "n": 12288, + "k": 16384, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 640, + "n": 14336, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 640, + "n": 16384, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 640, + "n": 16384, + "k": 12288, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 640, + "n": 65536, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 640, + "n": 81920, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 641, + "n": 384, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w3" + ] + }, + { + "type": "gemm", + "args": { + "m": 641, + "n": 768, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w3" + ] + }, + { + "type": "gemm", + "args": { + "m": 641, + "n": 896, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 641, + "n": 1792, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 641, + "n": 7168, + "k": 192, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w2" + ] + }, + { + "type": "gemm", + "args": { + "m": 641, + "n": 7168, + "k": 384, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w2" + ] + }, + { + "type": "gemm", + "args": { + "m": 641, + "n": 7168, + "k": 448, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 641, + "n": 7168, + "k": 896, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 641, + "n": 7168, + "k": 2048, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 641, + "n": 10240, + "k": 2048, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 642, + "n": 6144, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3" + ] + }, + { + "type": "gemm", + "args": { + "m": 642, + "n": 7168, + "k": 3072, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w2" + ] + }, + { + "type": "gemm", + "args": { + "m": 642, + "n": 7168, + "k": 16384, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 642, + "n": 65536, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 648, + "n": 24, + "k": 40960, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 648, + "n": 64, + "k": 10240, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 648, + "n": 192, + "k": 10240, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 648, + "n": 256, + "k": 10240, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 648, + "n": 320, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 648, + "n": 384, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w3" + ] + }, + { + "type": "gemm", + "args": { + "m": 648, + "n": 512, + "k": 10240, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 648, + "n": 512, + "k": 10240, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 648, + "n": 768, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w3" + ] + }, + { + "type": "gemm", + "args": { + "m": 648, + "n": 896, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 648, + "n": 1024, + "k": 10240, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 648, + "n": 1536, + "k": 10240, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 648, + "n": 1792, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 648, + "n": 4096, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 648, + "n": 7168, + "k": 192, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w2" + ] + }, + { + "type": "gemm", + "args": { + "m": 648, + "n": 7168, + "k": 384, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w2" + ] + }, + { + "type": "gemm", + "args": { + "m": 648, + "n": 7168, + "k": 448, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 648, + "n": 7168, + "k": 896, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 648, + "n": 7168, + "k": 1024, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 648, + "n": 7168, + "k": 2048, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 648, + "n": 10240, + "k": 1024, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 648, + "n": 10240, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 648, + "n": 10240, + "k": 2048, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 648, + "n": 12288, + "k": 2048, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 656, + "n": 24, + "k": 40960, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 656, + "n": 64, + "k": 10240, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 656, + "n": 192, + "k": 10240, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 656, + "n": 256, + "k": 10240, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 656, + "n": 320, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 656, + "n": 512, + "k": 10240, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 656, + "n": 512, + "k": 10240, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 656, + "n": 768, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w3" + ] + }, + { + "type": "gemm", + "args": { + "m": 656, + "n": 896, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 656, + "n": 1024, + "k": 10240, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 656, + "n": 1536, + "k": 10240, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 656, + "n": 4096, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 656, + "n": 7168, + "k": 384, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w2" + ] + }, + { + "type": "gemm", + "args": { + "m": 656, + "n": 7168, + "k": 448, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 656, + "n": 7168, + "k": 1024, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 656, + "n": 7168, + "k": 2048, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 656, + "n": 10240, + "k": 1024, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 672, + "n": 4096, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 672, + "n": 7168, + "k": 1024, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 704, + "n": 24, + "k": 40960, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 704, + "n": 64, + "k": 10240, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 704, + "n": 192, + "k": 10240, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 704, + "n": 256, + "k": 10240, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 704, + "n": 320, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 704, + "n": 384, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w3" + ] + }, + { + "type": "gemm", + "args": { + "m": 704, + "n": 512, + "k": 10240, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 704, + "n": 512, + "k": 10240, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 704, + "n": 768, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w3" + ] + }, + { + "type": "gemm", + "args": { + "m": 704, + "n": 1024, + "k": 10240, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 704, + "n": 1536, + "k": 10240, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 704, + "n": 1792, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 704, + "n": 2048, + "k": 12288, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 704, + "n": 4096, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 704, + "n": 6144, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3" + ] + }, + { + "type": "gemm", + "args": { + "m": 704, + "n": 7168, + "k": 192, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w2" + ] + }, + { + "type": "gemm", + "args": { + "m": 704, + "n": 7168, + "k": 384, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w2" + ] + }, + { + "type": "gemm", + "args": { + "m": 704, + "n": 7168, + "k": 896, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 704, + "n": 7168, + "k": 1024, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 704, + "n": 7168, + "k": 2048, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 704, + "n": 7168, + "k": 3072, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w2" + ] + }, + { + "type": "gemm", + "args": { + "m": 704, + "n": 7168, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.e_proj", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.h_proj", + "deepseek-ai/DeepSeek-V4-Pro/mtp.e_proj", + "deepseek-ai/DeepSeek-V4-Pro/mtp.h_proj", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.e_proj", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.h_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 704, + "n": 7168, + "k": 16384, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 704, + "n": 10240, + "k": 1024, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 704, + "n": 10240, + "k": 2048, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 704, + "n": 10240, + "k": 16384, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 704, + "n": 12288, + "k": 1024, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 704, + "n": 12288, + "k": 8192, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 704, + "n": 12288, + "k": 16384, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 704, + "n": 14336, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 704, + "n": 16384, + "k": 12288, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 704, + "n": 65536, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 704, + "n": 81920, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 709, + "n": 24, + "k": 28672, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/hc_attn", + "deepseek-ai/DeepSeek-V4-Pro-0813/hc_ffn", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.hc_attn", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.hc_ffn", + "deepseek-ai/DeepSeek-V4-Pro/hc_attn", + "deepseek-ai/DeepSeek-V4-Pro/hc_ffn", + "deepseek-ai/DeepSeek-V4-Pro/mtp.hc_attn", + "deepseek-ai/DeepSeek-V4-Pro/mtp.hc_ffn", + "sgl-project/DeepSeek-V4-Pro-FP8/hc_attn", + "sgl-project/DeepSeek-V4-Pro-FP8/hc_ffn", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.hc_attn", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.hc_ffn" + ] + }, + { + "type": "gemm", + "args": { + "m": 709, + "n": 64, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.weights_proj", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.weights_proj", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.weights_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 709, + "n": 256, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.compressor.wkv", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.compressor.wkv", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.compressor.wgate", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.compressor.wkv" + ] + }, + { + "type": "gemm", + "args": { + "m": 709, + "n": 512, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wkv", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wkv", + "deepseek-ai/DeepSeek-V4-Pro/attn.wkv", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wkv", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wkv", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wkv" + ] + }, + { + "type": "gemm", + "args": { + "m": 709, + "n": 512, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.compressor.wkv+wgate", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.compressor.wkv+wgate", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.compressor.wkv+wgate" + ] + }, + { + "type": "gemm", + "args": { + "m": 709, + "n": 1024, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.compressor.wkv", + "deepseek-ai/DeepSeek-V4-Pro/attn.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro/attn.compressor.wkv", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.compressor.wgate", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.compressor.wkv" + ] + }, + { + "type": "gemm", + "args": { + "m": 709, + "n": 1536, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w3" + ] + }, + { + "type": "gemm", + "args": { + "m": 709, + "n": 8192, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 711, + "n": 24, + "k": 49152, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 711, + "n": 64, + "k": 12288, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 711, + "n": 120, + "k": 12288, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 711, + "n": 256, + "k": 12288, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 711, + "n": 384, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.gate", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.gate", + "deepseek-ai/DeepSeek-V4-Pro/ffn.gate", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.gate", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.gate", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.gate" + ] + }, + { + "type": "gemm", + "args": { + "m": 711, + "n": 512, + "k": 12288, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 711, + "n": 512, + "k": 12288, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 711, + "n": 1024, + "k": 12288, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 711, + "n": 1536, + "k": 12288, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 711, + "n": 5120, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 711, + "n": 12288, + "k": 1024, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 712, + "n": 24, + "k": 28672, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/hc_attn", + "deepseek-ai/DeepSeek-V4-Pro-0813/hc_ffn", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.hc_attn", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.hc_ffn", + "deepseek-ai/DeepSeek-V4-Pro/hc_attn", + "deepseek-ai/DeepSeek-V4-Pro/hc_ffn", + "deepseek-ai/DeepSeek-V4-Pro/mtp.hc_attn", + "deepseek-ai/DeepSeek-V4-Pro/mtp.hc_ffn", + "sgl-project/DeepSeek-V4-Pro-FP8/hc_attn", + "sgl-project/DeepSeek-V4-Pro-FP8/hc_ffn", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.hc_attn", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.hc_ffn" + ] + }, + { + "type": "gemm", + "args": { + "m": 712, + "n": 24, + "k": 49152, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 712, + "n": 64, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.weights_proj", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.weights_proj", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.weights_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 712, + "n": 64, + "k": 12288, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 712, + "n": 120, + "k": 12288, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 712, + "n": 256, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.compressor.wkv", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.compressor.wkv", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.compressor.wgate", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.compressor.wkv" + ] + }, + { + "type": "gemm", + "args": { + "m": 712, + "n": 256, + "k": 12288, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 712, + "n": 384, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.gate", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.gate", + "deepseek-ai/DeepSeek-V4-Pro/ffn.gate", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.gate", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.gate", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.gate" + ] + }, + { + "type": "gemm", + "args": { + "m": 712, + "n": 512, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wkv", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wkv", + "deepseek-ai/DeepSeek-V4-Pro/attn.wkv", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wkv", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wkv", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wkv" + ] + }, + { + "type": "gemm", + "args": { + "m": 712, + "n": 512, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.compressor.wkv+wgate", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.compressor.wkv+wgate", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.compressor.wkv+wgate" + ] + }, + { + "type": "gemm", + "args": { + "m": 712, + "n": 512, + "k": 12288, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 712, + "n": 512, + "k": 12288, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 712, + "n": 1024, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.compressor.wkv", + "deepseek-ai/DeepSeek-V4-Pro/attn.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro/attn.compressor.wkv", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.compressor.wgate", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.compressor.wkv" + ] + }, + { + "type": "gemm", + "args": { + "m": 712, + "n": 1024, + "k": 12288, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 712, + "n": 1536, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w3" + ] + }, + { + "type": "gemm", + "args": { + "m": 712, + "n": 1536, + "k": 12288, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 712, + "n": 5120, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 712, + "n": 8192, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 712, + "n": 12288, + "k": 1024, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 714, + "n": 1536, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w3" + ] + }, + { + "type": "gemm", + "args": { + "m": 714, + "n": 7168, + "k": 768, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w2" + ] + }, + { + "type": "gemm", + "args": { + "m": 714, + "n": 7168, + "k": 4096, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 714, + "n": 16384, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 720, + "n": 24, + "k": 28672, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/hc_attn", + "deepseek-ai/DeepSeek-V4-Pro-0813/hc_ffn", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.hc_attn", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.hc_ffn", + "deepseek-ai/DeepSeek-V4-Pro/hc_attn", + "deepseek-ai/DeepSeek-V4-Pro/hc_ffn", + "deepseek-ai/DeepSeek-V4-Pro/mtp.hc_attn", + "deepseek-ai/DeepSeek-V4-Pro/mtp.hc_ffn", + "sgl-project/DeepSeek-V4-Pro-FP8/hc_attn", + "sgl-project/DeepSeek-V4-Pro-FP8/hc_ffn", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.hc_attn", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.hc_ffn" + ] + }, + { + "type": "gemm", + "args": { + "m": 720, + "n": 64, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.weights_proj", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.weights_proj", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.weights_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 720, + "n": 256, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.compressor.wkv", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.compressor.wkv", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.compressor.wgate", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.compressor.wkv" + ] + }, + { + "type": "gemm", + "args": { + "m": 720, + "n": 384, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.gate", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.gate", + "deepseek-ai/DeepSeek-V4-Pro/ffn.gate", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.gate", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.gate", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.gate" + ] + }, + { + "type": "gemm", + "args": { + "m": 720, + "n": 384, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w3" + ] + }, + { + "type": "gemm", + "args": { + "m": 720, + "n": 512, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wkv", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wkv", + "deepseek-ai/DeepSeek-V4-Pro/attn.wkv", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wkv", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wkv", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wkv" + ] + }, + { + "type": "gemm", + "args": { + "m": 720, + "n": 512, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.compressor.wkv+wgate", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.compressor.wkv+wgate", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.compressor.wkv+wgate" + ] + }, + { + "type": "gemm", + "args": { + "m": 720, + "n": 1024, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.compressor.wkv", + "deepseek-ai/DeepSeek-V4-Pro/attn.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro/attn.compressor.wkv", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.compressor.wgate", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.compressor.wkv" + ] + }, + { + "type": "gemm", + "args": { + "m": 720, + "n": 1024, + "k": 10240, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 720, + "n": 1536, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w3" + ] + }, + { + "type": "gemm", + "args": { + "m": 720, + "n": 7168, + "k": 192, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w2" + ] + }, + { + "type": "gemm", + "args": { + "m": 720, + "n": 7168, + "k": 768, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w2" + ] + }, + { + "type": "gemm", + "args": { + "m": 720, + "n": 7168, + "k": 4096, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 720, + "n": 8192, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 720, + "n": 10240, + "k": 512, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 720, + "n": 10240, + "k": 2048, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 720, + "n": 16384, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 728, + "n": 1536, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w3" + ] + }, + { + "type": "gemm", + "args": { + "m": 728, + "n": 7168, + "k": 768, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w2" + ] + }, + { + "type": "gemm", + "args": { + "m": 728, + "n": 7168, + "k": 4096, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 728, + "n": 16384, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 736, + "n": 24, + "k": 28672, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/hc_attn", + "deepseek-ai/DeepSeek-V4-Pro-0813/hc_ffn", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.hc_attn", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.hc_ffn", + "deepseek-ai/DeepSeek-V4-Pro/hc_attn", + "deepseek-ai/DeepSeek-V4-Pro/hc_ffn", + "deepseek-ai/DeepSeek-V4-Pro/mtp.hc_attn", + "deepseek-ai/DeepSeek-V4-Pro/mtp.hc_ffn", + "sgl-project/DeepSeek-V4-Pro-FP8/hc_attn", + "sgl-project/DeepSeek-V4-Pro-FP8/hc_ffn", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.hc_attn", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.hc_ffn" + ] + }, + { + "type": "gemm", + "args": { + "m": 736, + "n": 64, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.weights_proj", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.weights_proj", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.weights_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 736, + "n": 256, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.compressor.wkv", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.compressor.wkv", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.compressor.wgate", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.compressor.wkv" + ] + }, + { + "type": "gemm", + "args": { + "m": 736, + "n": 320, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 736, + "n": 384, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.gate", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.gate", + "deepseek-ai/DeepSeek-V4-Pro/ffn.gate", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.gate", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.gate", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.gate" + ] + }, + { + "type": "gemm", + "args": { + "m": 736, + "n": 512, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wkv", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wkv", + "deepseek-ai/DeepSeek-V4-Pro/attn.wkv", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wkv", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wkv", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wkv" + ] + }, + { + "type": "gemm", + "args": { + "m": 736, + "n": 512, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.compressor.wkv+wgate", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.compressor.wkv+wgate", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.compressor.wkv+wgate" + ] + }, + { + "type": "gemm", + "args": { + "m": 736, + "n": 896, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 736, + "n": 1024, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.compressor.wkv", + "deepseek-ai/DeepSeek-V4-Pro/attn.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro/attn.compressor.wkv", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.compressor.wgate", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.compressor.wkv" + ] + }, + { + "type": "gemm", + "args": { + "m": 736, + "n": 1536, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w3" + ] + }, + { + "type": "gemm", + "args": { + "m": 736, + "n": 7168, + "k": 448, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 736, + "n": 7168, + "k": 768, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w2" + ] + }, + { + "type": "gemm", + "args": { + "m": 736, + "n": 7168, + "k": 4096, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 736, + "n": 8192, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 736, + "n": 16384, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 740, + "n": 1024, + "k": 10240, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 740, + "n": 10240, + "k": 512, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 768, + "n": 24, + "k": 28672, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/hc_attn", + "deepseek-ai/DeepSeek-V4-Pro-0813/hc_ffn", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.hc_attn", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.hc_ffn", + "deepseek-ai/DeepSeek-V4-Pro/hc_attn", + "deepseek-ai/DeepSeek-V4-Pro/hc_ffn", + "deepseek-ai/DeepSeek-V4-Pro/mtp.hc_attn", + "deepseek-ai/DeepSeek-V4-Pro/mtp.hc_ffn", + "sgl-project/DeepSeek-V4-Pro-FP8/hc_attn", + "sgl-project/DeepSeek-V4-Pro-FP8/hc_ffn", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.hc_attn", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.hc_ffn" + ] + }, + { + "type": "gemm", + "args": { + "m": 768, + "n": 24, + "k": 40960, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 768, + "n": 24, + "k": 49152, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 768, + "n": 64, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.weights_proj", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.weights_proj", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.weights_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 768, + "n": 64, + "k": 10240, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 768, + "n": 64, + "k": 12288, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 768, + "n": 120, + "k": 12288, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 768, + "n": 192, + "k": 10240, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 768, + "n": 256, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.compressor.wkv", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.compressor.wkv", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.compressor.wgate", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.compressor.wkv" + ] + }, + { + "type": "gemm", + "args": { + "m": 768, + "n": 256, + "k": 10240, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 768, + "n": 256, + "k": 12288, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 768, + "n": 320, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 768, + "n": 384, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.gate", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.gate", + "deepseek-ai/DeepSeek-V4-Pro/ffn.gate", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.gate", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.gate", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.gate" + ] + }, + { + "type": "gemm", + "args": { + "m": 768, + "n": 384, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w3" + ] + }, + { + "type": "gemm", + "args": { + "m": 768, + "n": 512, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wkv", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wkv", + "deepseek-ai/DeepSeek-V4-Pro/attn.wkv", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wkv", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wkv", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wkv" + ] + }, + { + "type": "gemm", + "args": { + "m": 768, + "n": 512, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.compressor.wkv+wgate", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.compressor.wkv+wgate", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.compressor.wkv+wgate" + ] + }, + { + "type": "gemm", + "args": { + "m": 768, + "n": 512, + "k": 10240, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 768, + "n": 512, + "k": 10240, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 768, + "n": 512, + "k": 12288, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 768, + "n": 512, + "k": 12288, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 768, + "n": 768, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w3" + ] + }, + { + "type": "gemm", + "args": { + "m": 768, + "n": 1024, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.compressor.wkv", + "deepseek-ai/DeepSeek-V4-Pro/attn.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro/attn.compressor.wkv", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.compressor.wgate", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.compressor.wkv" + ] + }, + { + "type": "gemm", + "args": { + "m": 768, + "n": 1024, + "k": 10240, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 768, + "n": 1024, + "k": 10240, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 768, + "n": 1024, + "k": 12288, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 768, + "n": 1536, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w3" + ] + }, + { + "type": "gemm", + "args": { + "m": 768, + "n": 1536, + "k": 10240, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 768, + "n": 1536, + "k": 12288, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 768, + "n": 1792, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 768, + "n": 2048, + "k": 12288, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 768, + "n": 4096, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 768, + "n": 5120, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 768, + "n": 6144, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3" + ] + }, + { + "type": "gemm", + "args": { + "m": 768, + "n": 7168, + "k": 192, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w2" + ] + }, + { + "type": "gemm", + "args": { + "m": 768, + "n": 7168, + "k": 384, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w2" + ] + }, + { + "type": "gemm", + "args": { + "m": 768, + "n": 7168, + "k": 768, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w2" + ] + }, + { + "type": "gemm", + "args": { + "m": 768, + "n": 7168, + "k": 896, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 768, + "n": 7168, + "k": 1024, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 768, + "n": 7168, + "k": 2048, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 768, + "n": 7168, + "k": 3072, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w2" + ] + }, + { + "type": "gemm", + "args": { + "m": 768, + "n": 7168, + "k": 4096, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 768, + "n": 7168, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.e_proj", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.h_proj", + "deepseek-ai/DeepSeek-V4-Pro/mtp.e_proj", + "deepseek-ai/DeepSeek-V4-Pro/mtp.h_proj", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.e_proj", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.h_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 768, + "n": 7168, + "k": 16384, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 768, + "n": 8192, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 768, + "n": 10240, + "k": 512, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 768, + "n": 10240, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 768, + "n": 10240, + "k": 2048, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 768, + "n": 10240, + "k": 16384, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 768, + "n": 12288, + "k": 1024, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 768, + "n": 12288, + "k": 2048, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 768, + "n": 12288, + "k": 8192, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 768, + "n": 12288, + "k": 16384, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 768, + "n": 14336, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 768, + "n": 16384, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 768, + "n": 16384, + "k": 12288, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 768, + "n": 65536, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 768, + "n": 81920, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 771, + "n": 10240, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 771, + "n": 12288, + "k": 2048, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 784, + "n": 24, + "k": 49152, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 784, + "n": 64, + "k": 12288, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 784, + "n": 120, + "k": 12288, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 784, + "n": 256, + "k": 12288, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 784, + "n": 512, + "k": 12288, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 784, + "n": 512, + "k": 12288, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 784, + "n": 1024, + "k": 12288, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 784, + "n": 1536, + "k": 12288, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 784, + "n": 5120, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 784, + "n": 12288, + "k": 1024, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 796, + "n": 320, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 796, + "n": 4096, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 796, + "n": 7168, + "k": 1024, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 799, + "n": 24, + "k": 40960, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 799, + "n": 64, + "k": 10240, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 799, + "n": 192, + "k": 10240, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 799, + "n": 256, + "k": 10240, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 799, + "n": 512, + "k": 10240, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 799, + "n": 512, + "k": 10240, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 799, + "n": 1024, + "k": 10240, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 799, + "n": 1536, + "k": 10240, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 799, + "n": 10240, + "k": 1024, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 800, + "n": 320, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 800, + "n": 4096, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 800, + "n": 7168, + "k": 1024, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 806, + "n": 10240, + "k": 2048, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 816, + "n": 320, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 816, + "n": 896, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 816, + "n": 4096, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 816, + "n": 7168, + "k": 448, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 816, + "n": 7168, + "k": 1024, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 832, + "n": 24, + "k": 28672, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/hc_attn", + "deepseek-ai/DeepSeek-V4-Pro-0813/hc_ffn", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.hc_attn", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.hc_ffn", + "deepseek-ai/DeepSeek-V4-Pro/hc_attn", + "deepseek-ai/DeepSeek-V4-Pro/hc_ffn", + "deepseek-ai/DeepSeek-V4-Pro/mtp.hc_attn", + "deepseek-ai/DeepSeek-V4-Pro/mtp.hc_ffn", + "sgl-project/DeepSeek-V4-Pro-FP8/hc_attn", + "sgl-project/DeepSeek-V4-Pro-FP8/hc_ffn", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.hc_attn", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.hc_ffn" + ] + }, + { + "type": "gemm", + "args": { + "m": 832, + "n": 24, + "k": 40960, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 832, + "n": 24, + "k": 49152, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 832, + "n": 64, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.weights_proj", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.weights_proj", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.weights_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 832, + "n": 64, + "k": 10240, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 832, + "n": 64, + "k": 12288, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 832, + "n": 120, + "k": 12288, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 832, + "n": 192, + "k": 10240, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 832, + "n": 256, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.compressor.wkv", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.compressor.wkv", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.compressor.wgate", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.compressor.wkv" + ] + }, + { + "type": "gemm", + "args": { + "m": 832, + "n": 256, + "k": 10240, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 832, + "n": 256, + "k": 12288, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 832, + "n": 320, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 832, + "n": 384, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.gate", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.gate", + "deepseek-ai/DeepSeek-V4-Pro/ffn.gate", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.gate", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.gate", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.gate" + ] + }, + { + "type": "gemm", + "args": { + "m": 832, + "n": 384, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w3" + ] + }, + { + "type": "gemm", + "args": { + "m": 832, + "n": 512, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wkv", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wkv", + "deepseek-ai/DeepSeek-V4-Pro/attn.wkv", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wkv", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wkv", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wkv" + ] + }, + { + "type": "gemm", + "args": { + "m": 832, + "n": 512, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.compressor.wkv+wgate", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.compressor.wkv+wgate", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.compressor.wkv+wgate" + ] + }, + { + "type": "gemm", + "args": { + "m": 832, + "n": 512, + "k": 10240, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 832, + "n": 512, + "k": 10240, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 832, + "n": 512, + "k": 12288, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 832, + "n": 512, + "k": 12288, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 832, + "n": 1024, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.compressor.wkv", + "deepseek-ai/DeepSeek-V4-Pro/attn.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro/attn.compressor.wkv", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.compressor.wgate", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.compressor.wkv" + ] + }, + { + "type": "gemm", + "args": { + "m": 832, + "n": 1024, + "k": 10240, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 832, + "n": 1024, + "k": 10240, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 832, + "n": 1024, + "k": 12288, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 832, + "n": 1536, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w3" + ] + }, + { + "type": "gemm", + "args": { + "m": 832, + "n": 1536, + "k": 10240, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 832, + "n": 1536, + "k": 12288, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 832, + "n": 1792, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 832, + "n": 4096, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 832, + "n": 5120, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 832, + "n": 6144, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3" + ] + }, + { + "type": "gemm", + "args": { + "m": 832, + "n": 7168, + "k": 192, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w2" + ] + }, + { + "type": "gemm", + "args": { + "m": 832, + "n": 7168, + "k": 768, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w2" + ] + }, + { + "type": "gemm", + "args": { + "m": 832, + "n": 7168, + "k": 896, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 832, + "n": 7168, + "k": 1024, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 832, + "n": 7168, + "k": 2048, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 832, + "n": 7168, + "k": 3072, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w2" + ] + }, + { + "type": "gemm", + "args": { + "m": 832, + "n": 7168, + "k": 4096, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 832, + "n": 7168, + "k": 16384, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 832, + "n": 8192, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 832, + "n": 10240, + "k": 512, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 832, + "n": 10240, + "k": 1024, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 832, + "n": 10240, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 832, + "n": 10240, + "k": 2048, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 832, + "n": 12288, + "k": 1024, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 832, + "n": 12288, + "k": 2048, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 832, + "n": 16384, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 832, + "n": 65536, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 833, + "n": 384, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w3" + ] + }, + { + "type": "gemm", + "args": { + "m": 833, + "n": 768, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w3" + ] + }, + { + "type": "gemm", + "args": { + "m": 833, + "n": 896, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 833, + "n": 1792, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 833, + "n": 7168, + "k": 192, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w2" + ] + }, + { + "type": "gemm", + "args": { + "m": 833, + "n": 7168, + "k": 384, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w2" + ] + }, + { + "type": "gemm", + "args": { + "m": 833, + "n": 7168, + "k": 448, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 833, + "n": 7168, + "k": 896, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 833, + "n": 7168, + "k": 2048, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 840, + "n": 24, + "k": 40960, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 840, + "n": 64, + "k": 10240, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 840, + "n": 192, + "k": 10240, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 840, + "n": 256, + "k": 10240, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 840, + "n": 320, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 840, + "n": 384, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w3" + ] + }, + { + "type": "gemm", + "args": { + "m": 840, + "n": 512, + "k": 10240, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 840, + "n": 512, + "k": 10240, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 840, + "n": 768, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w3" + ] + }, + { + "type": "gemm", + "args": { + "m": 840, + "n": 896, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 840, + "n": 1024, + "k": 10240, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 840, + "n": 1024, + "k": 10240, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 840, + "n": 1536, + "k": 10240, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 840, + "n": 1792, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 840, + "n": 4096, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 840, + "n": 7168, + "k": 192, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w2" + ] + }, + { + "type": "gemm", + "args": { + "m": 840, + "n": 7168, + "k": 384, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w2" + ] + }, + { + "type": "gemm", + "args": { + "m": 840, + "n": 7168, + "k": 448, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 840, + "n": 7168, + "k": 896, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 840, + "n": 7168, + "k": 1024, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 840, + "n": 7168, + "k": 2048, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 840, + "n": 10240, + "k": 512, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 840, + "n": 10240, + "k": 1024, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 840, + "n": 10240, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 840, + "n": 10240, + "k": 2048, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 840, + "n": 12288, + "k": 2048, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 848, + "n": 24, + "k": 40960, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 848, + "n": 64, + "k": 10240, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 848, + "n": 192, + "k": 10240, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 848, + "n": 256, + "k": 10240, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 848, + "n": 384, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w3" + ] + }, + { + "type": "gemm", + "args": { + "m": 848, + "n": 512, + "k": 10240, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 848, + "n": 512, + "k": 10240, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 848, + "n": 1024, + "k": 10240, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 848, + "n": 1536, + "k": 10240, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 848, + "n": 7168, + "k": 192, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w2" + ] + }, + { + "type": "gemm", + "args": { + "m": 848, + "n": 10240, + "k": 1024, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 853, + "n": 384, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w3" + ] + }, + { + "type": "gemm", + "args": { + "m": 853, + "n": 7168, + "k": 192, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w2" + ] + }, + { + "type": "gemm", + "args": { + "m": 887, + "n": 24, + "k": 28672, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/hc_attn", + "deepseek-ai/DeepSeek-V4-Pro-0813/hc_ffn", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.hc_attn", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.hc_ffn", + "deepseek-ai/DeepSeek-V4-Pro/hc_attn", + "deepseek-ai/DeepSeek-V4-Pro/hc_ffn", + "deepseek-ai/DeepSeek-V4-Pro/mtp.hc_attn", + "deepseek-ai/DeepSeek-V4-Pro/mtp.hc_ffn", + "sgl-project/DeepSeek-V4-Pro-FP8/hc_attn", + "sgl-project/DeepSeek-V4-Pro-FP8/hc_ffn", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.hc_attn", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.hc_ffn" + ] + }, + { + "type": "gemm", + "args": { + "m": 887, + "n": 64, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.weights_proj", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.weights_proj", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.weights_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 887, + "n": 256, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.compressor.wkv", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.compressor.wkv", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.compressor.wgate", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.compressor.wkv" + ] + }, + { + "type": "gemm", + "args": { + "m": 887, + "n": 512, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wkv", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wkv", + "deepseek-ai/DeepSeek-V4-Pro/attn.wkv", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wkv", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wkv", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wkv" + ] + }, + { + "type": "gemm", + "args": { + "m": 887, + "n": 512, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.compressor.wkv+wgate", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.compressor.wkv+wgate", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.compressor.wkv+wgate" + ] + }, + { + "type": "gemm", + "args": { + "m": 887, + "n": 1024, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.compressor.wkv", + "deepseek-ai/DeepSeek-V4-Pro/attn.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro/attn.compressor.wkv", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.compressor.wgate", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.compressor.wkv" + ] + }, + { + "type": "gemm", + "args": { + "m": 887, + "n": 1536, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w3" + ] + }, + { + "type": "gemm", + "args": { + "m": 887, + "n": 8192, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 888, + "n": 24, + "k": 28672, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/hc_attn", + "deepseek-ai/DeepSeek-V4-Pro-0813/hc_ffn", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.hc_attn", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.hc_ffn", + "deepseek-ai/DeepSeek-V4-Pro/hc_attn", + "deepseek-ai/DeepSeek-V4-Pro/hc_ffn", + "deepseek-ai/DeepSeek-V4-Pro/mtp.hc_attn", + "deepseek-ai/DeepSeek-V4-Pro/mtp.hc_ffn", + "sgl-project/DeepSeek-V4-Pro-FP8/hc_attn", + "sgl-project/DeepSeek-V4-Pro-FP8/hc_ffn", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.hc_attn", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.hc_ffn" + ] + }, + { + "type": "gemm", + "args": { + "m": 888, + "n": 64, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.weights_proj", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.weights_proj", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.weights_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 888, + "n": 256, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.compressor.wkv", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.compressor.wkv", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.compressor.wgate", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.compressor.wkv" + ] + }, + { + "type": "gemm", + "args": { + "m": 888, + "n": 512, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wkv", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wkv", + "deepseek-ai/DeepSeek-V4-Pro/attn.wkv", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wkv", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wkv", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wkv" + ] + }, + { + "type": "gemm", + "args": { + "m": 888, + "n": 512, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.compressor.wkv+wgate", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.compressor.wkv+wgate", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.compressor.wkv+wgate" + ] + }, + { + "type": "gemm", + "args": { + "m": 888, + "n": 1024, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.compressor.wkv", + "deepseek-ai/DeepSeek-V4-Pro/attn.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro/attn.compressor.wkv", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.compressor.wgate", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.compressor.wkv" + ] + }, + { + "type": "gemm", + "args": { + "m": 888, + "n": 1536, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w3" + ] + }, + { + "type": "gemm", + "args": { + "m": 888, + "n": 8192, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 889, + "n": 384, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.gate", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.gate", + "deepseek-ai/DeepSeek-V4-Pro/ffn.gate", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.gate", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.gate", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.gate" + ] + }, + { + "type": "gemm", + "args": { + "m": 893, + "n": 1536, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w3" + ] + }, + { + "type": "gemm", + "args": { + "m": 893, + "n": 7168, + "k": 768, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w2" + ] + }, + { + "type": "gemm", + "args": { + "m": 893, + "n": 7168, + "k": 4096, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 893, + "n": 16384, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 896, + "n": 24, + "k": 28672, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/hc_attn", + "deepseek-ai/DeepSeek-V4-Pro-0813/hc_ffn", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.hc_attn", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.hc_ffn", + "deepseek-ai/DeepSeek-V4-Pro/hc_attn", + "deepseek-ai/DeepSeek-V4-Pro/hc_ffn", + "deepseek-ai/DeepSeek-V4-Pro/mtp.hc_attn", + "deepseek-ai/DeepSeek-V4-Pro/mtp.hc_ffn", + "sgl-project/DeepSeek-V4-Pro-FP8/hc_attn", + "sgl-project/DeepSeek-V4-Pro-FP8/hc_ffn", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.hc_attn", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.hc_ffn" + ] + }, + { + "type": "gemm", + "args": { + "m": 896, + "n": 24, + "k": 40960, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 896, + "n": 24, + "k": 49152, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 896, + "n": 64, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.weights_proj", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.weights_proj", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.weights_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 896, + "n": 64, + "k": 10240, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 896, + "n": 64, + "k": 12288, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 896, + "n": 120, + "k": 12288, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 896, + "n": 192, + "k": 10240, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 896, + "n": 256, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.compressor.wkv", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.compressor.wkv", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.compressor.wgate", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.compressor.wkv" + ] + }, + { + "type": "gemm", + "args": { + "m": 896, + "n": 256, + "k": 10240, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 896, + "n": 256, + "k": 12288, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 896, + "n": 320, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 896, + "n": 384, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.gate", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.gate", + "deepseek-ai/DeepSeek-V4-Pro/ffn.gate", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.gate", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.gate", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.gate" + ] + }, + { + "type": "gemm", + "args": { + "m": 896, + "n": 384, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w3" + ] + }, + { + "type": "gemm", + "args": { + "m": 896, + "n": 512, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wkv", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wkv", + "deepseek-ai/DeepSeek-V4-Pro/attn.wkv", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wkv", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wkv", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wkv" + ] + }, + { + "type": "gemm", + "args": { + "m": 896, + "n": 512, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.compressor.wkv+wgate", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.compressor.wkv+wgate", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.compressor.wkv+wgate" + ] + }, + { + "type": "gemm", + "args": { + "m": 896, + "n": 512, + "k": 10240, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 896, + "n": 512, + "k": 10240, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 896, + "n": 512, + "k": 12288, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 896, + "n": 512, + "k": 12288, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 896, + "n": 768, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w3" + ] + }, + { + "type": "gemm", + "args": { + "m": 896, + "n": 1024, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.compressor.wkv", + "deepseek-ai/DeepSeek-V4-Pro/attn.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro/attn.compressor.wkv", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.compressor.wgate", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.compressor.wkv" + ] + }, + { + "type": "gemm", + "args": { + "m": 896, + "n": 1024, + "k": 10240, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 896, + "n": 1024, + "k": 10240, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 896, + "n": 1024, + "k": 12288, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 896, + "n": 1536, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w3" + ] + }, + { + "type": "gemm", + "args": { + "m": 896, + "n": 1536, + "k": 10240, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 896, + "n": 1536, + "k": 12288, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 896, + "n": 1792, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 896, + "n": 2048, + "k": 12288, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 896, + "n": 4096, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 896, + "n": 5120, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 896, + "n": 6144, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3" + ] + }, + { + "type": "gemm", + "args": { + "m": 896, + "n": 7168, + "k": 192, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w2" + ] + }, + { + "type": "gemm", + "args": { + "m": 896, + "n": 7168, + "k": 384, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w2" + ] + }, + { + "type": "gemm", + "args": { + "m": 896, + "n": 7168, + "k": 768, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w2" + ] + }, + { + "type": "gemm", + "args": { + "m": 896, + "n": 7168, + "k": 896, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 896, + "n": 7168, + "k": 1024, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 896, + "n": 7168, + "k": 2048, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 896, + "n": 7168, + "k": 3072, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w2" + ] + }, + { + "type": "gemm", + "args": { + "m": 896, + "n": 7168, + "k": 4096, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 896, + "n": 7168, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.e_proj", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.h_proj", + "deepseek-ai/DeepSeek-V4-Pro/mtp.e_proj", + "deepseek-ai/DeepSeek-V4-Pro/mtp.h_proj", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.e_proj", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.h_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 896, + "n": 7168, + "k": 16384, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 896, + "n": 8192, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 896, + "n": 10240, + "k": 512, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 896, + "n": 10240, + "k": 1024, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 896, + "n": 10240, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 896, + "n": 10240, + "k": 2048, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 896, + "n": 10240, + "k": 16384, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 896, + "n": 12288, + "k": 1024, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 896, + "n": 12288, + "k": 2048, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 896, + "n": 12288, + "k": 8192, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 896, + "n": 12288, + "k": 16384, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 896, + "n": 14336, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 896, + "n": 16384, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 896, + "n": 16384, + "k": 12288, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 896, + "n": 65536, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 896, + "n": 81920, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 897, + "n": 24, + "k": 49152, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 897, + "n": 64, + "k": 12288, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 897, + "n": 120, + "k": 12288, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 897, + "n": 256, + "k": 12288, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 897, + "n": 512, + "k": 12288, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 897, + "n": 512, + "k": 12288, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 897, + "n": 1024, + "k": 12288, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 897, + "n": 1536, + "k": 12288, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 897, + "n": 5120, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 897, + "n": 12288, + "k": 1024, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 904, + "n": 24, + "k": 49152, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 904, + "n": 64, + "k": 12288, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 904, + "n": 120, + "k": 12288, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 904, + "n": 256, + "k": 12288, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 904, + "n": 384, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.gate", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.gate", + "deepseek-ai/DeepSeek-V4-Pro/ffn.gate", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.gate", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.gate", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.gate" + ] + }, + { + "type": "gemm", + "args": { + "m": 904, + "n": 512, + "k": 12288, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 904, + "n": 512, + "k": 12288, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 904, + "n": 1024, + "k": 12288, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 904, + "n": 1536, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w3" + ] + }, + { + "type": "gemm", + "args": { + "m": 904, + "n": 1536, + "k": 12288, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 904, + "n": 5120, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 904, + "n": 7168, + "k": 768, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w2" + ] + }, + { + "type": "gemm", + "args": { + "m": 904, + "n": 7168, + "k": 4096, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 904, + "n": 12288, + "k": 1024, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 904, + "n": 16384, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 912, + "n": 24, + "k": 28672, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/hc_attn", + "deepseek-ai/DeepSeek-V4-Pro-0813/hc_ffn", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.hc_attn", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.hc_ffn", + "deepseek-ai/DeepSeek-V4-Pro/hc_attn", + "deepseek-ai/DeepSeek-V4-Pro/hc_ffn", + "deepseek-ai/DeepSeek-V4-Pro/mtp.hc_attn", + "deepseek-ai/DeepSeek-V4-Pro/mtp.hc_ffn", + "sgl-project/DeepSeek-V4-Pro-FP8/hc_attn", + "sgl-project/DeepSeek-V4-Pro-FP8/hc_ffn", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.hc_attn", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.hc_ffn" + ] + }, + { + "type": "gemm", + "args": { + "m": 912, + "n": 24, + "k": 49152, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 912, + "n": 64, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.weights_proj", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.weights_proj", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.weights_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 912, + "n": 64, + "k": 12288, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 912, + "n": 120, + "k": 12288, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 912, + "n": 256, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.compressor.wkv", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.compressor.wkv", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.compressor.wgate", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.compressor.wkv" + ] + }, + { + "type": "gemm", + "args": { + "m": 912, + "n": 256, + "k": 12288, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 912, + "n": 384, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.gate", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.gate", + "deepseek-ai/DeepSeek-V4-Pro/ffn.gate", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.gate", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.gate", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.gate" + ] + }, + { + "type": "gemm", + "args": { + "m": 912, + "n": 512, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wkv", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wkv", + "deepseek-ai/DeepSeek-V4-Pro/attn.wkv", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wkv", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wkv", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wkv" + ] + }, + { + "type": "gemm", + "args": { + "m": 912, + "n": 512, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.compressor.wkv+wgate", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.compressor.wkv+wgate", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.compressor.wkv+wgate" + ] + }, + { + "type": "gemm", + "args": { + "m": 912, + "n": 512, + "k": 12288, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 912, + "n": 512, + "k": 12288, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 912, + "n": 1024, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.compressor.wkv", + "deepseek-ai/DeepSeek-V4-Pro/attn.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro/attn.compressor.wkv", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.compressor.wgate", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.compressor.wkv" + ] + }, + { + "type": "gemm", + "args": { + "m": 912, + "n": 1024, + "k": 10240, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 912, + "n": 1024, + "k": 12288, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 912, + "n": 1536, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w3" + ] + }, + { + "type": "gemm", + "args": { + "m": 912, + "n": 1536, + "k": 12288, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 912, + "n": 5120, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 912, + "n": 7168, + "k": 768, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w2" + ] + }, + { + "type": "gemm", + "args": { + "m": 912, + "n": 7168, + "k": 4096, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 912, + "n": 8192, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 912, + "n": 10240, + "k": 512, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 912, + "n": 10240, + "k": 2048, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 912, + "n": 12288, + "k": 1024, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 912, + "n": 16384, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 928, + "n": 24, + "k": 28672, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/hc_attn", + "deepseek-ai/DeepSeek-V4-Pro-0813/hc_ffn", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.hc_attn", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.hc_ffn", + "deepseek-ai/DeepSeek-V4-Pro/hc_attn", + "deepseek-ai/DeepSeek-V4-Pro/hc_ffn", + "deepseek-ai/DeepSeek-V4-Pro/mtp.hc_attn", + "deepseek-ai/DeepSeek-V4-Pro/mtp.hc_ffn", + "sgl-project/DeepSeek-V4-Pro-FP8/hc_attn", + "sgl-project/DeepSeek-V4-Pro-FP8/hc_ffn", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.hc_attn", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.hc_ffn" + ] + }, + { + "type": "gemm", + "args": { + "m": 928, + "n": 64, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.weights_proj", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.weights_proj", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.weights_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 928, + "n": 256, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.compressor.wkv", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.compressor.wkv", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.compressor.wgate", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.compressor.wkv" + ] + }, + { + "type": "gemm", + "args": { + "m": 928, + "n": 384, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.gate", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.gate", + "deepseek-ai/DeepSeek-V4-Pro/ffn.gate", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.gate", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.gate", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.gate" + ] + }, + { + "type": "gemm", + "args": { + "m": 928, + "n": 512, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wkv", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wkv", + "deepseek-ai/DeepSeek-V4-Pro/attn.wkv", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wkv", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wkv", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wkv" + ] + }, + { + "type": "gemm", + "args": { + "m": 928, + "n": 512, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.compressor.wkv+wgate", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.compressor.wkv+wgate", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.compressor.wkv+wgate" + ] + }, + { + "type": "gemm", + "args": { + "m": 928, + "n": 1024, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.compressor.wkv", + "deepseek-ai/DeepSeek-V4-Pro/attn.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro/attn.compressor.wkv", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.compressor.wgate", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.compressor.wkv" + ] + }, + { + "type": "gemm", + "args": { + "m": 928, + "n": 1536, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w3" + ] + }, + { + "type": "gemm", + "args": { + "m": 928, + "n": 7168, + "k": 768, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w2" + ] + }, + { + "type": "gemm", + "args": { + "m": 928, + "n": 7168, + "k": 4096, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 928, + "n": 8192, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 928, + "n": 16384, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 960, + "n": 24, + "k": 28672, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/hc_attn", + "deepseek-ai/DeepSeek-V4-Pro-0813/hc_ffn", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.hc_attn", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.hc_ffn", + "deepseek-ai/DeepSeek-V4-Pro/hc_attn", + "deepseek-ai/DeepSeek-V4-Pro/hc_ffn", + "deepseek-ai/DeepSeek-V4-Pro/mtp.hc_attn", + "deepseek-ai/DeepSeek-V4-Pro/mtp.hc_ffn", + "sgl-project/DeepSeek-V4-Pro-FP8/hc_attn", + "sgl-project/DeepSeek-V4-Pro-FP8/hc_ffn", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.hc_attn", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.hc_ffn" + ] + }, + { + "type": "gemm", + "args": { + "m": 960, + "n": 24, + "k": 49152, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 960, + "n": 64, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.weights_proj", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.weights_proj", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.weights_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 960, + "n": 64, + "k": 12288, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 960, + "n": 120, + "k": 12288, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 960, + "n": 256, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.compressor.wkv", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.compressor.wkv", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.compressor.wgate", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.compressor.wkv" + ] + }, + { + "type": "gemm", + "args": { + "m": 960, + "n": 256, + "k": 12288, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 960, + "n": 384, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.gate", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.gate", + "deepseek-ai/DeepSeek-V4-Pro/ffn.gate", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.gate", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.gate", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.gate" + ] + }, + { + "type": "gemm", + "args": { + "m": 960, + "n": 512, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wkv", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wkv", + "deepseek-ai/DeepSeek-V4-Pro/attn.wkv", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wkv", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wkv", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wkv" + ] + }, + { + "type": "gemm", + "args": { + "m": 960, + "n": 512, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.compressor.wkv+wgate", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.compressor.wkv+wgate", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.compressor.wkv+wgate" + ] + }, + { + "type": "gemm", + "args": { + "m": 960, + "n": 512, + "k": 12288, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 960, + "n": 512, + "k": 12288, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 960, + "n": 768, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w3" + ] + }, + { + "type": "gemm", + "args": { + "m": 960, + "n": 1024, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.compressor.wkv", + "deepseek-ai/DeepSeek-V4-Pro/attn.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro/attn.compressor.wkv", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.compressor.wgate", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.compressor.wkv" + ] + }, + { + "type": "gemm", + "args": { + "m": 960, + "n": 1024, + "k": 10240, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 960, + "n": 1024, + "k": 12288, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 960, + "n": 1536, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w3" + ] + }, + { + "type": "gemm", + "args": { + "m": 960, + "n": 1536, + "k": 12288, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 960, + "n": 2048, + "k": 12288, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 960, + "n": 5120, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 960, + "n": 6144, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3" + ] + }, + { + "type": "gemm", + "args": { + "m": 960, + "n": 7168, + "k": 384, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w2" + ] + }, + { + "type": "gemm", + "args": { + "m": 960, + "n": 7168, + "k": 768, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w2" + ] + }, + { + "type": "gemm", + "args": { + "m": 960, + "n": 7168, + "k": 3072, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w2" + ] + }, + { + "type": "gemm", + "args": { + "m": 960, + "n": 7168, + "k": 4096, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 960, + "n": 7168, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.e_proj", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.h_proj", + "deepseek-ai/DeepSeek-V4-Pro/mtp.e_proj", + "deepseek-ai/DeepSeek-V4-Pro/mtp.h_proj", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.e_proj", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.h_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 960, + "n": 8192, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 960, + "n": 10240, + "k": 512, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 960, + "n": 10240, + "k": 16384, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 960, + "n": 12288, + "k": 1024, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 960, + "n": 12288, + "k": 8192, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 960, + "n": 12288, + "k": 16384, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 960, + "n": 14336, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 960, + "n": 16384, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 960, + "n": 16384, + "k": 12288, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 960, + "n": 81920, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 961, + "n": 1024, + "k": 10240, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 961, + "n": 10240, + "k": 512, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 962, + "n": 10240, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 962, + "n": 12288, + "k": 2048, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 968, + "n": 1024, + "k": 10240, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 968, + "n": 10240, + "k": 512, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 976, + "n": 768, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w3" + ] + }, + { + "type": "gemm", + "args": { + "m": 976, + "n": 7168, + "k": 384, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w2" + ] + }, + { + "type": "gemm", + "args": { + "m": 976, + "n": 7168, + "k": 2048, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 1005, + "n": 320, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 1005, + "n": 4096, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 1005, + "n": 7168, + "k": 1024, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 1024, + "n": 24, + "k": 28672, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/hc_attn", + "deepseek-ai/DeepSeek-V4-Pro-0813/hc_ffn", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.hc_attn", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.hc_ffn", + "deepseek-ai/DeepSeek-V4-Pro/hc_attn", + "deepseek-ai/DeepSeek-V4-Pro/hc_ffn", + "deepseek-ai/DeepSeek-V4-Pro/mtp.hc_attn", + "deepseek-ai/DeepSeek-V4-Pro/mtp.hc_ffn", + "sgl-project/DeepSeek-V4-Pro-FP8/hc_attn", + "sgl-project/DeepSeek-V4-Pro-FP8/hc_ffn", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.hc_attn", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.hc_ffn" + ] + }, + { + "type": "gemm", + "args": { + "m": 1024, + "n": 24, + "k": 40960, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 1024, + "n": 24, + "k": 49152, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 1024, + "n": 64, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.weights_proj", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.weights_proj", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.weights_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 1024, + "n": 64, + "k": 10240, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 1024, + "n": 64, + "k": 12288, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 1024, + "n": 120, + "k": 12288, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 1024, + "n": 192, + "k": 10240, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 1024, + "n": 256, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.compressor.wkv", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.compressor.wkv", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.compressor.wgate", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.compressor.wkv" + ] + }, + { + "type": "gemm", + "args": { + "m": 1024, + "n": 256, + "k": 10240, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 1024, + "n": 256, + "k": 12288, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 1024, + "n": 320, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 1024, + "n": 384, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.gate", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.gate", + "deepseek-ai/DeepSeek-V4-Pro/ffn.gate", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.gate", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.gate", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.gate" + ] + }, + { + "type": "gemm", + "args": { + "m": 1024, + "n": 384, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w3" + ] + }, + { + "type": "gemm", + "args": { + "m": 1024, + "n": 512, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wkv", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wkv", + "deepseek-ai/DeepSeek-V4-Pro/attn.wkv", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wkv", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wkv", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wkv" + ] + }, + { + "type": "gemm", + "args": { + "m": 1024, + "n": 512, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.compressor.wkv+wgate", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.compressor.wkv+wgate", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.compressor.wkv+wgate" + ] + }, + { + "type": "gemm", + "args": { + "m": 1024, + "n": 512, + "k": 10240, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 1024, + "n": 512, + "k": 10240, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 1024, + "n": 512, + "k": 12288, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 1024, + "n": 512, + "k": 12288, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 1024, + "n": 768, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w3" + ] + }, + { + "type": "gemm", + "args": { + "m": 1024, + "n": 1024, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.compressor.wkv", + "deepseek-ai/DeepSeek-V4-Pro/attn.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro/attn.compressor.wkv", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.compressor.wgate", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.compressor.wkv" + ] + }, + { + "type": "gemm", + "args": { + "m": 1024, + "n": 1024, + "k": 10240, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 1024, + "n": 1024, + "k": 10240, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 1024, + "n": 1024, + "k": 12288, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 1024, + "n": 1536, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w3" + ] + }, + { + "type": "gemm", + "args": { + "m": 1024, + "n": 1536, + "k": 10240, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 1024, + "n": 1536, + "k": 12288, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 1024, + "n": 1792, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 1024, + "n": 2048, + "k": 12288, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 1024, + "n": 4096, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 1024, + "n": 5120, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 1024, + "n": 6144, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3" + ] + }, + { + "type": "gemm", + "args": { + "m": 1024, + "n": 7168, + "k": 192, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w2" + ] + }, + { + "type": "gemm", + "args": { + "m": 1024, + "n": 7168, + "k": 384, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w2" + ] + }, + { + "type": "gemm", + "args": { + "m": 1024, + "n": 7168, + "k": 768, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w2" + ] + }, + { + "type": "gemm", + "args": { + "m": 1024, + "n": 7168, + "k": 896, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 1024, + "n": 7168, + "k": 1024, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 1024, + "n": 7168, + "k": 2048, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 1024, + "n": 7168, + "k": 3072, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w2" + ] + }, + { + "type": "gemm", + "args": { + "m": 1024, + "n": 7168, + "k": 4096, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 1024, + "n": 7168, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.e_proj", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.h_proj", + "deepseek-ai/DeepSeek-V4-Pro/mtp.e_proj", + "deepseek-ai/DeepSeek-V4-Pro/mtp.h_proj", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.e_proj", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.h_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 1024, + "n": 7168, + "k": 16384, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 1024, + "n": 8192, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 1024, + "n": 10240, + "k": 512, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 1024, + "n": 10240, + "k": 1024, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 1024, + "n": 10240, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 1024, + "n": 10240, + "k": 2048, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 1024, + "n": 10240, + "k": 16384, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 1024, + "n": 12288, + "k": 1024, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 1024, + "n": 12288, + "k": 2048, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 1024, + "n": 12288, + "k": 8192, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 1024, + "n": 12288, + "k": 16384, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 1024, + "n": 14336, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 1024, + "n": 16384, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 1024, + "n": 16384, + "k": 12288, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 1024, + "n": 65536, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 1024, + "n": 81920, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 1025, + "n": 24, + "k": 40960, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 1025, + "n": 64, + "k": 10240, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 1025, + "n": 192, + "k": 10240, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 1025, + "n": 256, + "k": 10240, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 1025, + "n": 512, + "k": 10240, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 1025, + "n": 512, + "k": 10240, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 1025, + "n": 896, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 1025, + "n": 1024, + "k": 10240, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 1025, + "n": 1536, + "k": 10240, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 1025, + "n": 7168, + "k": 448, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 1025, + "n": 10240, + "k": 1024, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 1025, + "n": 10240, + "k": 2048, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 1032, + "n": 24, + "k": 40960, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 1032, + "n": 64, + "k": 10240, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 1032, + "n": 192, + "k": 10240, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 1032, + "n": 256, + "k": 10240, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 1032, + "n": 320, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 1032, + "n": 512, + "k": 10240, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 1032, + "n": 512, + "k": 10240, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 1032, + "n": 896, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 1032, + "n": 1024, + "k": 10240, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 1032, + "n": 1536, + "k": 10240, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 1032, + "n": 4096, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 1032, + "n": 7168, + "k": 448, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 1032, + "n": 7168, + "k": 1024, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 1032, + "n": 10240, + "k": 1024, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 1032, + "n": 10240, + "k": 2048, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 1040, + "n": 24, + "k": 40960, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 1040, + "n": 64, + "k": 10240, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 1040, + "n": 192, + "k": 10240, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 1040, + "n": 256, + "k": 10240, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 1040, + "n": 320, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 1040, + "n": 512, + "k": 10240, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 1040, + "n": 512, + "k": 10240, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 1040, + "n": 896, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 1040, + "n": 1024, + "k": 10240, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 1040, + "n": 1536, + "k": 10240, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 1040, + "n": 4096, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 1040, + "n": 7168, + "k": 448, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 1040, + "n": 7168, + "k": 1024, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 1040, + "n": 10240, + "k": 1024, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 1041, + "n": 7168, + "k": 2048, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 1042, + "n": 768, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w3" + ] + }, + { + "type": "gemm", + "args": { + "m": 1042, + "n": 7168, + "k": 384, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w2" + ] + }, + { + "type": "gemm", + "args": { + "m": 1056, + "n": 320, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 1056, + "n": 896, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 1056, + "n": 4096, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 1056, + "n": 7168, + "k": 448, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 1056, + "n": 7168, + "k": 1024, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 1060, + "n": 1792, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 1060, + "n": 7168, + "k": 896, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 1072, + "n": 768, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w3" + ] + }, + { + "type": "gemm", + "args": { + "m": 1072, + "n": 7168, + "k": 384, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w2" + ] + }, + { + "type": "gemm", + "args": { + "m": 1072, + "n": 7168, + "k": 2048, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 1072, + "n": 10240, + "k": 2048, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 1088, + "n": 24, + "k": 40960, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 1088, + "n": 64, + "k": 10240, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 1088, + "n": 192, + "k": 10240, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 1088, + "n": 256, + "k": 10240, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 1088, + "n": 320, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 1088, + "n": 384, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w3" + ] + }, + { + "type": "gemm", + "args": { + "m": 1088, + "n": 512, + "k": 10240, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 1088, + "n": 512, + "k": 10240, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 1088, + "n": 1024, + "k": 10240, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 1088, + "n": 1536, + "k": 10240, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 1088, + "n": 1792, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 1088, + "n": 4096, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 1088, + "n": 6144, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3" + ] + }, + { + "type": "gemm", + "args": { + "m": 1088, + "n": 7168, + "k": 192, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w2" + ] + }, + { + "type": "gemm", + "args": { + "m": 1088, + "n": 7168, + "k": 896, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 1088, + "n": 7168, + "k": 1024, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 1088, + "n": 7168, + "k": 2048, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 1088, + "n": 7168, + "k": 3072, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w2" + ] + }, + { + "type": "gemm", + "args": { + "m": 1088, + "n": 7168, + "k": 16384, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 1088, + "n": 10240, + "k": 1024, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 1088, + "n": 10240, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 1088, + "n": 10240, + "k": 2048, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 1088, + "n": 12288, + "k": 2048, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 1088, + "n": 65536, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 1089, + "n": 384, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w3" + ] + }, + { + "type": "gemm", + "args": { + "m": 1089, + "n": 7168, + "k": 192, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w2" + ] + }, + { + "type": "gemm", + "args": { + "m": 1096, + "n": 384, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w3" + ] + }, + { + "type": "gemm", + "args": { + "m": 1096, + "n": 768, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w3" + ] + }, + { + "type": "gemm", + "args": { + "m": 1096, + "n": 1792, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 1096, + "n": 7168, + "k": 192, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w2" + ] + }, + { + "type": "gemm", + "args": { + "m": 1096, + "n": 7168, + "k": 384, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w2" + ] + }, + { + "type": "gemm", + "args": { + "m": 1096, + "n": 7168, + "k": 896, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 1096, + "n": 7168, + "k": 2048, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 1104, + "n": 384, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w3" + ] + }, + { + "type": "gemm", + "args": { + "m": 1104, + "n": 1024, + "k": 10240, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 1104, + "n": 7168, + "k": 192, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w2" + ] + }, + { + "type": "gemm", + "args": { + "m": 1104, + "n": 10240, + "k": 512, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 1109, + "n": 24, + "k": 28672, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/hc_attn", + "deepseek-ai/DeepSeek-V4-Pro-0813/hc_ffn", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.hc_attn", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.hc_ffn", + "deepseek-ai/DeepSeek-V4-Pro/hc_attn", + "deepseek-ai/DeepSeek-V4-Pro/hc_ffn", + "deepseek-ai/DeepSeek-V4-Pro/mtp.hc_attn", + "deepseek-ai/DeepSeek-V4-Pro/mtp.hc_ffn", + "sgl-project/DeepSeek-V4-Pro-FP8/hc_attn", + "sgl-project/DeepSeek-V4-Pro-FP8/hc_ffn", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.hc_attn", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.hc_ffn" + ] + }, + { + "type": "gemm", + "args": { + "m": 1109, + "n": 64, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.weights_proj", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.weights_proj", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.weights_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 1109, + "n": 256, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.compressor.wkv", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.compressor.wkv", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.compressor.wgate", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.compressor.wkv" + ] + }, + { + "type": "gemm", + "args": { + "m": 1109, + "n": 512, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wkv", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wkv", + "deepseek-ai/DeepSeek-V4-Pro/attn.wkv", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wkv", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wkv", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wkv" + ] + }, + { + "type": "gemm", + "args": { + "m": 1109, + "n": 512, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.compressor.wkv+wgate", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.compressor.wkv+wgate", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.compressor.wkv+wgate" + ] + }, + { + "type": "gemm", + "args": { + "m": 1109, + "n": 1024, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.compressor.wkv", + "deepseek-ai/DeepSeek-V4-Pro/attn.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro/attn.compressor.wkv", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.compressor.wgate", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.compressor.wkv" + ] + }, + { + "type": "gemm", + "args": { + "m": 1109, + "n": 1536, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w3" + ] + }, + { + "type": "gemm", + "args": { + "m": 1109, + "n": 8192, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 1112, + "n": 24, + "k": 28672, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/hc_attn", + "deepseek-ai/DeepSeek-V4-Pro-0813/hc_ffn", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.hc_attn", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.hc_ffn", + "deepseek-ai/DeepSeek-V4-Pro/hc_attn", + "deepseek-ai/DeepSeek-V4-Pro/hc_ffn", + "deepseek-ai/DeepSeek-V4-Pro/mtp.hc_attn", + "deepseek-ai/DeepSeek-V4-Pro/mtp.hc_ffn", + "sgl-project/DeepSeek-V4-Pro-FP8/hc_attn", + "sgl-project/DeepSeek-V4-Pro-FP8/hc_ffn", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.hc_attn", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.hc_ffn" + ] + }, + { + "type": "gemm", + "args": { + "m": 1112, + "n": 64, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.weights_proj", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.weights_proj", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.weights_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 1112, + "n": 256, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.compressor.wkv", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.compressor.wkv", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.compressor.wgate", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.compressor.wkv" + ] + }, + { + "type": "gemm", + "args": { + "m": 1112, + "n": 384, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.gate", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.gate", + "deepseek-ai/DeepSeek-V4-Pro/ffn.gate", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.gate", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.gate", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.gate" + ] + }, + { + "type": "gemm", + "args": { + "m": 1112, + "n": 512, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wkv", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wkv", + "deepseek-ai/DeepSeek-V4-Pro/attn.wkv", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wkv", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wkv", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wkv" + ] + }, + { + "type": "gemm", + "args": { + "m": 1112, + "n": 512, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.compressor.wkv+wgate", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.compressor.wkv+wgate", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.compressor.wkv+wgate" + ] + }, + { + "type": "gemm", + "args": { + "m": 1112, + "n": 1024, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.compressor.wkv", + "deepseek-ai/DeepSeek-V4-Pro/attn.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro/attn.compressor.wkv", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.compressor.wgate", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.compressor.wkv" + ] + }, + { + "type": "gemm", + "args": { + "m": 1112, + "n": 1536, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w3" + ] + }, + { + "type": "gemm", + "args": { + "m": 1112, + "n": 8192, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 1113, + "n": 384, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.gate", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.gate", + "deepseek-ai/DeepSeek-V4-Pro/ffn.gate", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.gate", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.gate", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.gate" + ] + }, + { + "type": "gemm", + "args": { + "m": 1117, + "n": 1536, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w3" + ] + }, + { + "type": "gemm", + "args": { + "m": 1117, + "n": 7168, + "k": 768, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w2" + ] + }, + { + "type": "gemm", + "args": { + "m": 1117, + "n": 7168, + "k": 4096, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 1117, + "n": 16384, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 1120, + "n": 24, + "k": 28672, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/hc_attn", + "deepseek-ai/DeepSeek-V4-Pro-0813/hc_ffn", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.hc_attn", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.hc_ffn", + "deepseek-ai/DeepSeek-V4-Pro/hc_attn", + "deepseek-ai/DeepSeek-V4-Pro/hc_ffn", + "deepseek-ai/DeepSeek-V4-Pro/mtp.hc_attn", + "deepseek-ai/DeepSeek-V4-Pro/mtp.hc_ffn", + "sgl-project/DeepSeek-V4-Pro-FP8/hc_attn", + "sgl-project/DeepSeek-V4-Pro-FP8/hc_ffn", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.hc_attn", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.hc_ffn" + ] + }, + { + "type": "gemm", + "args": { + "m": 1120, + "n": 64, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.weights_proj", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.weights_proj", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.weights_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 1120, + "n": 256, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.compressor.wkv", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.compressor.wkv", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.compressor.wgate", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.compressor.wkv" + ] + }, + { + "type": "gemm", + "args": { + "m": 1120, + "n": 384, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.gate", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.gate", + "deepseek-ai/DeepSeek-V4-Pro/ffn.gate", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.gate", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.gate", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.gate" + ] + }, + { + "type": "gemm", + "args": { + "m": 1120, + "n": 512, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wkv", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wkv", + "deepseek-ai/DeepSeek-V4-Pro/attn.wkv", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wkv", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wkv", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wkv" + ] + }, + { + "type": "gemm", + "args": { + "m": 1120, + "n": 512, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.compressor.wkv+wgate", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.compressor.wkv+wgate", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.compressor.wkv+wgate" + ] + }, + { + "type": "gemm", + "args": { + "m": 1120, + "n": 1024, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.compressor.wkv", + "deepseek-ai/DeepSeek-V4-Pro/attn.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro/attn.compressor.wkv", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.compressor.wgate", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.compressor.wkv" + ] + }, + { + "type": "gemm", + "args": { + "m": 1120, + "n": 1536, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w3" + ] + }, + { + "type": "gemm", + "args": { + "m": 1120, + "n": 7168, + "k": 768, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w2" + ] + }, + { + "type": "gemm", + "args": { + "m": 1120, + "n": 7168, + "k": 4096, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 1120, + "n": 8192, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 1120, + "n": 16384, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 1128, + "n": 1536, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w3" + ] + }, + { + "type": "gemm", + "args": { + "m": 1128, + "n": 7168, + "k": 768, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w2" + ] + }, + { + "type": "gemm", + "args": { + "m": 1128, + "n": 7168, + "k": 4096, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 1128, + "n": 16384, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 1136, + "n": 24, + "k": 28672, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/hc_attn", + "deepseek-ai/DeepSeek-V4-Pro-0813/hc_ffn", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.hc_attn", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.hc_ffn", + "deepseek-ai/DeepSeek-V4-Pro/hc_attn", + "deepseek-ai/DeepSeek-V4-Pro/hc_ffn", + "deepseek-ai/DeepSeek-V4-Pro/mtp.hc_attn", + "deepseek-ai/DeepSeek-V4-Pro/mtp.hc_ffn", + "sgl-project/DeepSeek-V4-Pro-FP8/hc_attn", + "sgl-project/DeepSeek-V4-Pro-FP8/hc_ffn", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.hc_attn", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.hc_ffn" + ] + }, + { + "type": "gemm", + "args": { + "m": 1136, + "n": 64, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.weights_proj", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.weights_proj", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.weights_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 1136, + "n": 256, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.compressor.wkv", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.compressor.wkv", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.compressor.wgate", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.compressor.wkv" + ] + }, + { + "type": "gemm", + "args": { + "m": 1136, + "n": 384, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.gate", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.gate", + "deepseek-ai/DeepSeek-V4-Pro/ffn.gate", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.gate", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.gate", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.gate" + ] + }, + { + "type": "gemm", + "args": { + "m": 1136, + "n": 512, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wkv", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wkv", + "deepseek-ai/DeepSeek-V4-Pro/attn.wkv", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wkv", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wkv", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wkv" + ] + }, + { + "type": "gemm", + "args": { + "m": 1136, + "n": 512, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.compressor.wkv+wgate", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.compressor.wkv+wgate", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.compressor.wkv+wgate" + ] + }, + { + "type": "gemm", + "args": { + "m": 1136, + "n": 1024, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.compressor.wkv", + "deepseek-ai/DeepSeek-V4-Pro/attn.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro/attn.compressor.wkv", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.compressor.wgate", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.compressor.wkv" + ] + }, + { + "type": "gemm", + "args": { + "m": 1136, + "n": 1536, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w3" + ] + }, + { + "type": "gemm", + "args": { + "m": 1136, + "n": 7168, + "k": 768, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w2" + ] + }, + { + "type": "gemm", + "args": { + "m": 1136, + "n": 7168, + "k": 4096, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 1136, + "n": 8192, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 1136, + "n": 16384, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 1152, + "n": 24, + "k": 28672, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/hc_attn", + "deepseek-ai/DeepSeek-V4-Pro-0813/hc_ffn", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.hc_attn", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.hc_ffn", + "deepseek-ai/DeepSeek-V4-Pro/hc_attn", + "deepseek-ai/DeepSeek-V4-Pro/hc_ffn", + "deepseek-ai/DeepSeek-V4-Pro/mtp.hc_attn", + "deepseek-ai/DeepSeek-V4-Pro/mtp.hc_ffn", + "sgl-project/DeepSeek-V4-Pro-FP8/hc_attn", + "sgl-project/DeepSeek-V4-Pro-FP8/hc_ffn", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.hc_attn", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.hc_ffn" + ] + }, + { + "type": "gemm", + "args": { + "m": 1152, + "n": 24, + "k": 40960, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 1152, + "n": 24, + "k": 49152, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 1152, + "n": 64, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.weights_proj", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.weights_proj", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.weights_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 1152, + "n": 64, + "k": 10240, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 1152, + "n": 64, + "k": 12288, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 1152, + "n": 120, + "k": 12288, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 1152, + "n": 192, + "k": 10240, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 1152, + "n": 256, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.compressor.wkv", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.compressor.wkv", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.compressor.wgate", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.compressor.wkv" + ] + }, + { + "type": "gemm", + "args": { + "m": 1152, + "n": 256, + "k": 10240, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 1152, + "n": 256, + "k": 12288, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 1152, + "n": 320, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 1152, + "n": 384, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.gate", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.gate", + "deepseek-ai/DeepSeek-V4-Pro/ffn.gate", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.gate", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.gate", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.gate" + ] + }, + { + "type": "gemm", + "args": { + "m": 1152, + "n": 384, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w3" + ] + }, + { + "type": "gemm", + "args": { + "m": 1152, + "n": 512, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wkv", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wkv", + "deepseek-ai/DeepSeek-V4-Pro/attn.wkv", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wkv", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wkv", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wkv" + ] + }, + { + "type": "gemm", + "args": { + "m": 1152, + "n": 512, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.compressor.wkv+wgate", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.compressor.wkv+wgate", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.compressor.wkv+wgate" + ] + }, + { + "type": "gemm", + "args": { + "m": 1152, + "n": 512, + "k": 10240, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 1152, + "n": 512, + "k": 10240, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 1152, + "n": 512, + "k": 12288, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 1152, + "n": 512, + "k": 12288, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 1152, + "n": 1024, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.compressor.wkv", + "deepseek-ai/DeepSeek-V4-Pro/attn.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro/attn.compressor.wkv", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.compressor.wgate", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.compressor.wkv" + ] + }, + { + "type": "gemm", + "args": { + "m": 1152, + "n": 1024, + "k": 10240, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 1152, + "n": 1024, + "k": 10240, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 1152, + "n": 1024, + "k": 12288, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 1152, + "n": 1536, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w3" + ] + }, + { + "type": "gemm", + "args": { + "m": 1152, + "n": 1536, + "k": 10240, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 1152, + "n": 1536, + "k": 12288, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 1152, + "n": 1792, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 1152, + "n": 2048, + "k": 12288, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 1152, + "n": 4096, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 1152, + "n": 5120, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 1152, + "n": 6144, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3" + ] + }, + { + "type": "gemm", + "args": { + "m": 1152, + "n": 7168, + "k": 192, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w2" + ] + }, + { + "type": "gemm", + "args": { + "m": 1152, + "n": 7168, + "k": 768, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w2" + ] + }, + { + "type": "gemm", + "args": { + "m": 1152, + "n": 7168, + "k": 896, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 1152, + "n": 7168, + "k": 1024, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 1152, + "n": 7168, + "k": 2048, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 1152, + "n": 7168, + "k": 3072, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w2" + ] + }, + { + "type": "gemm", + "args": { + "m": 1152, + "n": 7168, + "k": 4096, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 1152, + "n": 7168, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.e_proj", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.h_proj", + "deepseek-ai/DeepSeek-V4-Pro/mtp.e_proj", + "deepseek-ai/DeepSeek-V4-Pro/mtp.h_proj", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.e_proj", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.h_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 1152, + "n": 7168, + "k": 16384, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 1152, + "n": 8192, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 1152, + "n": 10240, + "k": 512, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 1152, + "n": 10240, + "k": 1024, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 1152, + "n": 10240, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 1152, + "n": 10240, + "k": 2048, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 1152, + "n": 10240, + "k": 16384, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 1152, + "n": 12288, + "k": 1024, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 1152, + "n": 12288, + "k": 2048, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 1152, + "n": 12288, + "k": 8192, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 1152, + "n": 12288, + "k": 16384, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 1152, + "n": 14336, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 1152, + "n": 16384, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 1152, + "n": 16384, + "k": 12288, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 1152, + "n": 65536, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 1152, + "n": 81920, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 1153, + "n": 24, + "k": 49152, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 1153, + "n": 64, + "k": 12288, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 1153, + "n": 120, + "k": 12288, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 1153, + "n": 256, + "k": 12288, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 1153, + "n": 512, + "k": 12288, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 1153, + "n": 512, + "k": 12288, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 1153, + "n": 1024, + "k": 12288, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 1153, + "n": 1536, + "k": 12288, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 1153, + "n": 5120, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 1153, + "n": 12288, + "k": 1024, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 1160, + "n": 24, + "k": 49152, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 1160, + "n": 64, + "k": 12288, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 1160, + "n": 120, + "k": 12288, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 1160, + "n": 256, + "k": 12288, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 1160, + "n": 512, + "k": 12288, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 1160, + "n": 512, + "k": 12288, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 1160, + "n": 1024, + "k": 12288, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 1160, + "n": 1536, + "k": 12288, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 1160, + "n": 5120, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 1160, + "n": 10240, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 1160, + "n": 12288, + "k": 1024, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 1160, + "n": 12288, + "k": 2048, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 1168, + "n": 24, + "k": 49152, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 1168, + "n": 64, + "k": 12288, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 1168, + "n": 120, + "k": 12288, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 1168, + "n": 256, + "k": 12288, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 1168, + "n": 512, + "k": 12288, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 1168, + "n": 512, + "k": 12288, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 1168, + "n": 1024, + "k": 12288, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 1168, + "n": 1536, + "k": 12288, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 1168, + "n": 1792, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 1168, + "n": 5120, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 1168, + "n": 7168, + "k": 896, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 1168, + "n": 12288, + "k": 1024, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 1216, + "n": 24, + "k": 28672, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/hc_attn", + "deepseek-ai/DeepSeek-V4-Pro-0813/hc_ffn", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.hc_attn", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.hc_ffn", + "deepseek-ai/DeepSeek-V4-Pro/hc_attn", + "deepseek-ai/DeepSeek-V4-Pro/hc_ffn", + "deepseek-ai/DeepSeek-V4-Pro/mtp.hc_attn", + "deepseek-ai/DeepSeek-V4-Pro/mtp.hc_ffn", + "sgl-project/DeepSeek-V4-Pro-FP8/hc_attn", + "sgl-project/DeepSeek-V4-Pro-FP8/hc_ffn", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.hc_attn", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.hc_ffn" + ] + }, + { + "type": "gemm", + "args": { + "m": 1216, + "n": 24, + "k": 49152, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 1216, + "n": 64, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.weights_proj", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.weights_proj", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.weights_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 1216, + "n": 64, + "k": 12288, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 1216, + "n": 120, + "k": 12288, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 1216, + "n": 256, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.compressor.wkv", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.compressor.wkv", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.compressor.wgate", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.compressor.wkv" + ] + }, + { + "type": "gemm", + "args": { + "m": 1216, + "n": 256, + "k": 12288, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 1216, + "n": 384, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.gate", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.gate", + "deepseek-ai/DeepSeek-V4-Pro/ffn.gate", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.gate", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.gate", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.gate" + ] + }, + { + "type": "gemm", + "args": { + "m": 1216, + "n": 512, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wkv", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wkv", + "deepseek-ai/DeepSeek-V4-Pro/attn.wkv", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wkv", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wkv", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wkv" + ] + }, + { + "type": "gemm", + "args": { + "m": 1216, + "n": 512, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.compressor.wkv+wgate", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.compressor.wkv+wgate", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.compressor.wkv+wgate" + ] + }, + { + "type": "gemm", + "args": { + "m": 1216, + "n": 512, + "k": 12288, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 1216, + "n": 512, + "k": 12288, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 1216, + "n": 1024, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.compressor.wkv", + "deepseek-ai/DeepSeek-V4-Pro/attn.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro/attn.compressor.wkv", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.compressor.wgate", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.compressor.wkv" + ] + }, + { + "type": "gemm", + "args": { + "m": 1216, + "n": 1024, + "k": 10240, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 1216, + "n": 1024, + "k": 12288, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 1216, + "n": 1536, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w3" + ] + }, + { + "type": "gemm", + "args": { + "m": 1216, + "n": 1536, + "k": 12288, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 1216, + "n": 2048, + "k": 12288, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 1216, + "n": 5120, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 1216, + "n": 6144, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3" + ] + }, + { + "type": "gemm", + "args": { + "m": 1216, + "n": 7168, + "k": 768, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w2" + ] + }, + { + "type": "gemm", + "args": { + "m": 1216, + "n": 7168, + "k": 3072, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w2" + ] + }, + { + "type": "gemm", + "args": { + "m": 1216, + "n": 7168, + "k": 4096, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 1216, + "n": 7168, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.e_proj", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.h_proj", + "deepseek-ai/DeepSeek-V4-Pro/mtp.e_proj", + "deepseek-ai/DeepSeek-V4-Pro/mtp.h_proj", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.e_proj", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.h_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 1216, + "n": 8192, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 1216, + "n": 10240, + "k": 512, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 1216, + "n": 10240, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 1216, + "n": 10240, + "k": 16384, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 1216, + "n": 12288, + "k": 1024, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 1216, + "n": 12288, + "k": 2048, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 1216, + "n": 12288, + "k": 8192, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 1216, + "n": 12288, + "k": 16384, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 1216, + "n": 14336, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 1216, + "n": 16384, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 1216, + "n": 16384, + "k": 12288, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 1216, + "n": 81920, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 1217, + "n": 1024, + "k": 10240, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 1217, + "n": 10240, + "k": 512, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 1217, + "n": 10240, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 1217, + "n": 12288, + "k": 2048, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 1264, + "n": 4096, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 1264, + "n": 7168, + "k": 1024, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 1273, + "n": 4096, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 1273, + "n": 7168, + "k": 1024, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 1279, + "n": 320, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 1280, + "n": 24, + "k": 28672, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/hc_attn", + "deepseek-ai/DeepSeek-V4-Pro-0813/hc_ffn", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.hc_attn", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.hc_ffn", + "deepseek-ai/DeepSeek-V4-Pro/hc_attn", + "deepseek-ai/DeepSeek-V4-Pro/hc_ffn", + "deepseek-ai/DeepSeek-V4-Pro/mtp.hc_attn", + "deepseek-ai/DeepSeek-V4-Pro/mtp.hc_ffn", + "sgl-project/DeepSeek-V4-Pro-FP8/hc_attn", + "sgl-project/DeepSeek-V4-Pro-FP8/hc_ffn", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.hc_attn", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.hc_ffn" + ] + }, + { + "type": "gemm", + "args": { + "m": 1280, + "n": 24, + "k": 49152, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 1280, + "n": 64, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.weights_proj", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.weights_proj", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.weights_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 1280, + "n": 64, + "k": 12288, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 1280, + "n": 120, + "k": 12288, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 1280, + "n": 256, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.compressor.wkv", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.compressor.wkv", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.compressor.wgate", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.compressor.wkv" + ] + }, + { + "type": "gemm", + "args": { + "m": 1280, + "n": 256, + "k": 12288, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 1280, + "n": 320, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 1280, + "n": 384, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.gate", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.gate", + "deepseek-ai/DeepSeek-V4-Pro/ffn.gate", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.gate", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.gate", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.gate" + ] + }, + { + "type": "gemm", + "args": { + "m": 1280, + "n": 384, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w3" + ] + }, + { + "type": "gemm", + "args": { + "m": 1280, + "n": 512, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wkv", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wkv", + "deepseek-ai/DeepSeek-V4-Pro/attn.wkv", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wkv", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wkv", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wkv" + ] + }, + { + "type": "gemm", + "args": { + "m": 1280, + "n": 512, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.compressor.wkv+wgate", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.compressor.wkv+wgate", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.compressor.wkv+wgate" + ] + }, + { + "type": "gemm", + "args": { + "m": 1280, + "n": 512, + "k": 12288, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 1280, + "n": 512, + "k": 12288, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 1280, + "n": 1024, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.compressor.wkv", + "deepseek-ai/DeepSeek-V4-Pro/attn.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro/attn.compressor.wkv", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.compressor.wgate", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.compressor.wkv" + ] + }, + { + "type": "gemm", + "args": { + "m": 1280, + "n": 1024, + "k": 10240, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 1280, + "n": 1024, + "k": 12288, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 1280, + "n": 1536, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w3" + ] + }, + { + "type": "gemm", + "args": { + "m": 1280, + "n": 1536, + "k": 12288, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 1280, + "n": 1792, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 1280, + "n": 2048, + "k": 12288, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 1280, + "n": 4096, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 1280, + "n": 5120, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 1280, + "n": 6144, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3" + ] + }, + { + "type": "gemm", + "args": { + "m": 1280, + "n": 7168, + "k": 192, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w2" + ] + }, + { + "type": "gemm", + "args": { + "m": 1280, + "n": 7168, + "k": 768, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w2" + ] + }, + { + "type": "gemm", + "args": { + "m": 1280, + "n": 7168, + "k": 896, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 1280, + "n": 7168, + "k": 1024, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 1280, + "n": 7168, + "k": 2048, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 1280, + "n": 7168, + "k": 3072, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w2" + ] + }, + { + "type": "gemm", + "args": { + "m": 1280, + "n": 7168, + "k": 4096, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 1280, + "n": 7168, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.e_proj", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.h_proj", + "deepseek-ai/DeepSeek-V4-Pro/mtp.e_proj", + "deepseek-ai/DeepSeek-V4-Pro/mtp.h_proj", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.e_proj", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.h_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 1280, + "n": 7168, + "k": 16384, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 1280, + "n": 8192, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 1280, + "n": 10240, + "k": 512, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 1280, + "n": 10240, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 1280, + "n": 10240, + "k": 16384, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 1280, + "n": 12288, + "k": 1024, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 1280, + "n": 12288, + "k": 2048, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 1280, + "n": 12288, + "k": 8192, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 1280, + "n": 12288, + "k": 16384, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 1280, + "n": 14336, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 1280, + "n": 16384, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 1280, + "n": 16384, + "k": 12288, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 1280, + "n": 65536, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 1280, + "n": 81920, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 1281, + "n": 24, + "k": 40960, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 1281, + "n": 64, + "k": 10240, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 1281, + "n": 192, + "k": 10240, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 1281, + "n": 256, + "k": 10240, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 1281, + "n": 512, + "k": 10240, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 1281, + "n": 512, + "k": 10240, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 1281, + "n": 1024, + "k": 10240, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 1281, + "n": 1536, + "k": 10240, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 1281, + "n": 10240, + "k": 1024, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 1281, + "n": 10240, + "k": 2048, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 1283, + "n": 896, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 1283, + "n": 7168, + "k": 448, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 1288, + "n": 24, + "k": 40960, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 1288, + "n": 64, + "k": 10240, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 1288, + "n": 192, + "k": 10240, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 1288, + "n": 256, + "k": 10240, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 1288, + "n": 320, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 1288, + "n": 512, + "k": 10240, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 1288, + "n": 512, + "k": 10240, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 1288, + "n": 896, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 1288, + "n": 1024, + "k": 10240, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 1288, + "n": 1536, + "k": 10240, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 1288, + "n": 4096, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 1288, + "n": 7168, + "k": 448, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 1288, + "n": 7168, + "k": 1024, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 1288, + "n": 10240, + "k": 1024, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 1288, + "n": 10240, + "k": 2048, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 1296, + "n": 24, + "k": 40960, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 1296, + "n": 64, + "k": 10240, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 1296, + "n": 192, + "k": 10240, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 1296, + "n": 256, + "k": 10240, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 1296, + "n": 320, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 1296, + "n": 512, + "k": 10240, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 1296, + "n": 512, + "k": 10240, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 1296, + "n": 896, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 1296, + "n": 1024, + "k": 10240, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 1296, + "n": 1536, + "k": 10240, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 1296, + "n": 7168, + "k": 448, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 1296, + "n": 10240, + "k": 1024, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 1308, + "n": 7168, + "k": 2048, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 1312, + "n": 320, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 1312, + "n": 896, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 1312, + "n": 4096, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 1312, + "n": 7168, + "k": 448, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 1312, + "n": 7168, + "k": 1024, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 1330, + "n": 768, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w3" + ] + }, + { + "type": "gemm", + "args": { + "m": 1330, + "n": 7168, + "k": 384, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w2" + ] + }, + { + "type": "gemm", + "args": { + "m": 1344, + "n": 24, + "k": 40960, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 1344, + "n": 64, + "k": 10240, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 1344, + "n": 192, + "k": 10240, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 1344, + "n": 256, + "k": 10240, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 1344, + "n": 320, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 1344, + "n": 512, + "k": 10240, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 1344, + "n": 512, + "k": 10240, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 1344, + "n": 1024, + "k": 10240, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 1344, + "n": 1536, + "k": 10240, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 1344, + "n": 1792, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 1344, + "n": 4096, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 1344, + "n": 7168, + "k": 896, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 1344, + "n": 7168, + "k": 1024, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 1344, + "n": 7168, + "k": 2048, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 1344, + "n": 10240, + "k": 1024, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 1344, + "n": 10240, + "k": 2048, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 1345, + "n": 1792, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 1345, + "n": 7168, + "k": 896, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 1352, + "n": 768, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w3" + ] + }, + { + "type": "gemm", + "args": { + "m": 1352, + "n": 1024, + "k": 10240, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 1352, + "n": 1792, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 1352, + "n": 7168, + "k": 384, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w2" + ] + }, + { + "type": "gemm", + "args": { + "m": 1352, + "n": 7168, + "k": 896, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 1352, + "n": 7168, + "k": 2048, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 1352, + "n": 10240, + "k": 512, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 1360, + "n": 768, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w3" + ] + }, + { + "type": "gemm", + "args": { + "m": 1360, + "n": 7168, + "k": 384, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w2" + ] + }, + { + "type": "gemm", + "args": { + "m": 1360, + "n": 7168, + "k": 2048, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 1369, + "n": 384, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w3" + ] + }, + { + "type": "gemm", + "args": { + "m": 1369, + "n": 7168, + "k": 192, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w2" + ] + }, + { + "type": "gemm", + "args": { + "m": 1387, + "n": 24, + "k": 28672, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/hc_attn", + "deepseek-ai/DeepSeek-V4-Pro-0813/hc_ffn", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.hc_attn", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.hc_ffn", + "deepseek-ai/DeepSeek-V4-Pro/hc_attn", + "deepseek-ai/DeepSeek-V4-Pro/hc_ffn", + "deepseek-ai/DeepSeek-V4-Pro/mtp.hc_attn", + "deepseek-ai/DeepSeek-V4-Pro/mtp.hc_ffn", + "sgl-project/DeepSeek-V4-Pro-FP8/hc_attn", + "sgl-project/DeepSeek-V4-Pro-FP8/hc_ffn", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.hc_attn", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.hc_ffn" + ] + }, + { + "type": "gemm", + "args": { + "m": 1387, + "n": 64, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.weights_proj", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.weights_proj", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.weights_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 1387, + "n": 256, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.compressor.wkv", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.compressor.wkv", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.compressor.wgate", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.compressor.wkv" + ] + }, + { + "type": "gemm", + "args": { + "m": 1387, + "n": 512, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wkv", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wkv", + "deepseek-ai/DeepSeek-V4-Pro/attn.wkv", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wkv", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wkv", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wkv" + ] + }, + { + "type": "gemm", + "args": { + "m": 1387, + "n": 512, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.compressor.wkv+wgate", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.compressor.wkv+wgate", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.compressor.wkv+wgate" + ] + }, + { + "type": "gemm", + "args": { + "m": 1387, + "n": 1024, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.compressor.wkv", + "deepseek-ai/DeepSeek-V4-Pro/attn.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro/attn.compressor.wkv", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.compressor.wgate", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.compressor.wkv" + ] + }, + { + "type": "gemm", + "args": { + "m": 1387, + "n": 1536, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w3" + ] + }, + { + "type": "gemm", + "args": { + "m": 1387, + "n": 8192, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 1391, + "n": 384, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.gate", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.gate", + "deepseek-ai/DeepSeek-V4-Pro/ffn.gate", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.gate", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.gate", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.gate" + ] + }, + { + "type": "gemm", + "args": { + "m": 1397, + "n": 1536, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w3" + ] + }, + { + "type": "gemm", + "args": { + "m": 1397, + "n": 7168, + "k": 768, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w2" + ] + }, + { + "type": "gemm", + "args": { + "m": 1397, + "n": 7168, + "k": 4096, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 1397, + "n": 16384, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 1400, + "n": 24, + "k": 28672, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/hc_attn", + "deepseek-ai/DeepSeek-V4-Pro-0813/hc_ffn", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.hc_attn", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.hc_ffn", + "deepseek-ai/DeepSeek-V4-Pro/hc_attn", + "deepseek-ai/DeepSeek-V4-Pro/hc_ffn", + "deepseek-ai/DeepSeek-V4-Pro/mtp.hc_attn", + "deepseek-ai/DeepSeek-V4-Pro/mtp.hc_ffn", + "sgl-project/DeepSeek-V4-Pro-FP8/hc_attn", + "sgl-project/DeepSeek-V4-Pro-FP8/hc_ffn", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.hc_attn", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.hc_ffn" + ] + }, + { + "type": "gemm", + "args": { + "m": 1400, + "n": 64, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.weights_proj", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.weights_proj", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.weights_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 1400, + "n": 256, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.compressor.wkv", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.compressor.wkv", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.compressor.wgate", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.compressor.wkv" + ] + }, + { + "type": "gemm", + "args": { + "m": 1400, + "n": 384, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.gate", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.gate", + "deepseek-ai/DeepSeek-V4-Pro/ffn.gate", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.gate", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.gate", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.gate" + ] + }, + { + "type": "gemm", + "args": { + "m": 1400, + "n": 512, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wkv", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wkv", + "deepseek-ai/DeepSeek-V4-Pro/attn.wkv", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wkv", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wkv", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wkv" + ] + }, + { + "type": "gemm", + "args": { + "m": 1400, + "n": 512, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.compressor.wkv+wgate", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.compressor.wkv+wgate", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.compressor.wkv+wgate" + ] + }, + { + "type": "gemm", + "args": { + "m": 1400, + "n": 1024, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.compressor.wkv", + "deepseek-ai/DeepSeek-V4-Pro/attn.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro/attn.compressor.wkv", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.compressor.wgate", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.compressor.wkv" + ] + }, + { + "type": "gemm", + "args": { + "m": 1400, + "n": 1536, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w3" + ] + }, + { + "type": "gemm", + "args": { + "m": 1400, + "n": 7168, + "k": 768, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w2" + ] + }, + { + "type": "gemm", + "args": { + "m": 1400, + "n": 7168, + "k": 4096, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 1400, + "n": 8192, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 1400, + "n": 16384, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 1408, + "n": 24, + "k": 28672, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/hc_attn", + "deepseek-ai/DeepSeek-V4-Pro-0813/hc_ffn", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.hc_attn", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.hc_ffn", + "deepseek-ai/DeepSeek-V4-Pro/hc_attn", + "deepseek-ai/DeepSeek-V4-Pro/hc_ffn", + "deepseek-ai/DeepSeek-V4-Pro/mtp.hc_attn", + "deepseek-ai/DeepSeek-V4-Pro/mtp.hc_ffn", + "sgl-project/DeepSeek-V4-Pro-FP8/hc_attn", + "sgl-project/DeepSeek-V4-Pro-FP8/hc_ffn", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.hc_attn", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.hc_ffn" + ] + }, + { + "type": "gemm", + "args": { + "m": 1408, + "n": 24, + "k": 40960, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 1408, + "n": 64, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.weights_proj", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.weights_proj", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.weights_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 1408, + "n": 64, + "k": 10240, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 1408, + "n": 192, + "k": 10240, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 1408, + "n": 256, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.compressor.wkv", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.compressor.wkv", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.compressor.wgate", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.compressor.wkv" + ] + }, + { + "type": "gemm", + "args": { + "m": 1408, + "n": 256, + "k": 10240, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 1408, + "n": 320, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 1408, + "n": 384, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.gate", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.gate", + "deepseek-ai/DeepSeek-V4-Pro/ffn.gate", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.gate", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.gate", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.gate" + ] + }, + { + "type": "gemm", + "args": { + "m": 1408, + "n": 384, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w3" + ] + }, + { + "type": "gemm", + "args": { + "m": 1408, + "n": 512, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wkv", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wkv", + "deepseek-ai/DeepSeek-V4-Pro/attn.wkv", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wkv", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wkv", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wkv" + ] + }, + { + "type": "gemm", + "args": { + "m": 1408, + "n": 512, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.compressor.wkv+wgate", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.compressor.wkv+wgate", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.compressor.wkv+wgate" + ] + }, + { + "type": "gemm", + "args": { + "m": 1408, + "n": 512, + "k": 10240, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 1408, + "n": 512, + "k": 10240, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 1408, + "n": 768, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w3" + ] + }, + { + "type": "gemm", + "args": { + "m": 1408, + "n": 1024, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.compressor.wkv", + "deepseek-ai/DeepSeek-V4-Pro/attn.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro/attn.compressor.wkv", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.compressor.wgate", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.compressor.wkv" + ] + }, + { + "type": "gemm", + "args": { + "m": 1408, + "n": 1024, + "k": 10240, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 1408, + "n": 1024, + "k": 10240, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 1408, + "n": 1536, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w3" + ] + }, + { + "type": "gemm", + "args": { + "m": 1408, + "n": 1536, + "k": 10240, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 1408, + "n": 1792, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 1408, + "n": 4096, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 1408, + "n": 6144, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3" + ] + }, + { + "type": "gemm", + "args": { + "m": 1408, + "n": 7168, + "k": 192, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w2" + ] + }, + { + "type": "gemm", + "args": { + "m": 1408, + "n": 7168, + "k": 384, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w2" + ] + }, + { + "type": "gemm", + "args": { + "m": 1408, + "n": 7168, + "k": 768, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w2" + ] + }, + { + "type": "gemm", + "args": { + "m": 1408, + "n": 7168, + "k": 896, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 1408, + "n": 7168, + "k": 1024, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 1408, + "n": 7168, + "k": 2048, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 1408, + "n": 7168, + "k": 3072, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w2" + ] + }, + { + "type": "gemm", + "args": { + "m": 1408, + "n": 7168, + "k": 4096, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 1408, + "n": 7168, + "k": 16384, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 1408, + "n": 8192, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 1408, + "n": 10240, + "k": 512, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 1408, + "n": 10240, + "k": 1024, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 1408, + "n": 10240, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 1408, + "n": 10240, + "k": 2048, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 1408, + "n": 12288, + "k": 2048, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 1408, + "n": 16384, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 1408, + "n": 65536, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 1416, + "n": 384, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w3" + ] + }, + { + "type": "gemm", + "args": { + "m": 1416, + "n": 7168, + "k": 192, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w2" + ] + }, + { + "type": "gemm", + "args": { + "m": 1424, + "n": 24, + "k": 28672, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/hc_attn", + "deepseek-ai/DeepSeek-V4-Pro-0813/hc_ffn", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.hc_attn", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.hc_ffn", + "deepseek-ai/DeepSeek-V4-Pro/hc_attn", + "deepseek-ai/DeepSeek-V4-Pro/hc_ffn", + "deepseek-ai/DeepSeek-V4-Pro/mtp.hc_attn", + "deepseek-ai/DeepSeek-V4-Pro/mtp.hc_ffn", + "sgl-project/DeepSeek-V4-Pro-FP8/hc_attn", + "sgl-project/DeepSeek-V4-Pro-FP8/hc_ffn", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.hc_attn", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.hc_ffn" + ] + }, + { + "type": "gemm", + "args": { + "m": 1424, + "n": 64, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.weights_proj", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.weights_proj", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.weights_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 1424, + "n": 256, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.compressor.wkv", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.compressor.wkv", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.compressor.wgate", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.compressor.wkv" + ] + }, + { + "type": "gemm", + "args": { + "m": 1424, + "n": 384, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.gate", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.gate", + "deepseek-ai/DeepSeek-V4-Pro/ffn.gate", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.gate", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.gate", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.gate" + ] + }, + { + "type": "gemm", + "args": { + "m": 1424, + "n": 512, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wkv", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wkv", + "deepseek-ai/DeepSeek-V4-Pro/attn.wkv", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wkv", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wkv", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wkv" + ] + }, + { + "type": "gemm", + "args": { + "m": 1424, + "n": 512, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.compressor.wkv+wgate", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.compressor.wkv+wgate", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.compressor.wkv+wgate" + ] + }, + { + "type": "gemm", + "args": { + "m": 1424, + "n": 1024, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.compressor.wkv", + "deepseek-ai/DeepSeek-V4-Pro/attn.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro/attn.compressor.wkv", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.compressor.wgate", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.compressor.wkv" + ] + }, + { + "type": "gemm", + "args": { + "m": 1424, + "n": 1536, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w3" + ] + }, + { + "type": "gemm", + "args": { + "m": 1424, + "n": 7168, + "k": 768, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w2" + ] + }, + { + "type": "gemm", + "args": { + "m": 1424, + "n": 7168, + "k": 4096, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 1424, + "n": 8192, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 1424, + "n": 16384, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 1440, + "n": 24, + "k": 28672, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/hc_attn", + "deepseek-ai/DeepSeek-V4-Pro-0813/hc_ffn", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.hc_attn", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.hc_ffn", + "deepseek-ai/DeepSeek-V4-Pro/hc_attn", + "deepseek-ai/DeepSeek-V4-Pro/hc_ffn", + "deepseek-ai/DeepSeek-V4-Pro/mtp.hc_attn", + "deepseek-ai/DeepSeek-V4-Pro/mtp.hc_ffn", + "sgl-project/DeepSeek-V4-Pro-FP8/hc_attn", + "sgl-project/DeepSeek-V4-Pro-FP8/hc_ffn", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.hc_attn", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.hc_ffn" + ] + }, + { + "type": "gemm", + "args": { + "m": 1440, + "n": 64, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.weights_proj", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.weights_proj", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.weights_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 1440, + "n": 256, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.compressor.wkv", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.compressor.wkv", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.compressor.wgate", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.compressor.wkv" + ] + }, + { + "type": "gemm", + "args": { + "m": 1440, + "n": 384, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.gate", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.gate", + "deepseek-ai/DeepSeek-V4-Pro/ffn.gate", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.gate", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.gate", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.gate" + ] + }, + { + "type": "gemm", + "args": { + "m": 1440, + "n": 512, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wkv", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wkv", + "deepseek-ai/DeepSeek-V4-Pro/attn.wkv", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wkv", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wkv", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wkv" + ] + }, + { + "type": "gemm", + "args": { + "m": 1440, + "n": 512, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.compressor.wkv+wgate", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.compressor.wkv+wgate", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.compressor.wkv+wgate" + ] + }, + { + "type": "gemm", + "args": { + "m": 1440, + "n": 1024, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.compressor.wkv", + "deepseek-ai/DeepSeek-V4-Pro/attn.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro/attn.compressor.wkv", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.compressor.wgate", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.compressor.wkv" + ] + }, + { + "type": "gemm", + "args": { + "m": 1440, + "n": 1536, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w3" + ] + }, + { + "type": "gemm", + "args": { + "m": 1440, + "n": 7168, + "k": 768, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w2" + ] + }, + { + "type": "gemm", + "args": { + "m": 1440, + "n": 7168, + "k": 4096, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 1440, + "n": 8192, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 1440, + "n": 16384, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 1472, + "n": 24, + "k": 28672, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/hc_attn", + "deepseek-ai/DeepSeek-V4-Pro-0813/hc_ffn", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.hc_attn", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.hc_ffn", + "deepseek-ai/DeepSeek-V4-Pro/hc_attn", + "deepseek-ai/DeepSeek-V4-Pro/hc_ffn", + "deepseek-ai/DeepSeek-V4-Pro/mtp.hc_attn", + "deepseek-ai/DeepSeek-V4-Pro/mtp.hc_ffn", + "sgl-project/DeepSeek-V4-Pro-FP8/hc_attn", + "sgl-project/DeepSeek-V4-Pro-FP8/hc_ffn", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.hc_attn", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.hc_ffn" + ] + }, + { + "type": "gemm", + "args": { + "m": 1472, + "n": 24, + "k": 49152, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 1472, + "n": 64, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.weights_proj", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.weights_proj", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.weights_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 1472, + "n": 64, + "k": 12288, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 1472, + "n": 120, + "k": 12288, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 1472, + "n": 256, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.compressor.wkv", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.compressor.wkv", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.compressor.wgate", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.compressor.wkv" + ] + }, + { + "type": "gemm", + "args": { + "m": 1472, + "n": 256, + "k": 12288, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 1472, + "n": 384, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.gate", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.gate", + "deepseek-ai/DeepSeek-V4-Pro/ffn.gate", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.gate", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.gate", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.gate" + ] + }, + { + "type": "gemm", + "args": { + "m": 1472, + "n": 384, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w3" + ] + }, + { + "type": "gemm", + "args": { + "m": 1472, + "n": 512, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wkv", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wkv", + "deepseek-ai/DeepSeek-V4-Pro/attn.wkv", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wkv", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wkv", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wkv" + ] + }, + { + "type": "gemm", + "args": { + "m": 1472, + "n": 512, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.compressor.wkv+wgate", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.compressor.wkv+wgate", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.compressor.wkv+wgate" + ] + }, + { + "type": "gemm", + "args": { + "m": 1472, + "n": 512, + "k": 12288, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 1472, + "n": 512, + "k": 12288, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 1472, + "n": 768, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w3" + ] + }, + { + "type": "gemm", + "args": { + "m": 1472, + "n": 1024, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.compressor.wkv", + "deepseek-ai/DeepSeek-V4-Pro/attn.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro/attn.compressor.wkv", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.compressor.wgate", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.compressor.wkv" + ] + }, + { + "type": "gemm", + "args": { + "m": 1472, + "n": 1024, + "k": 12288, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 1472, + "n": 1536, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w3" + ] + }, + { + "type": "gemm", + "args": { + "m": 1472, + "n": 1536, + "k": 12288, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 1472, + "n": 2048, + "k": 12288, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 1472, + "n": 5120, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 1472, + "n": 6144, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3" + ] + }, + { + "type": "gemm", + "args": { + "m": 1472, + "n": 7168, + "k": 192, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w2" + ] + }, + { + "type": "gemm", + "args": { + "m": 1472, + "n": 7168, + "k": 384, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w2" + ] + }, + { + "type": "gemm", + "args": { + "m": 1472, + "n": 7168, + "k": 768, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w2" + ] + }, + { + "type": "gemm", + "args": { + "m": 1472, + "n": 7168, + "k": 3072, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w2" + ] + }, + { + "type": "gemm", + "args": { + "m": 1472, + "n": 7168, + "k": 4096, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 1472, + "n": 7168, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.e_proj", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.h_proj", + "deepseek-ai/DeepSeek-V4-Pro/mtp.e_proj", + "deepseek-ai/DeepSeek-V4-Pro/mtp.h_proj", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.e_proj", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.h_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 1472, + "n": 7168, + "k": 16384, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 1472, + "n": 8192, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 1472, + "n": 10240, + "k": 16384, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 1472, + "n": 12288, + "k": 1024, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 1472, + "n": 12288, + "k": 8192, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 1472, + "n": 12288, + "k": 16384, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 1472, + "n": 14336, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 1472, + "n": 16384, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 1472, + "n": 16384, + "k": 12288, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 1472, + "n": 65536, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 1472, + "n": 81920, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 1473, + "n": 24, + "k": 49152, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 1473, + "n": 64, + "k": 12288, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 1473, + "n": 120, + "k": 12288, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 1473, + "n": 256, + "k": 12288, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 1473, + "n": 512, + "k": 12288, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 1473, + "n": 512, + "k": 12288, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 1473, + "n": 1024, + "k": 12288, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 1473, + "n": 1536, + "k": 12288, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 1473, + "n": 5120, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 1473, + "n": 6144, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3" + ] + }, + { + "type": "gemm", + "args": { + "m": 1473, + "n": 7168, + "k": 3072, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w2" + ] + }, + { + "type": "gemm", + "args": { + "m": 1473, + "n": 7168, + "k": 16384, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 1473, + "n": 12288, + "k": 1024, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 1473, + "n": 12288, + "k": 16384, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 1473, + "n": 65536, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 1473, + "n": 81920, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 1480, + "n": 24, + "k": 49152, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 1480, + "n": 64, + "k": 12288, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 1480, + "n": 120, + "k": 12288, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 1480, + "n": 256, + "k": 12288, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 1480, + "n": 512, + "k": 12288, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 1480, + "n": 512, + "k": 12288, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 1480, + "n": 1024, + "k": 12288, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 1480, + "n": 1536, + "k": 12288, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 1480, + "n": 5120, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 1480, + "n": 12288, + "k": 1024, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 1488, + "n": 24, + "k": 49152, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 1488, + "n": 64, + "k": 12288, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 1488, + "n": 120, + "k": 12288, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 1488, + "n": 256, + "k": 12288, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 1488, + "n": 384, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w3" + ] + }, + { + "type": "gemm", + "args": { + "m": 1488, + "n": 512, + "k": 12288, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 1488, + "n": 512, + "k": 12288, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 1488, + "n": 1024, + "k": 12288, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 1488, + "n": 1536, + "k": 12288, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 1488, + "n": 5120, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 1488, + "n": 7168, + "k": 192, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w2" + ] + }, + { + "type": "gemm", + "args": { + "m": 1488, + "n": 12288, + "k": 1024, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 1536, + "n": 24, + "k": 28672, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/hc_attn", + "deepseek-ai/DeepSeek-V4-Pro-0813/hc_ffn", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.hc_attn", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.hc_ffn", + "deepseek-ai/DeepSeek-V4-Pro/hc_attn", + "deepseek-ai/DeepSeek-V4-Pro/hc_ffn", + "deepseek-ai/DeepSeek-V4-Pro/mtp.hc_attn", + "deepseek-ai/DeepSeek-V4-Pro/mtp.hc_ffn", + "sgl-project/DeepSeek-V4-Pro-FP8/hc_attn", + "sgl-project/DeepSeek-V4-Pro-FP8/hc_ffn", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.hc_attn", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.hc_ffn" + ] + }, + { + "type": "gemm", + "args": { + "m": 1536, + "n": 24, + "k": 40960, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 1536, + "n": 24, + "k": 49152, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 1536, + "n": 64, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.weights_proj", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.weights_proj", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.weights_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 1536, + "n": 64, + "k": 10240, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 1536, + "n": 64, + "k": 12288, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 1536, + "n": 120, + "k": 12288, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 1536, + "n": 192, + "k": 10240, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 1536, + "n": 256, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.compressor.wkv", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.compressor.wkv", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.compressor.wgate", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.compressor.wkv" + ] + }, + { + "type": "gemm", + "args": { + "m": 1536, + "n": 256, + "k": 10240, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 1536, + "n": 256, + "k": 12288, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 1536, + "n": 384, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.gate", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.gate", + "deepseek-ai/DeepSeek-V4-Pro/ffn.gate", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.gate", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.gate", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.gate" + ] + }, + { + "type": "gemm", + "args": { + "m": 1536, + "n": 384, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w3" + ] + }, + { + "type": "gemm", + "args": { + "m": 1536, + "n": 512, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wkv", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wkv", + "deepseek-ai/DeepSeek-V4-Pro/attn.wkv", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wkv", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wkv", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wkv" + ] + }, + { + "type": "gemm", + "args": { + "m": 1536, + "n": 512, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.compressor.wkv+wgate", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.compressor.wkv+wgate", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.compressor.wkv+wgate" + ] + }, + { + "type": "gemm", + "args": { + "m": 1536, + "n": 512, + "k": 10240, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 1536, + "n": 512, + "k": 10240, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 1536, + "n": 512, + "k": 12288, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 1536, + "n": 512, + "k": 12288, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 1536, + "n": 768, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w3" + ] + }, + { + "type": "gemm", + "args": { + "m": 1536, + "n": 896, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 1536, + "n": 1024, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.compressor.wkv", + "deepseek-ai/DeepSeek-V4-Pro/attn.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro/attn.compressor.wkv", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.compressor.wgate", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.compressor.wkv" + ] + }, + { + "type": "gemm", + "args": { + "m": 1536, + "n": 1024, + "k": 10240, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 1536, + "n": 1024, + "k": 10240, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 1536, + "n": 1024, + "k": 12288, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 1536, + "n": 1536, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w3" + ] + }, + { + "type": "gemm", + "args": { + "m": 1536, + "n": 1536, + "k": 10240, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 1536, + "n": 1536, + "k": 12288, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 1536, + "n": 1792, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 1536, + "n": 2048, + "k": 12288, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 1536, + "n": 5120, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 1536, + "n": 6144, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3" + ] + }, + { + "type": "gemm", + "args": { + "m": 1536, + "n": 7168, + "k": 192, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w2" + ] + }, + { + "type": "gemm", + "args": { + "m": 1536, + "n": 7168, + "k": 384, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w2" + ] + }, + { + "type": "gemm", + "args": { + "m": 1536, + "n": 7168, + "k": 448, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 1536, + "n": 7168, + "k": 768, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w2" + ] + }, + { + "type": "gemm", + "args": { + "m": 1536, + "n": 7168, + "k": 896, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 1536, + "n": 7168, + "k": 2048, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 1536, + "n": 7168, + "k": 3072, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w2" + ] + }, + { + "type": "gemm", + "args": { + "m": 1536, + "n": 7168, + "k": 4096, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 1536, + "n": 7168, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.e_proj", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.h_proj", + "deepseek-ai/DeepSeek-V4-Pro/mtp.e_proj", + "deepseek-ai/DeepSeek-V4-Pro/mtp.h_proj", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.e_proj", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.h_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 1536, + "n": 7168, + "k": 16384, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 1536, + "n": 8192, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 1536, + "n": 10240, + "k": 512, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 1536, + "n": 10240, + "k": 1024, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 1536, + "n": 10240, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 1536, + "n": 10240, + "k": 2048, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 1536, + "n": 10240, + "k": 16384, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 1536, + "n": 12288, + "k": 1024, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 1536, + "n": 12288, + "k": 2048, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 1536, + "n": 12288, + "k": 8192, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 1536, + "n": 12288, + "k": 16384, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 1536, + "n": 14336, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 1536, + "n": 16384, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 1536, + "n": 16384, + "k": 12288, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 1536, + "n": 65536, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 1536, + "n": 81920, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 1537, + "n": 10240, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 1537, + "n": 12288, + "k": 2048, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 1538, + "n": 1024, + "k": 10240, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 1538, + "n": 10240, + "k": 512, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 1568, + "n": 24, + "k": 40960, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 1568, + "n": 64, + "k": 10240, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 1568, + "n": 192, + "k": 10240, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 1568, + "n": 256, + "k": 10240, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 1568, + "n": 512, + "k": 10240, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 1568, + "n": 512, + "k": 10240, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 1568, + "n": 1024, + "k": 10240, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 1568, + "n": 1024, + "k": 10240, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 1568, + "n": 1536, + "k": 10240, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 1568, + "n": 10240, + "k": 512, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 1568, + "n": 10240, + "k": 2048, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 1581, + "n": 4096, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 1581, + "n": 7168, + "k": 1024, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 1592, + "n": 4096, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 1592, + "n": 7168, + "k": 1024, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 1600, + "n": 320, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 1600, + "n": 1024, + "k": 10240, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 1600, + "n": 4096, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 1600, + "n": 7168, + "k": 1024, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 1600, + "n": 10240, + "k": 512, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 1600, + "n": 10240, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 1600, + "n": 12288, + "k": 2048, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 1601, + "n": 320, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 1602, + "n": 24, + "k": 40960, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 1602, + "n": 64, + "k": 10240, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 1602, + "n": 192, + "k": 10240, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 1602, + "n": 256, + "k": 10240, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 1602, + "n": 512, + "k": 10240, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 1602, + "n": 512, + "k": 10240, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 1602, + "n": 1024, + "k": 10240, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 1602, + "n": 1536, + "k": 10240, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 1602, + "n": 10240, + "k": 1024, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 1602, + "n": 10240, + "k": 2048, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 1604, + "n": 896, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 1604, + "n": 7168, + "k": 448, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 1608, + "n": 24, + "k": 40960, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 1608, + "n": 64, + "k": 10240, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 1608, + "n": 192, + "k": 10240, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 1608, + "n": 256, + "k": 10240, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 1608, + "n": 320, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 1608, + "n": 512, + "k": 10240, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 1608, + "n": 512, + "k": 10240, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 1608, + "n": 1024, + "k": 10240, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 1608, + "n": 1024, + "k": 10240, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 1608, + "n": 1536, + "k": 10240, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 1608, + "n": 10240, + "k": 512, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 1608, + "n": 10240, + "k": 1024, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 1608, + "n": 10240, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 1608, + "n": 10240, + "k": 2048, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 1608, + "n": 12288, + "k": 2048, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 1616, + "n": 24, + "k": 40960, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 1616, + "n": 64, + "k": 10240, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 1616, + "n": 192, + "k": 10240, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 1616, + "n": 256, + "k": 10240, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 1616, + "n": 320, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 1616, + "n": 512, + "k": 10240, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 1616, + "n": 512, + "k": 10240, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 1616, + "n": 896, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 1616, + "n": 1024, + "k": 10240, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 1616, + "n": 1536, + "k": 10240, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 1616, + "n": 4096, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 1616, + "n": 7168, + "k": 448, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 1616, + "n": 7168, + "k": 1024, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 1616, + "n": 10240, + "k": 1024, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 1624, + "n": 896, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 1624, + "n": 7168, + "k": 448, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 1632, + "n": 4096, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 1632, + "n": 7168, + "k": 1024, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 1664, + "n": 24, + "k": 40960, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 1664, + "n": 24, + "k": 49152, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 1664, + "n": 64, + "k": 10240, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 1664, + "n": 64, + "k": 12288, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 1664, + "n": 120, + "k": 12288, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 1664, + "n": 192, + "k": 10240, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 1664, + "n": 256, + "k": 10240, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 1664, + "n": 256, + "k": 12288, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 1664, + "n": 320, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 1664, + "n": 512, + "k": 10240, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 1664, + "n": 512, + "k": 10240, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 1664, + "n": 512, + "k": 12288, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 1664, + "n": 512, + "k": 12288, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 1664, + "n": 768, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w3" + ] + }, + { + "type": "gemm", + "args": { + "m": 1664, + "n": 1024, + "k": 10240, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 1664, + "n": 1024, + "k": 10240, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 1664, + "n": 1024, + "k": 12288, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 1664, + "n": 1536, + "k": 10240, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 1664, + "n": 1536, + "k": 12288, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 1664, + "n": 2048, + "k": 12288, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 1664, + "n": 4096, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 1664, + "n": 5120, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 1664, + "n": 6144, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3" + ] + }, + { + "type": "gemm", + "args": { + "m": 1664, + "n": 7168, + "k": 384, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w2" + ] + }, + { + "type": "gemm", + "args": { + "m": 1664, + "n": 7168, + "k": 1024, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 1664, + "n": 7168, + "k": 2048, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 1664, + "n": 7168, + "k": 3072, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w2" + ] + }, + { + "type": "gemm", + "args": { + "m": 1664, + "n": 7168, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.e_proj", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.h_proj", + "deepseek-ai/DeepSeek-V4-Pro/mtp.e_proj", + "deepseek-ai/DeepSeek-V4-Pro/mtp.h_proj", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.e_proj", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.h_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 1664, + "n": 10240, + "k": 512, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 1664, + "n": 10240, + "k": 1024, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 1664, + "n": 10240, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 1664, + "n": 10240, + "k": 2048, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 1664, + "n": 10240, + "k": 16384, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 1664, + "n": 12288, + "k": 1024, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 1664, + "n": 12288, + "k": 2048, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 1664, + "n": 12288, + "k": 8192, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 1664, + "n": 12288, + "k": 16384, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 1664, + "n": 14336, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 1664, + "n": 16384, + "k": 12288, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 1664, + "n": 81920, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 1665, + "n": 768, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w3" + ] + }, + { + "type": "gemm", + "args": { + "m": 1665, + "n": 7168, + "k": 384, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w2" + ] + }, + { + "type": "gemm", + "args": { + "m": 1665, + "n": 7168, + "k": 2048, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 1672, + "n": 768, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w3" + ] + }, + { + "type": "gemm", + "args": { + "m": 1672, + "n": 7168, + "k": 384, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w2" + ] + }, + { + "type": "gemm", + "args": { + "m": 1672, + "n": 7168, + "k": 2048, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 1680, + "n": 10240, + "k": 2048, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 1696, + "n": 320, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 1696, + "n": 896, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 1696, + "n": 7168, + "k": 448, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 1701, + "n": 1792, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 1701, + "n": 7168, + "k": 896, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 1728, + "n": 24, + "k": 40960, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 1728, + "n": 64, + "k": 10240, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 1728, + "n": 192, + "k": 10240, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 1728, + "n": 256, + "k": 10240, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 1728, + "n": 384, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w3" + ] + }, + { + "type": "gemm", + "args": { + "m": 1728, + "n": 512, + "k": 10240, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 1728, + "n": 512, + "k": 10240, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 1728, + "n": 768, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w3" + ] + }, + { + "type": "gemm", + "args": { + "m": 1728, + "n": 1024, + "k": 10240, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 1728, + "n": 1536, + "k": 10240, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 1728, + "n": 1792, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 1728, + "n": 7168, + "k": 192, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w2" + ] + }, + { + "type": "gemm", + "args": { + "m": 1728, + "n": 7168, + "k": 384, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w2" + ] + }, + { + "type": "gemm", + "args": { + "m": 1728, + "n": 7168, + "k": 896, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 1728, + "n": 7168, + "k": 2048, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 1728, + "n": 10240, + "k": 1024, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 1728, + "n": 10240, + "k": 2048, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 1729, + "n": 384, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w3" + ] + }, + { + "type": "gemm", + "args": { + "m": 1729, + "n": 7168, + "k": 192, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w2" + ] + }, + { + "type": "gemm", + "args": { + "m": 1734, + "n": 24, + "k": 28672, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/hc_attn", + "deepseek-ai/DeepSeek-V4-Pro-0813/hc_ffn", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.hc_attn", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.hc_ffn", + "deepseek-ai/DeepSeek-V4-Pro/hc_attn", + "deepseek-ai/DeepSeek-V4-Pro/hc_ffn", + "deepseek-ai/DeepSeek-V4-Pro/mtp.hc_attn", + "deepseek-ai/DeepSeek-V4-Pro/mtp.hc_ffn", + "sgl-project/DeepSeek-V4-Pro-FP8/hc_attn", + "sgl-project/DeepSeek-V4-Pro-FP8/hc_ffn", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.hc_attn", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.hc_ffn" + ] + }, + { + "type": "gemm", + "args": { + "m": 1734, + "n": 64, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.weights_proj", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.weights_proj", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.weights_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 1734, + "n": 256, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.compressor.wkv", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.compressor.wkv", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.compressor.wgate", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.compressor.wkv" + ] + }, + { + "type": "gemm", + "args": { + "m": 1734, + "n": 512, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wkv", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wkv", + "deepseek-ai/DeepSeek-V4-Pro/attn.wkv", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wkv", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wkv", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wkv" + ] + }, + { + "type": "gemm", + "args": { + "m": 1734, + "n": 512, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.compressor.wkv+wgate", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.compressor.wkv+wgate", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.compressor.wkv+wgate" + ] + }, + { + "type": "gemm", + "args": { + "m": 1734, + "n": 1024, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.compressor.wkv", + "deepseek-ai/DeepSeek-V4-Pro/attn.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro/attn.compressor.wkv", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.compressor.wgate", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.compressor.wkv" + ] + }, + { + "type": "gemm", + "args": { + "m": 1734, + "n": 1536, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w3" + ] + }, + { + "type": "gemm", + "args": { + "m": 1734, + "n": 8192, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 1736, + "n": 24, + "k": 28672, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/hc_attn", + "deepseek-ai/DeepSeek-V4-Pro-0813/hc_ffn", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.hc_attn", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.hc_ffn", + "deepseek-ai/DeepSeek-V4-Pro/hc_attn", + "deepseek-ai/DeepSeek-V4-Pro/hc_ffn", + "deepseek-ai/DeepSeek-V4-Pro/mtp.hc_attn", + "deepseek-ai/DeepSeek-V4-Pro/mtp.hc_ffn", + "sgl-project/DeepSeek-V4-Pro-FP8/hc_attn", + "sgl-project/DeepSeek-V4-Pro-FP8/hc_ffn", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.hc_attn", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.hc_ffn" + ] + }, + { + "type": "gemm", + "args": { + "m": 1736, + "n": 64, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.weights_proj", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.weights_proj", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.weights_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 1736, + "n": 256, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.compressor.wkv", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.compressor.wkv", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.compressor.wgate", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.compressor.wkv" + ] + }, + { + "type": "gemm", + "args": { + "m": 1736, + "n": 384, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w3" + ] + }, + { + "type": "gemm", + "args": { + "m": 1736, + "n": 512, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wkv", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wkv", + "deepseek-ai/DeepSeek-V4-Pro/attn.wkv", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wkv", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wkv", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wkv" + ] + }, + { + "type": "gemm", + "args": { + "m": 1736, + "n": 512, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.compressor.wkv+wgate", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.compressor.wkv+wgate", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.compressor.wkv+wgate" + ] + }, + { + "type": "gemm", + "args": { + "m": 1736, + "n": 1024, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.compressor.wkv", + "deepseek-ai/DeepSeek-V4-Pro/attn.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro/attn.compressor.wkv", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.compressor.wgate", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.compressor.wkv" + ] + }, + { + "type": "gemm", + "args": { + "m": 1736, + "n": 1536, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w3" + ] + }, + { + "type": "gemm", + "args": { + "m": 1736, + "n": 1792, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 1736, + "n": 7168, + "k": 192, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w2" + ] + }, + { + "type": "gemm", + "args": { + "m": 1736, + "n": 7168, + "k": 896, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 1736, + "n": 8192, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 1739, + "n": 384, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.gate", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.gate", + "deepseek-ai/DeepSeek-V4-Pro/ffn.gate", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.gate", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.gate", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.gate" + ] + }, + { + "type": "gemm", + "args": { + "m": 1744, + "n": 24, + "k": 28672, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/hc_attn", + "deepseek-ai/DeepSeek-V4-Pro-0813/hc_ffn", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.hc_attn", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.hc_ffn", + "deepseek-ai/DeepSeek-V4-Pro/hc_attn", + "deepseek-ai/DeepSeek-V4-Pro/hc_ffn", + "deepseek-ai/DeepSeek-V4-Pro/mtp.hc_attn", + "deepseek-ai/DeepSeek-V4-Pro/mtp.hc_ffn", + "sgl-project/DeepSeek-V4-Pro-FP8/hc_attn", + "sgl-project/DeepSeek-V4-Pro-FP8/hc_ffn", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.hc_attn", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.hc_ffn" + ] + }, + { + "type": "gemm", + "args": { + "m": 1744, + "n": 64, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.weights_proj", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.weights_proj", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.weights_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 1744, + "n": 256, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.compressor.wkv", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.compressor.wkv", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.compressor.wgate", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.compressor.wkv" + ] + }, + { + "type": "gemm", + "args": { + "m": 1744, + "n": 384, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.gate", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.gate", + "deepseek-ai/DeepSeek-V4-Pro/ffn.gate", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.gate", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.gate", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.gate" + ] + }, + { + "type": "gemm", + "args": { + "m": 1744, + "n": 384, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w3" + ] + }, + { + "type": "gemm", + "args": { + "m": 1744, + "n": 512, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wkv", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wkv", + "deepseek-ai/DeepSeek-V4-Pro/attn.wkv", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wkv", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wkv", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wkv" + ] + }, + { + "type": "gemm", + "args": { + "m": 1744, + "n": 512, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.compressor.wkv+wgate", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.compressor.wkv+wgate", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.compressor.wkv+wgate" + ] + }, + { + "type": "gemm", + "args": { + "m": 1744, + "n": 1024, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.compressor.wkv", + "deepseek-ai/DeepSeek-V4-Pro/attn.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro/attn.compressor.wkv", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.compressor.wgate", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.compressor.wkv" + ] + }, + { + "type": "gemm", + "args": { + "m": 1744, + "n": 1536, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w3" + ] + }, + { + "type": "gemm", + "args": { + "m": 1744, + "n": 1792, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 1744, + "n": 7168, + "k": 192, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w2" + ] + }, + { + "type": "gemm", + "args": { + "m": 1744, + "n": 7168, + "k": 896, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 1744, + "n": 7168, + "k": 2048, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 1744, + "n": 8192, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 1747, + "n": 1536, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w3" + ] + }, + { + "type": "gemm", + "args": { + "m": 1747, + "n": 7168, + "k": 768, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w2" + ] + }, + { + "type": "gemm", + "args": { + "m": 1747, + "n": 7168, + "k": 4096, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 1747, + "n": 16384, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 1752, + "n": 384, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.gate", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.gate", + "deepseek-ai/DeepSeek-V4-Pro/ffn.gate", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.gate", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.gate", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.gate" + ] + }, + { + "type": "gemm", + "args": { + "m": 1752, + "n": 1536, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w3" + ] + }, + { + "type": "gemm", + "args": { + "m": 1752, + "n": 7168, + "k": 768, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w2" + ] + }, + { + "type": "gemm", + "args": { + "m": 1752, + "n": 7168, + "k": 4096, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 1752, + "n": 16384, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 1760, + "n": 24, + "k": 28672, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/hc_attn", + "deepseek-ai/DeepSeek-V4-Pro-0813/hc_ffn", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.hc_attn", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.hc_ffn", + "deepseek-ai/DeepSeek-V4-Pro/hc_attn", + "deepseek-ai/DeepSeek-V4-Pro/hc_ffn", + "deepseek-ai/DeepSeek-V4-Pro/mtp.hc_attn", + "deepseek-ai/DeepSeek-V4-Pro/mtp.hc_ffn", + "sgl-project/DeepSeek-V4-Pro-FP8/hc_attn", + "sgl-project/DeepSeek-V4-Pro-FP8/hc_ffn", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.hc_attn", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.hc_ffn" + ] + }, + { + "type": "gemm", + "args": { + "m": 1760, + "n": 64, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.weights_proj", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.weights_proj", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.weights_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 1760, + "n": 256, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.compressor.wkv", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.compressor.wkv", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.compressor.wgate", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.compressor.wkv" + ] + }, + { + "type": "gemm", + "args": { + "m": 1760, + "n": 384, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.gate", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.gate", + "deepseek-ai/DeepSeek-V4-Pro/ffn.gate", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.gate", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.gate", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.gate" + ] + }, + { + "type": "gemm", + "args": { + "m": 1760, + "n": 512, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wkv", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wkv", + "deepseek-ai/DeepSeek-V4-Pro/attn.wkv", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wkv", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wkv", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wkv" + ] + }, + { + "type": "gemm", + "args": { + "m": 1760, + "n": 512, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.compressor.wkv+wgate", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.compressor.wkv+wgate", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.compressor.wkv+wgate" + ] + }, + { + "type": "gemm", + "args": { + "m": 1760, + "n": 1024, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.compressor.wkv", + "deepseek-ai/DeepSeek-V4-Pro/attn.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro/attn.compressor.wkv", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.compressor.wgate", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.compressor.wkv" + ] + }, + { + "type": "gemm", + "args": { + "m": 1760, + "n": 1536, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w3" + ] + }, + { + "type": "gemm", + "args": { + "m": 1760, + "n": 7168, + "k": 768, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w2" + ] + }, + { + "type": "gemm", + "args": { + "m": 1760, + "n": 7168, + "k": 4096, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 1760, + "n": 8192, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 1760, + "n": 16384, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 1776, + "n": 1536, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w3" + ] + }, + { + "type": "gemm", + "args": { + "m": 1776, + "n": 7168, + "k": 768, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w2" + ] + }, + { + "type": "gemm", + "args": { + "m": 1776, + "n": 7168, + "k": 4096, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 1776, + "n": 16384, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 1792, + "n": 24, + "k": 28672, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/hc_attn", + "deepseek-ai/DeepSeek-V4-Pro-0813/hc_ffn", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.hc_attn", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.hc_ffn", + "deepseek-ai/DeepSeek-V4-Pro/hc_attn", + "deepseek-ai/DeepSeek-V4-Pro/hc_ffn", + "deepseek-ai/DeepSeek-V4-Pro/mtp.hc_attn", + "deepseek-ai/DeepSeek-V4-Pro/mtp.hc_ffn", + "sgl-project/DeepSeek-V4-Pro-FP8/hc_attn", + "sgl-project/DeepSeek-V4-Pro-FP8/hc_ffn", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.hc_attn", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.hc_ffn" + ] + }, + { + "type": "gemm", + "args": { + "m": 1792, + "n": 24, + "k": 40960, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 1792, + "n": 64, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.weights_proj", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.weights_proj", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.weights_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 1792, + "n": 64, + "k": 10240, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 1792, + "n": 192, + "k": 10240, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 1792, + "n": 256, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.compressor.wkv", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.compressor.wkv", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.compressor.wgate", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.compressor.wkv" + ] + }, + { + "type": "gemm", + "args": { + "m": 1792, + "n": 256, + "k": 10240, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 1792, + "n": 320, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 1792, + "n": 384, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.gate", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.gate", + "deepseek-ai/DeepSeek-V4-Pro/ffn.gate", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.gate", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.gate", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.gate" + ] + }, + { + "type": "gemm", + "args": { + "m": 1792, + "n": 384, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w3" + ] + }, + { + "type": "gemm", + "args": { + "m": 1792, + "n": 512, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wkv", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wkv", + "deepseek-ai/DeepSeek-V4-Pro/attn.wkv", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wkv", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wkv", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wkv" + ] + }, + { + "type": "gemm", + "args": { + "m": 1792, + "n": 512, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.compressor.wkv+wgate", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.compressor.wkv+wgate", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.compressor.wkv+wgate" + ] + }, + { + "type": "gemm", + "args": { + "m": 1792, + "n": 512, + "k": 10240, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 1792, + "n": 512, + "k": 10240, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 1792, + "n": 768, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w3" + ] + }, + { + "type": "gemm", + "args": { + "m": 1792, + "n": 1024, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.compressor.wkv", + "deepseek-ai/DeepSeek-V4-Pro/attn.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro/attn.compressor.wkv", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.compressor.wgate", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.compressor.wkv" + ] + }, + { + "type": "gemm", + "args": { + "m": 1792, + "n": 1024, + "k": 10240, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 1792, + "n": 1536, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w3" + ] + }, + { + "type": "gemm", + "args": { + "m": 1792, + "n": 1536, + "k": 10240, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 1792, + "n": 1792, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 1792, + "n": 4096, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 1792, + "n": 6144, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3" + ] + }, + { + "type": "gemm", + "args": { + "m": 1792, + "n": 7168, + "k": 192, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w2" + ] + }, + { + "type": "gemm", + "args": { + "m": 1792, + "n": 7168, + "k": 384, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w2" + ] + }, + { + "type": "gemm", + "args": { + "m": 1792, + "n": 7168, + "k": 768, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w2" + ] + }, + { + "type": "gemm", + "args": { + "m": 1792, + "n": 7168, + "k": 896, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 1792, + "n": 7168, + "k": 1024, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 1792, + "n": 7168, + "k": 2048, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 1792, + "n": 7168, + "k": 3072, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w2" + ] + }, + { + "type": "gemm", + "args": { + "m": 1792, + "n": 7168, + "k": 4096, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 1792, + "n": 7168, + "k": 16384, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 1792, + "n": 8192, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 1792, + "n": 10240, + "k": 2048, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 1792, + "n": 16384, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 1792, + "n": 65536, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 1856, + "n": 24, + "k": 28672, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/hc_attn", + "deepseek-ai/DeepSeek-V4-Pro-0813/hc_ffn", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.hc_attn", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.hc_ffn", + "deepseek-ai/DeepSeek-V4-Pro/hc_attn", + "deepseek-ai/DeepSeek-V4-Pro/hc_ffn", + "deepseek-ai/DeepSeek-V4-Pro/mtp.hc_attn", + "deepseek-ai/DeepSeek-V4-Pro/mtp.hc_ffn", + "sgl-project/DeepSeek-V4-Pro-FP8/hc_attn", + "sgl-project/DeepSeek-V4-Pro-FP8/hc_ffn", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.hc_attn", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.hc_ffn" + ] + }, + { + "type": "gemm", + "args": { + "m": 1856, + "n": 24, + "k": 49152, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 1856, + "n": 64, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.weights_proj", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.weights_proj", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.weights_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 1856, + "n": 64, + "k": 12288, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 1856, + "n": 120, + "k": 12288, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 1856, + "n": 256, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.compressor.wkv", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.compressor.wkv", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.compressor.wgate", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.compressor.wkv" + ] + }, + { + "type": "gemm", + "args": { + "m": 1856, + "n": 256, + "k": 12288, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 1856, + "n": 384, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.gate", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.gate", + "deepseek-ai/DeepSeek-V4-Pro/ffn.gate", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.gate", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.gate", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.gate" + ] + }, + { + "type": "gemm", + "args": { + "m": 1856, + "n": 512, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wkv", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wkv", + "deepseek-ai/DeepSeek-V4-Pro/attn.wkv", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wkv", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wkv", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wkv" + ] + }, + { + "type": "gemm", + "args": { + "m": 1856, + "n": 512, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.compressor.wkv+wgate", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.compressor.wkv+wgate", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.compressor.wkv+wgate" + ] + }, + { + "type": "gemm", + "args": { + "m": 1856, + "n": 512, + "k": 12288, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 1856, + "n": 512, + "k": 12288, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 1856, + "n": 1024, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.compressor.wkv", + "deepseek-ai/DeepSeek-V4-Pro/attn.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro/attn.compressor.wkv", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.compressor.wgate", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.compressor.wkv" + ] + }, + { + "type": "gemm", + "args": { + "m": 1856, + "n": 1024, + "k": 12288, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 1856, + "n": 1536, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w3" + ] + }, + { + "type": "gemm", + "args": { + "m": 1856, + "n": 1536, + "k": 12288, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 1856, + "n": 2048, + "k": 12288, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 1856, + "n": 5120, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 1856, + "n": 6144, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3" + ] + }, + { + "type": "gemm", + "args": { + "m": 1856, + "n": 7168, + "k": 768, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w2" + ] + }, + { + "type": "gemm", + "args": { + "m": 1856, + "n": 7168, + "k": 3072, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w2" + ] + }, + { + "type": "gemm", + "args": { + "m": 1856, + "n": 7168, + "k": 4096, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 1856, + "n": 7168, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.e_proj", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.h_proj", + "deepseek-ai/DeepSeek-V4-Pro/mtp.e_proj", + "deepseek-ai/DeepSeek-V4-Pro/mtp.h_proj", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.e_proj", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.h_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 1856, + "n": 7168, + "k": 16384, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 1856, + "n": 8192, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 1856, + "n": 10240, + "k": 16384, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 1856, + "n": 12288, + "k": 1024, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 1856, + "n": 12288, + "k": 8192, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 1856, + "n": 12288, + "k": 16384, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 1856, + "n": 14336, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 1856, + "n": 16384, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 1856, + "n": 16384, + "k": 12288, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 1856, + "n": 65536, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 1856, + "n": 81920, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 1857, + "n": 24, + "k": 49152, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 1857, + "n": 64, + "k": 12288, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 1857, + "n": 120, + "k": 12288, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 1857, + "n": 256, + "k": 12288, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 1857, + "n": 512, + "k": 12288, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 1857, + "n": 512, + "k": 12288, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 1857, + "n": 1024, + "k": 12288, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 1857, + "n": 1536, + "k": 12288, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 1857, + "n": 5120, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 1857, + "n": 6144, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3" + ] + }, + { + "type": "gemm", + "args": { + "m": 1857, + "n": 7168, + "k": 3072, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w2" + ] + }, + { + "type": "gemm", + "args": { + "m": 1857, + "n": 7168, + "k": 16384, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 1857, + "n": 10240, + "k": 16384, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 1857, + "n": 12288, + "k": 1024, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 1857, + "n": 65536, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 1864, + "n": 24, + "k": 49152, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 1864, + "n": 64, + "k": 12288, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 1864, + "n": 120, + "k": 12288, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 1864, + "n": 256, + "k": 12288, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 1864, + "n": 512, + "k": 12288, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 1864, + "n": 512, + "k": 12288, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 1864, + "n": 1024, + "k": 12288, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 1864, + "n": 1536, + "k": 12288, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 1864, + "n": 5120, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 1864, + "n": 12288, + "k": 1024, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 1872, + "n": 24, + "k": 49152, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 1872, + "n": 64, + "k": 12288, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 1872, + "n": 120, + "k": 12288, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 1872, + "n": 256, + "k": 12288, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 1872, + "n": 512, + "k": 12288, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 1872, + "n": 512, + "k": 12288, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 1872, + "n": 768, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w3" + ] + }, + { + "type": "gemm", + "args": { + "m": 1872, + "n": 1024, + "k": 12288, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 1872, + "n": 1536, + "k": 12288, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 1872, + "n": 5120, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 1872, + "n": 7168, + "k": 384, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w2" + ] + }, + { + "type": "gemm", + "args": { + "m": 1872, + "n": 12288, + "k": 1024, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 1888, + "n": 24, + "k": 40960, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 1888, + "n": 24, + "k": 49152, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 1888, + "n": 64, + "k": 10240, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 1888, + "n": 64, + "k": 12288, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 1888, + "n": 120, + "k": 12288, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 1888, + "n": 192, + "k": 10240, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 1888, + "n": 256, + "k": 10240, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 1888, + "n": 256, + "k": 12288, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 1888, + "n": 512, + "k": 10240, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 1888, + "n": 512, + "k": 10240, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 1888, + "n": 512, + "k": 12288, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 1888, + "n": 512, + "k": 12288, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 1888, + "n": 1024, + "k": 10240, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 1888, + "n": 1024, + "k": 12288, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 1888, + "n": 1536, + "k": 10240, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 1888, + "n": 1536, + "k": 12288, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 1888, + "n": 5120, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 1888, + "n": 10240, + "k": 1024, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 1888, + "n": 12288, + "k": 1024, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 1920, + "n": 24, + "k": 28672, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/hc_attn", + "deepseek-ai/DeepSeek-V4-Pro-0813/hc_ffn", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.hc_attn", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.hc_ffn", + "deepseek-ai/DeepSeek-V4-Pro/hc_attn", + "deepseek-ai/DeepSeek-V4-Pro/hc_ffn", + "deepseek-ai/DeepSeek-V4-Pro/mtp.hc_attn", + "deepseek-ai/DeepSeek-V4-Pro/mtp.hc_ffn", + "sgl-project/DeepSeek-V4-Pro-FP8/hc_attn", + "sgl-project/DeepSeek-V4-Pro-FP8/hc_ffn", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.hc_attn", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.hc_ffn" + ] + }, + { + "type": "gemm", + "args": { + "m": 1920, + "n": 24, + "k": 49152, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 1920, + "n": 64, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.weights_proj", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.weights_proj", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.weights_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 1920, + "n": 64, + "k": 12288, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 1920, + "n": 120, + "k": 12288, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 1920, + "n": 256, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.compressor.wkv", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.compressor.wkv", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.compressor.wgate", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.compressor.wkv" + ] + }, + { + "type": "gemm", + "args": { + "m": 1920, + "n": 256, + "k": 12288, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 1920, + "n": 384, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.gate", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.gate", + "deepseek-ai/DeepSeek-V4-Pro/ffn.gate", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.gate", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.gate", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.gate" + ] + }, + { + "type": "gemm", + "args": { + "m": 1920, + "n": 384, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w3" + ] + }, + { + "type": "gemm", + "args": { + "m": 1920, + "n": 512, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wkv", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wkv", + "deepseek-ai/DeepSeek-V4-Pro/attn.wkv", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wkv", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wkv", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wkv" + ] + }, + { + "type": "gemm", + "args": { + "m": 1920, + "n": 512, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.compressor.wkv+wgate", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.compressor.wkv+wgate", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.compressor.wkv+wgate" + ] + }, + { + "type": "gemm", + "args": { + "m": 1920, + "n": 512, + "k": 12288, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 1920, + "n": 512, + "k": 12288, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 1920, + "n": 1024, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.compressor.wkv", + "deepseek-ai/DeepSeek-V4-Pro/attn.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro/attn.compressor.wkv", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.compressor.wgate", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.compressor.wkv" + ] + }, + { + "type": "gemm", + "args": { + "m": 1920, + "n": 1024, + "k": 12288, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 1920, + "n": 1536, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w3" + ] + }, + { + "type": "gemm", + "args": { + "m": 1920, + "n": 1536, + "k": 12288, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 1920, + "n": 1792, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 1920, + "n": 2048, + "k": 12288, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 1920, + "n": 5120, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 1920, + "n": 6144, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3" + ] + }, + { + "type": "gemm", + "args": { + "m": 1920, + "n": 7168, + "k": 192, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w2" + ] + }, + { + "type": "gemm", + "args": { + "m": 1920, + "n": 7168, + "k": 768, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w2" + ] + }, + { + "type": "gemm", + "args": { + "m": 1920, + "n": 7168, + "k": 896, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 1920, + "n": 7168, + "k": 3072, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w2" + ] + }, + { + "type": "gemm", + "args": { + "m": 1920, + "n": 7168, + "k": 4096, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 1920, + "n": 7168, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.e_proj", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.h_proj", + "deepseek-ai/DeepSeek-V4-Pro/mtp.e_proj", + "deepseek-ai/DeepSeek-V4-Pro/mtp.h_proj", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.e_proj", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.h_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 1920, + "n": 7168, + "k": 16384, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 1920, + "n": 8192, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 1920, + "n": 10240, + "k": 16384, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 1920, + "n": 12288, + "k": 1024, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 1920, + "n": 12288, + "k": 8192, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 1920, + "n": 12288, + "k": 16384, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 1920, + "n": 14336, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 1920, + "n": 16384, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 1920, + "n": 16384, + "k": 12288, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 1920, + "n": 65536, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 1920, + "n": 81920, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 1921, + "n": 1024, + "k": 10240, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 1921, + "n": 10240, + "k": 512, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 1928, + "n": 10240, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 1928, + "n": 12288, + "k": 2048, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 1930, + "n": 10240, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 1930, + "n": 12288, + "k": 2048, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 1979, + "n": 1024, + "k": 12288, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 1979, + "n": 12288, + "k": 512, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 1984, + "n": 1024, + "k": 10240, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 1984, + "n": 4096, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 1984, + "n": 7168, + "k": 1024, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 1984, + "n": 10240, + "k": 512, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 1984, + "n": 10240, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 1984, + "n": 12288, + "k": 2048, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 1985, + "n": 4096, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 1985, + "n": 7168, + "k": 1024, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 1992, + "n": 1024, + "k": 10240, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 1992, + "n": 4096, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 1992, + "n": 7168, + "k": 1024, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 1992, + "n": 10240, + "k": 512, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 1994, + "n": 7168, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.e_proj", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.h_proj", + "deepseek-ai/DeepSeek-V4-Pro/mtp.e_proj", + "deepseek-ai/DeepSeek-V4-Pro/mtp.h_proj", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.e_proj", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.h_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 1994, + "n": 8192, + "k": 10240, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 1994, + "n": 10240, + "k": 4096, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 1994, + "n": 14336, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 2000, + "n": 4096, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 2000, + "n": 7168, + "k": 1024, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 2001, + "n": 320, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 2005, + "n": 24, + "k": 40960, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 2005, + "n": 64, + "k": 10240, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 2005, + "n": 192, + "k": 10240, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 2005, + "n": 256, + "k": 10240, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 2005, + "n": 512, + "k": 10240, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 2005, + "n": 512, + "k": 10240, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 2005, + "n": 1024, + "k": 10240, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 2005, + "n": 1536, + "k": 10240, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 2005, + "n": 10240, + "k": 1024, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 2007, + "n": 896, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 2007, + "n": 7168, + "k": 448, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 2007, + "n": 10240, + "k": 2048, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 2008, + "n": 24, + "k": 40960, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 2008, + "n": 64, + "k": 10240, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 2008, + "n": 192, + "k": 10240, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 2008, + "n": 256, + "k": 10240, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 2008, + "n": 320, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 2008, + "n": 512, + "k": 10240, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 2008, + "n": 512, + "k": 10240, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 2008, + "n": 896, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 2008, + "n": 1024, + "k": 10240, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 2008, + "n": 1536, + "k": 10240, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 2008, + "n": 7168, + "k": 448, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 2008, + "n": 10240, + "k": 1024, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 2008, + "n": 10240, + "k": 2048, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 2016, + "n": 320, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 2016, + "n": 896, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 2016, + "n": 4096, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 2016, + "n": 7168, + "k": 448, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 2016, + "n": 7168, + "k": 1024, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 2048, + "n": 24, + "k": 40960, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 2048, + "n": 24, + "k": 49152, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 2048, + "n": 64, + "k": 10240, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 2048, + "n": 64, + "k": 12288, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 2048, + "n": 120, + "k": 12288, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 2048, + "n": 192, + "k": 10240, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 2048, + "n": 256, + "k": 10240, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 2048, + "n": 256, + "k": 12288, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 2048, + "n": 320, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 2048, + "n": 512, + "k": 10240, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 2048, + "n": 512, + "k": 10240, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 2048, + "n": 512, + "k": 12288, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 2048, + "n": 512, + "k": 12288, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 2048, + "n": 1024, + "k": 10240, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 2048, + "n": 1024, + "k": 10240, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 2048, + "n": 1024, + "k": 12288, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 2048, + "n": 1536, + "k": 10240, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 2048, + "n": 1536, + "k": 12288, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 2048, + "n": 2048, + "k": 12288, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 2048, + "n": 4096, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 2048, + "n": 5120, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 2048, + "n": 6144, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3" + ] + }, + { + "type": "gemm", + "args": { + "m": 2048, + "n": 7168, + "k": 1024, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 2048, + "n": 7168, + "k": 3072, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w2" + ] + }, + { + "type": "gemm", + "args": { + "m": 2048, + "n": 7168, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.e_proj", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.h_proj", + "deepseek-ai/DeepSeek-V4-Pro/mtp.e_proj", + "deepseek-ai/DeepSeek-V4-Pro/mtp.h_proj", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.e_proj", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.h_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 2048, + "n": 10240, + "k": 512, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 2048, + "n": 10240, + "k": 1024, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 2048, + "n": 10240, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 2048, + "n": 10240, + "k": 2048, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 2048, + "n": 10240, + "k": 16384, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 2048, + "n": 12288, + "k": 1024, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 2048, + "n": 12288, + "k": 2048, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 2048, + "n": 12288, + "k": 8192, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 2048, + "n": 12288, + "k": 16384, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 2048, + "n": 14336, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 2048, + "n": 16384, + "k": 12288, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 2048, + "n": 81920, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 2064, + "n": 24, + "k": 40960, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 2064, + "n": 64, + "k": 10240, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 2064, + "n": 192, + "k": 10240, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 2064, + "n": 256, + "k": 10240, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 2064, + "n": 320, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 2064, + "n": 512, + "k": 10240, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 2064, + "n": 512, + "k": 10240, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 2064, + "n": 896, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 2064, + "n": 1024, + "k": 10240, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 2064, + "n": 1536, + "k": 10240, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 2064, + "n": 7168, + "k": 448, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 2064, + "n": 10240, + "k": 2048, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 2112, + "n": 24, + "k": 40960, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 2112, + "n": 64, + "k": 10240, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 2112, + "n": 192, + "k": 10240, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 2112, + "n": 256, + "k": 10240, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 2112, + "n": 320, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 2112, + "n": 512, + "k": 10240, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 2112, + "n": 512, + "k": 10240, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 2112, + "n": 768, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w3" + ] + }, + { + "type": "gemm", + "args": { + "m": 2112, + "n": 896, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 2112, + "n": 1024, + "k": 10240, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 2112, + "n": 1536, + "k": 10240, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 2112, + "n": 7168, + "k": 384, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w2" + ] + }, + { + "type": "gemm", + "args": { + "m": 2112, + "n": 7168, + "k": 448, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 2112, + "n": 7168, + "k": 2048, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 2112, + "n": 10240, + "k": 1024, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 2112, + "n": 10240, + "k": 2048, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 2113, + "n": 768, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w3" + ] + }, + { + "type": "gemm", + "args": { + "m": 2113, + "n": 7168, + "k": 384, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w2" + ] + }, + { + "type": "gemm", + "args": { + "m": 2113, + "n": 7168, + "k": 2048, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 2120, + "n": 768, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w3" + ] + }, + { + "type": "gemm", + "args": { + "m": 2120, + "n": 7168, + "k": 384, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w2" + ] + }, + { + "type": "gemm", + "args": { + "m": 2120, + "n": 7168, + "k": 2048, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 2128, + "n": 768, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w3" + ] + }, + { + "type": "gemm", + "args": { + "m": 2128, + "n": 7168, + "k": 384, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w2" + ] + }, + { + "type": "gemm", + "args": { + "m": 2128, + "n": 7168, + "k": 2048, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 2128, + "n": 10240, + "k": 1024, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 2168, + "n": 24, + "k": 28672, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/hc_attn", + "deepseek-ai/DeepSeek-V4-Pro-0813/hc_ffn", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.hc_attn", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.hc_ffn", + "deepseek-ai/DeepSeek-V4-Pro/hc_attn", + "deepseek-ai/DeepSeek-V4-Pro/hc_ffn", + "deepseek-ai/DeepSeek-V4-Pro/mtp.hc_attn", + "deepseek-ai/DeepSeek-V4-Pro/mtp.hc_ffn", + "sgl-project/DeepSeek-V4-Pro-FP8/hc_attn", + "sgl-project/DeepSeek-V4-Pro-FP8/hc_ffn", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.hc_attn", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.hc_ffn" + ] + }, + { + "type": "gemm", + "args": { + "m": 2168, + "n": 64, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.weights_proj", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.weights_proj", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.weights_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 2168, + "n": 256, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.compressor.wkv", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.compressor.wkv", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.compressor.wgate", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.compressor.wkv" + ] + }, + { + "type": "gemm", + "args": { + "m": 2168, + "n": 512, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wkv", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wkv", + "deepseek-ai/DeepSeek-V4-Pro/attn.wkv", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wkv", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wkv", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wkv" + ] + }, + { + "type": "gemm", + "args": { + "m": 2168, + "n": 512, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.compressor.wkv+wgate", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.compressor.wkv+wgate", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.compressor.wkv+wgate" + ] + }, + { + "type": "gemm", + "args": { + "m": 2168, + "n": 1024, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.compressor.wkv", + "deepseek-ai/DeepSeek-V4-Pro/attn.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro/attn.compressor.wkv", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.compressor.wgate", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.compressor.wkv" + ] + }, + { + "type": "gemm", + "args": { + "m": 2168, + "n": 1536, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w3" + ] + }, + { + "type": "gemm", + "args": { + "m": 2168, + "n": 8192, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 2169, + "n": 24, + "k": 28672, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/hc_attn", + "deepseek-ai/DeepSeek-V4-Pro-0813/hc_ffn", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.hc_attn", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.hc_ffn", + "deepseek-ai/DeepSeek-V4-Pro/hc_attn", + "deepseek-ai/DeepSeek-V4-Pro/hc_ffn", + "deepseek-ai/DeepSeek-V4-Pro/mtp.hc_attn", + "deepseek-ai/DeepSeek-V4-Pro/mtp.hc_ffn", + "sgl-project/DeepSeek-V4-Pro-FP8/hc_attn", + "sgl-project/DeepSeek-V4-Pro-FP8/hc_ffn", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.hc_attn", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.hc_ffn" + ] + }, + { + "type": "gemm", + "args": { + "m": 2169, + "n": 64, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.weights_proj", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.weights_proj", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.weights_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 2169, + "n": 256, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.compressor.wkv", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.compressor.wkv", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.compressor.wgate", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.compressor.wkv" + ] + }, + { + "type": "gemm", + "args": { + "m": 2169, + "n": 512, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wkv", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wkv", + "deepseek-ai/DeepSeek-V4-Pro/attn.wkv", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wkv", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wkv", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wkv" + ] + }, + { + "type": "gemm", + "args": { + "m": 2169, + "n": 512, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.compressor.wkv+wgate", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.compressor.wkv+wgate", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.compressor.wkv+wgate" + ] + }, + { + "type": "gemm", + "args": { + "m": 2169, + "n": 1024, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.compressor.wkv", + "deepseek-ai/DeepSeek-V4-Pro/attn.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro/attn.compressor.wkv", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.compressor.wgate", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.compressor.wkv" + ] + }, + { + "type": "gemm", + "args": { + "m": 2169, + "n": 1536, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w3" + ] + }, + { + "type": "gemm", + "args": { + "m": 2169, + "n": 8192, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 2174, + "n": 384, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.gate", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.gate", + "deepseek-ai/DeepSeek-V4-Pro/ffn.gate", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.gate", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.gate", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.gate" + ] + }, + { + "type": "gemm", + "args": { + "m": 2176, + "n": 24, + "k": 28672, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/hc_attn", + "deepseek-ai/DeepSeek-V4-Pro-0813/hc_ffn", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.hc_attn", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.hc_ffn", + "deepseek-ai/DeepSeek-V4-Pro/hc_attn", + "deepseek-ai/DeepSeek-V4-Pro/hc_ffn", + "deepseek-ai/DeepSeek-V4-Pro/mtp.hc_attn", + "deepseek-ai/DeepSeek-V4-Pro/mtp.hc_ffn", + "sgl-project/DeepSeek-V4-Pro-FP8/hc_attn", + "sgl-project/DeepSeek-V4-Pro-FP8/hc_ffn", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.hc_attn", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.hc_ffn" + ] + }, + { + "type": "gemm", + "args": { + "m": 2176, + "n": 24, + "k": 40960, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 2176, + "n": 64, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.weights_proj", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.weights_proj", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.weights_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 2176, + "n": 64, + "k": 10240, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 2176, + "n": 192, + "k": 10240, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 2176, + "n": 256, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.compressor.wkv", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.compressor.wkv", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.compressor.wgate", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.compressor.wkv" + ] + }, + { + "type": "gemm", + "args": { + "m": 2176, + "n": 256, + "k": 10240, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 2176, + "n": 320, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 2176, + "n": 384, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.gate", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.gate", + "deepseek-ai/DeepSeek-V4-Pro/ffn.gate", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.gate", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.gate", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.gate" + ] + }, + { + "type": "gemm", + "args": { + "m": 2176, + "n": 384, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w3" + ] + }, + { + "type": "gemm", + "args": { + "m": 2176, + "n": 512, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wkv", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wkv", + "deepseek-ai/DeepSeek-V4-Pro/attn.wkv", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wkv", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wkv", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wkv" + ] + }, + { + "type": "gemm", + "args": { + "m": 2176, + "n": 512, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.compressor.wkv+wgate", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.compressor.wkv+wgate", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.compressor.wkv+wgate" + ] + }, + { + "type": "gemm", + "args": { + "m": 2176, + "n": 512, + "k": 10240, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 2176, + "n": 512, + "k": 10240, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 2176, + "n": 1024, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.compressor.wkv", + "deepseek-ai/DeepSeek-V4-Pro/attn.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro/attn.compressor.wkv", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.compressor.wgate", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.compressor.wkv" + ] + }, + { + "type": "gemm", + "args": { + "m": 2176, + "n": 1024, + "k": 10240, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 2176, + "n": 1024, + "k": 10240, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 2176, + "n": 1536, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w3" + ] + }, + { + "type": "gemm", + "args": { + "m": 2176, + "n": 1536, + "k": 10240, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 2176, + "n": 1792, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 2176, + "n": 4096, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 2176, + "n": 6144, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3" + ] + }, + { + "type": "gemm", + "args": { + "m": 2176, + "n": 7168, + "k": 192, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w2" + ] + }, + { + "type": "gemm", + "args": { + "m": 2176, + "n": 7168, + "k": 896, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 2176, + "n": 7168, + "k": 1024, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 2176, + "n": 7168, + "k": 2048, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 2176, + "n": 7168, + "k": 3072, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w2" + ] + }, + { + "type": "gemm", + "args": { + "m": 2176, + "n": 8192, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 2176, + "n": 10240, + "k": 512, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 2176, + "n": 10240, + "k": 1024, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 2176, + "n": 10240, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 2176, + "n": 10240, + "k": 2048, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 2176, + "n": 12288, + "k": 2048, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 2177, + "n": 384, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w3" + ] + }, + { + "type": "gemm", + "args": { + "m": 2177, + "n": 7168, + "k": 192, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w2" + ] + }, + { + "type": "gemm", + "args": { + "m": 2178, + "n": 1792, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 2178, + "n": 7168, + "k": 896, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 2184, + "n": 384, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.gate", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.gate", + "deepseek-ai/DeepSeek-V4-Pro/ffn.gate", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.gate", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.gate", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.gate" + ] + }, + { + "type": "gemm", + "args": { + "m": 2184, + "n": 1536, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w3" + ] + }, + { + "type": "gemm", + "args": { + "m": 2184, + "n": 1792, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 2184, + "n": 7168, + "k": 768, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w2" + ] + }, + { + "type": "gemm", + "args": { + "m": 2184, + "n": 7168, + "k": 896, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 2184, + "n": 7168, + "k": 4096, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 2184, + "n": 16384, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 2185, + "n": 1536, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w3" + ] + }, + { + "type": "gemm", + "args": { + "m": 2185, + "n": 7168, + "k": 768, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w2" + ] + }, + { + "type": "gemm", + "args": { + "m": 2185, + "n": 7168, + "k": 4096, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 2185, + "n": 16384, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 2192, + "n": 24, + "k": 28672, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/hc_attn", + "deepseek-ai/DeepSeek-V4-Pro-0813/hc_ffn", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.hc_attn", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.hc_ffn", + "deepseek-ai/DeepSeek-V4-Pro/hc_attn", + "deepseek-ai/DeepSeek-V4-Pro/hc_ffn", + "deepseek-ai/DeepSeek-V4-Pro/mtp.hc_attn", + "deepseek-ai/DeepSeek-V4-Pro/mtp.hc_ffn", + "sgl-project/DeepSeek-V4-Pro-FP8/hc_attn", + "sgl-project/DeepSeek-V4-Pro-FP8/hc_ffn", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.hc_attn", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.hc_ffn" + ] + }, + { + "type": "gemm", + "args": { + "m": 2192, + "n": 64, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.weights_proj", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.weights_proj", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.weights_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 2192, + "n": 256, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.compressor.wkv", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.compressor.wkv", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.compressor.wgate", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.compressor.wkv" + ] + }, + { + "type": "gemm", + "args": { + "m": 2192, + "n": 384, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.gate", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.gate", + "deepseek-ai/DeepSeek-V4-Pro/ffn.gate", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.gate", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.gate", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.gate" + ] + }, + { + "type": "gemm", + "args": { + "m": 2192, + "n": 512, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wkv", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wkv", + "deepseek-ai/DeepSeek-V4-Pro/attn.wkv", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wkv", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wkv", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wkv" + ] + }, + { + "type": "gemm", + "args": { + "m": 2192, + "n": 512, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.compressor.wkv+wgate", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.compressor.wkv+wgate", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.compressor.wkv+wgate" + ] + }, + { + "type": "gemm", + "args": { + "m": 2192, + "n": 1024, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.compressor.wkv", + "deepseek-ai/DeepSeek-V4-Pro/attn.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro/attn.compressor.wkv", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.compressor.wgate", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.compressor.wkv" + ] + }, + { + "type": "gemm", + "args": { + "m": 2192, + "n": 1536, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w3" + ] + }, + { + "type": "gemm", + "args": { + "m": 2192, + "n": 7168, + "k": 768, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w2" + ] + }, + { + "type": "gemm", + "args": { + "m": 2192, + "n": 7168, + "k": 4096, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 2192, + "n": 8192, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 2192, + "n": 16384, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 2208, + "n": 24, + "k": 28672, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/hc_attn", + "deepseek-ai/DeepSeek-V4-Pro-0813/hc_ffn", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.hc_attn", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.hc_ffn", + "deepseek-ai/DeepSeek-V4-Pro/hc_attn", + "deepseek-ai/DeepSeek-V4-Pro/hc_ffn", + "deepseek-ai/DeepSeek-V4-Pro/mtp.hc_attn", + "deepseek-ai/DeepSeek-V4-Pro/mtp.hc_ffn", + "sgl-project/DeepSeek-V4-Pro-FP8/hc_attn", + "sgl-project/DeepSeek-V4-Pro-FP8/hc_ffn", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.hc_attn", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.hc_ffn" + ] + }, + { + "type": "gemm", + "args": { + "m": 2208, + "n": 64, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.weights_proj", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.weights_proj", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.weights_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 2208, + "n": 256, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.compressor.wkv", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.compressor.wkv", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.compressor.wgate", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.compressor.wkv" + ] + }, + { + "type": "gemm", + "args": { + "m": 2208, + "n": 384, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.gate", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.gate", + "deepseek-ai/DeepSeek-V4-Pro/ffn.gate", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.gate", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.gate", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.gate" + ] + }, + { + "type": "gemm", + "args": { + "m": 2208, + "n": 512, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wkv", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wkv", + "deepseek-ai/DeepSeek-V4-Pro/attn.wkv", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wkv", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wkv", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wkv" + ] + }, + { + "type": "gemm", + "args": { + "m": 2208, + "n": 512, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.compressor.wkv+wgate", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.compressor.wkv+wgate", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.compressor.wkv+wgate" + ] + }, + { + "type": "gemm", + "args": { + "m": 2208, + "n": 1024, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.compressor.wkv", + "deepseek-ai/DeepSeek-V4-Pro/attn.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro/attn.compressor.wkv", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.compressor.wgate", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.compressor.wkv" + ] + }, + { + "type": "gemm", + "args": { + "m": 2208, + "n": 1536, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w3" + ] + }, + { + "type": "gemm", + "args": { + "m": 2208, + "n": 7168, + "k": 768, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w2" + ] + }, + { + "type": "gemm", + "args": { + "m": 2208, + "n": 7168, + "k": 4096, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 2208, + "n": 8192, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 2208, + "n": 16384, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 2240, + "n": 24, + "k": 28672, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/hc_attn", + "deepseek-ai/DeepSeek-V4-Pro-0813/hc_ffn", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.hc_attn", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.hc_ffn", + "deepseek-ai/DeepSeek-V4-Pro/hc_attn", + "deepseek-ai/DeepSeek-V4-Pro/hc_ffn", + "deepseek-ai/DeepSeek-V4-Pro/mtp.hc_attn", + "deepseek-ai/DeepSeek-V4-Pro/mtp.hc_ffn", + "sgl-project/DeepSeek-V4-Pro-FP8/hc_attn", + "sgl-project/DeepSeek-V4-Pro-FP8/hc_ffn", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.hc_attn", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.hc_ffn" + ] + }, + { + "type": "gemm", + "args": { + "m": 2240, + "n": 64, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.weights_proj", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.weights_proj", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.weights_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 2240, + "n": 256, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.compressor.wkv", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.compressor.wkv", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.compressor.wgate", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.compressor.wkv" + ] + }, + { + "type": "gemm", + "args": { + "m": 2240, + "n": 384, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.gate", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.gate", + "deepseek-ai/DeepSeek-V4-Pro/ffn.gate", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.gate", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.gate", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.gate" + ] + }, + { + "type": "gemm", + "args": { + "m": 2240, + "n": 384, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w3" + ] + }, + { + "type": "gemm", + "args": { + "m": 2240, + "n": 512, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wkv", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wkv", + "deepseek-ai/DeepSeek-V4-Pro/attn.wkv", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wkv", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wkv", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wkv" + ] + }, + { + "type": "gemm", + "args": { + "m": 2240, + "n": 512, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.compressor.wkv+wgate", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.compressor.wkv+wgate", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.compressor.wkv+wgate" + ] + }, + { + "type": "gemm", + "args": { + "m": 2240, + "n": 1024, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.compressor.wkv", + "deepseek-ai/DeepSeek-V4-Pro/attn.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro/attn.compressor.wkv", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.compressor.wgate", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.compressor.wkv" + ] + }, + { + "type": "gemm", + "args": { + "m": 2240, + "n": 1536, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w3" + ] + }, + { + "type": "gemm", + "args": { + "m": 2240, + "n": 1792, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 2240, + "n": 7168, + "k": 192, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w2" + ] + }, + { + "type": "gemm", + "args": { + "m": 2240, + "n": 7168, + "k": 768, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w2" + ] + }, + { + "type": "gemm", + "args": { + "m": 2240, + "n": 7168, + "k": 896, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 2240, + "n": 7168, + "k": 4096, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 2240, + "n": 8192, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 2240, + "n": 16384, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 2248, + "n": 384, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w3" + ] + }, + { + "type": "gemm", + "args": { + "m": 2248, + "n": 7168, + "k": 192, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w2" + ] + }, + { + "type": "gemm", + "args": { + "m": 2272, + "n": 1792, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 2272, + "n": 7168, + "k": 896, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 2272, + "n": 7168, + "k": 2048, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 2304, + "n": 24, + "k": 28672, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/hc_attn", + "deepseek-ai/DeepSeek-V4-Pro-0813/hc_ffn", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.hc_attn", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.hc_ffn", + "deepseek-ai/DeepSeek-V4-Pro/hc_attn", + "deepseek-ai/DeepSeek-V4-Pro/hc_ffn", + "deepseek-ai/DeepSeek-V4-Pro/mtp.hc_attn", + "deepseek-ai/DeepSeek-V4-Pro/mtp.hc_ffn", + "sgl-project/DeepSeek-V4-Pro-FP8/hc_attn", + "sgl-project/DeepSeek-V4-Pro-FP8/hc_ffn", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.hc_attn", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.hc_ffn" + ] + }, + { + "type": "gemm", + "args": { + "m": 2304, + "n": 64, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.weights_proj", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.weights_proj", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.weights_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 2304, + "n": 256, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.compressor.wkv", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.compressor.wkv", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.compressor.wgate", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.compressor.wkv" + ] + }, + { + "type": "gemm", + "args": { + "m": 2304, + "n": 384, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.gate", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.gate", + "deepseek-ai/DeepSeek-V4-Pro/ffn.gate", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.gate", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.gate", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.gate" + ] + }, + { + "type": "gemm", + "args": { + "m": 2304, + "n": 384, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w3" + ] + }, + { + "type": "gemm", + "args": { + "m": 2304, + "n": 512, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wkv", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wkv", + "deepseek-ai/DeepSeek-V4-Pro/attn.wkv", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wkv", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wkv", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wkv" + ] + }, + { + "type": "gemm", + "args": { + "m": 2304, + "n": 512, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.compressor.wkv+wgate", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.compressor.wkv+wgate", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.compressor.wkv+wgate" + ] + }, + { + "type": "gemm", + "args": { + "m": 2304, + "n": 1024, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.compressor.wkv", + "deepseek-ai/DeepSeek-V4-Pro/attn.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro/attn.compressor.wkv", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.compressor.wgate", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.compressor.wkv" + ] + }, + { + "type": "gemm", + "args": { + "m": 2304, + "n": 1536, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w3" + ] + }, + { + "type": "gemm", + "args": { + "m": 2304, + "n": 1792, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 2304, + "n": 6144, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3" + ] + }, + { + "type": "gemm", + "args": { + "m": 2304, + "n": 7168, + "k": 192, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w2" + ] + }, + { + "type": "gemm", + "args": { + "m": 2304, + "n": 7168, + "k": 768, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w2" + ] + }, + { + "type": "gemm", + "args": { + "m": 2304, + "n": 7168, + "k": 896, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 2304, + "n": 7168, + "k": 2048, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 2304, + "n": 7168, + "k": 3072, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w2" + ] + }, + { + "type": "gemm", + "args": { + "m": 2304, + "n": 7168, + "k": 4096, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 2304, + "n": 7168, + "k": 16384, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 2304, + "n": 8192, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 2304, + "n": 16384, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 2304, + "n": 65536, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 2306, + "n": 6144, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3" + ] + }, + { + "type": "gemm", + "args": { + "m": 2306, + "n": 7168, + "k": 3072, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w2" + ] + }, + { + "type": "gemm", + "args": { + "m": 2306, + "n": 7168, + "k": 16384, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 2306, + "n": 65536, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 2320, + "n": 384, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w3" + ] + }, + { + "type": "gemm", + "args": { + "m": 2320, + "n": 7168, + "k": 192, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w2" + ] + }, + { + "type": "gemm", + "args": { + "m": 2321, + "n": 24, + "k": 49152, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 2321, + "n": 64, + "k": 12288, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 2321, + "n": 120, + "k": 12288, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 2321, + "n": 256, + "k": 12288, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 2321, + "n": 512, + "k": 12288, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 2321, + "n": 512, + "k": 12288, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 2321, + "n": 1024, + "k": 12288, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 2321, + "n": 1536, + "k": 12288, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 2321, + "n": 5120, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 2321, + "n": 12288, + "k": 1024, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 2335, + "n": 10240, + "k": 16384, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 2368, + "n": 24, + "k": 49152, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 2368, + "n": 64, + "k": 12288, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 2368, + "n": 120, + "k": 12288, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 2368, + "n": 256, + "k": 12288, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 2368, + "n": 512, + "k": 12288, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 2368, + "n": 512, + "k": 12288, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 2368, + "n": 1024, + "k": 12288, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 2368, + "n": 1536, + "k": 12288, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 2368, + "n": 2048, + "k": 12288, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 2368, + "n": 5120, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 2368, + "n": 6144, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3" + ] + }, + { + "type": "gemm", + "args": { + "m": 2368, + "n": 7168, + "k": 3072, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w2" + ] + }, + { + "type": "gemm", + "args": { + "m": 2368, + "n": 7168, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.e_proj", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.h_proj", + "deepseek-ai/DeepSeek-V4-Pro/mtp.e_proj", + "deepseek-ai/DeepSeek-V4-Pro/mtp.h_proj", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.e_proj", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.h_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 2368, + "n": 7168, + "k": 16384, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 2368, + "n": 10240, + "k": 16384, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 2368, + "n": 12288, + "k": 1024, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 2368, + "n": 12288, + "k": 8192, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 2368, + "n": 12288, + "k": 16384, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 2368, + "n": 14336, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 2368, + "n": 16384, + "k": 12288, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 2368, + "n": 65536, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 2368, + "n": 81920, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 2376, + "n": 24, + "k": 49152, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 2376, + "n": 64, + "k": 12288, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 2376, + "n": 120, + "k": 12288, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 2376, + "n": 256, + "k": 12288, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 2376, + "n": 512, + "k": 12288, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 2376, + "n": 512, + "k": 12288, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 2376, + "n": 1024, + "k": 12288, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 2376, + "n": 1536, + "k": 12288, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 2376, + "n": 5120, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 2376, + "n": 12288, + "k": 1024, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 2420, + "n": 384, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w3" + ] + }, + { + "type": "gemm", + "args": { + "m": 2420, + "n": 7168, + "k": 192, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w2" + ] + }, + { + "type": "gemm", + "args": { + "m": 2432, + "n": 24, + "k": 49152, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 2432, + "n": 64, + "k": 12288, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 2432, + "n": 120, + "k": 12288, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 2432, + "n": 256, + "k": 12288, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 2432, + "n": 512, + "k": 12288, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 2432, + "n": 512, + "k": 12288, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 2432, + "n": 768, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w3" + ] + }, + { + "type": "gemm", + "args": { + "m": 2432, + "n": 1024, + "k": 10240, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 2432, + "n": 1024, + "k": 12288, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 2432, + "n": 1536, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w3" + ] + }, + { + "type": "gemm", + "args": { + "m": 2432, + "n": 1536, + "k": 12288, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 2432, + "n": 2048, + "k": 12288, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 2432, + "n": 5120, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 2432, + "n": 6144, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3" + ] + }, + { + "type": "gemm", + "args": { + "m": 2432, + "n": 7168, + "k": 384, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w2" + ] + }, + { + "type": "gemm", + "args": { + "m": 2432, + "n": 7168, + "k": 768, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w2" + ] + }, + { + "type": "gemm", + "args": { + "m": 2432, + "n": 7168, + "k": 3072, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w2" + ] + }, + { + "type": "gemm", + "args": { + "m": 2432, + "n": 7168, + "k": 4096, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 2432, + "n": 7168, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.e_proj", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.h_proj", + "deepseek-ai/DeepSeek-V4-Pro/mtp.e_proj", + "deepseek-ai/DeepSeek-V4-Pro/mtp.h_proj", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.e_proj", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.h_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 2432, + "n": 7168, + "k": 16384, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 2432, + "n": 10240, + "k": 512, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 2432, + "n": 10240, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 2432, + "n": 10240, + "k": 16384, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 2432, + "n": 12288, + "k": 1024, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 2432, + "n": 12288, + "k": 2048, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 2432, + "n": 12288, + "k": 8192, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 2432, + "n": 12288, + "k": 16384, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 2432, + "n": 14336, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 2432, + "n": 16384, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 2432, + "n": 16384, + "k": 12288, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 2432, + "n": 65536, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 2432, + "n": 81920, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 2433, + "n": 1024, + "k": 10240, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 2433, + "n": 10240, + "k": 512, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 2433, + "n": 10240, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 2433, + "n": 12288, + "k": 2048, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 2448, + "n": 24, + "k": 49152, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 2448, + "n": 64, + "k": 12288, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 2448, + "n": 120, + "k": 12288, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 2448, + "n": 256, + "k": 12288, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 2448, + "n": 512, + "k": 12288, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 2448, + "n": 512, + "k": 12288, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 2448, + "n": 1024, + "k": 12288, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 2448, + "n": 1536, + "k": 12288, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 2448, + "n": 1792, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 2448, + "n": 5120, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 2448, + "n": 7168, + "k": 896, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 2448, + "n": 12288, + "k": 1024, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 2484, + "n": 4096, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 2484, + "n": 7168, + "k": 1024, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 2496, + "n": 1024, + "k": 10240, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 2496, + "n": 4096, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 2496, + "n": 7168, + "k": 1024, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 2496, + "n": 10240, + "k": 512, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 2496, + "n": 10240, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 2496, + "n": 12288, + "k": 2048, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 2502, + "n": 320, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 2504, + "n": 320, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 2504, + "n": 1024, + "k": 10240, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 2504, + "n": 4096, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 2504, + "n": 7168, + "k": 1024, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 2504, + "n": 10240, + "k": 512, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 2507, + "n": 24, + "k": 40960, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 2507, + "n": 64, + "k": 10240, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 2507, + "n": 192, + "k": 10240, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 2507, + "n": 256, + "k": 10240, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 2507, + "n": 512, + "k": 10240, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 2507, + "n": 512, + "k": 10240, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 2507, + "n": 1024, + "k": 10240, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 2507, + "n": 1536, + "k": 10240, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 2507, + "n": 10240, + "k": 1024, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 2509, + "n": 896, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 2509, + "n": 7168, + "k": 448, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 2509, + "n": 10240, + "k": 2048, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 2512, + "n": 320, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 2512, + "n": 896, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 2512, + "n": 4096, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 2512, + "n": 7168, + "k": 448, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 2512, + "n": 7168, + "k": 1024, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 2528, + "n": 24, + "k": 49152, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 2528, + "n": 64, + "k": 12288, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 2528, + "n": 120, + "k": 12288, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 2528, + "n": 256, + "k": 12288, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 2528, + "n": 512, + "k": 12288, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 2528, + "n": 512, + "k": 12288, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 2528, + "n": 1024, + "k": 12288, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 2528, + "n": 1536, + "k": 12288, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 2528, + "n": 5120, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 2528, + "n": 12288, + "k": 1024, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 2560, + "n": 24, + "k": 40960, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 2560, + "n": 24, + "k": 49152, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 2560, + "n": 64, + "k": 10240, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 2560, + "n": 64, + "k": 12288, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 2560, + "n": 120, + "k": 12288, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 2560, + "n": 192, + "k": 10240, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 2560, + "n": 256, + "k": 10240, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 2560, + "n": 256, + "k": 12288, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 2560, + "n": 320, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 2560, + "n": 512, + "k": 10240, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 2560, + "n": 512, + "k": 10240, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 2560, + "n": 512, + "k": 12288, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 2560, + "n": 512, + "k": 12288, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 2560, + "n": 768, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w3" + ] + }, + { + "type": "gemm", + "args": { + "m": 2560, + "n": 1024, + "k": 10240, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 2560, + "n": 1024, + "k": 10240, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 2560, + "n": 1024, + "k": 12288, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 2560, + "n": 1536, + "k": 10240, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 2560, + "n": 1536, + "k": 12288, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 2560, + "n": 2048, + "k": 12288, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 2560, + "n": 4096, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 2560, + "n": 5120, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 2560, + "n": 6144, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3" + ] + }, + { + "type": "gemm", + "args": { + "m": 2560, + "n": 7168, + "k": 384, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w2" + ] + }, + { + "type": "gemm", + "args": { + "m": 2560, + "n": 7168, + "k": 1024, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 2560, + "n": 7168, + "k": 3072, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w2" + ] + }, + { + "type": "gemm", + "args": { + "m": 2560, + "n": 7168, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.e_proj", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.h_proj", + "deepseek-ai/DeepSeek-V4-Pro/mtp.e_proj", + "deepseek-ai/DeepSeek-V4-Pro/mtp.h_proj", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.e_proj", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.h_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 2560, + "n": 10240, + "k": 512, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 2560, + "n": 10240, + "k": 1024, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 2560, + "n": 10240, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 2560, + "n": 10240, + "k": 2048, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 2560, + "n": 10240, + "k": 16384, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 2560, + "n": 12288, + "k": 1024, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 2560, + "n": 12288, + "k": 2048, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 2560, + "n": 12288, + "k": 8192, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 2560, + "n": 12288, + "k": 16384, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 2560, + "n": 14336, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 2560, + "n": 16384, + "k": 12288, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 2560, + "n": 81920, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 2568, + "n": 24, + "k": 40960, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 2568, + "n": 64, + "k": 10240, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 2568, + "n": 192, + "k": 10240, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 2568, + "n": 256, + "k": 10240, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 2568, + "n": 512, + "k": 10240, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 2568, + "n": 512, + "k": 10240, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 2568, + "n": 1024, + "k": 10240, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 2568, + "n": 1536, + "k": 10240, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 2568, + "n": 10240, + "k": 1024, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 2568, + "n": 10240, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 2568, + "n": 12288, + "k": 2048, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 2576, + "n": 24, + "k": 40960, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 2576, + "n": 64, + "k": 10240, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 2576, + "n": 192, + "k": 10240, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 2576, + "n": 256, + "k": 10240, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 2576, + "n": 512, + "k": 10240, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 2576, + "n": 512, + "k": 10240, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 2576, + "n": 1024, + "k": 10240, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 2576, + "n": 1536, + "k": 10240, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 2576, + "n": 10240, + "k": 1024, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 2576, + "n": 10240, + "k": 2048, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 2592, + "n": 320, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 2592, + "n": 896, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 2592, + "n": 4096, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 2592, + "n": 7168, + "k": 448, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 2592, + "n": 7168, + "k": 1024, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 2624, + "n": 24, + "k": 40960, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 2624, + "n": 64, + "k": 10240, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 2624, + "n": 192, + "k": 10240, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 2624, + "n": 256, + "k": 10240, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 2624, + "n": 320, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 2624, + "n": 512, + "k": 10240, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 2624, + "n": 512, + "k": 10240, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 2624, + "n": 1024, + "k": 10240, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 2624, + "n": 1536, + "k": 10240, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 2624, + "n": 10240, + "k": 1024, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 2624, + "n": 10240, + "k": 2048, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 2632, + "n": 10240, + "k": 2048, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 2641, + "n": 768, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w3" + ] + }, + { + "type": "gemm", + "args": { + "m": 2641, + "n": 7168, + "k": 384, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w2" + ] + }, + { + "type": "gemm", + "args": { + "m": 2641, + "n": 7168, + "k": 2048, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 2648, + "n": 896, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 2648, + "n": 7168, + "k": 448, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 2671, + "n": 8192, + "k": 10240, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 2671, + "n": 10240, + "k": 4096, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 2688, + "n": 24, + "k": 40960, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 2688, + "n": 64, + "k": 10240, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 2688, + "n": 192, + "k": 10240, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 2688, + "n": 256, + "k": 10240, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 2688, + "n": 320, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 2688, + "n": 512, + "k": 10240, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 2688, + "n": 512, + "k": 10240, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 2688, + "n": 768, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w3" + ] + }, + { + "type": "gemm", + "args": { + "m": 2688, + "n": 1024, + "k": 10240, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 2688, + "n": 1536, + "k": 10240, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 2688, + "n": 4096, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 2688, + "n": 7168, + "k": 384, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w2" + ] + }, + { + "type": "gemm", + "args": { + "m": 2688, + "n": 7168, + "k": 1024, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 2688, + "n": 7168, + "k": 2048, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 2688, + "n": 10240, + "k": 1024, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 2688, + "n": 10240, + "k": 2048, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 2696, + "n": 768, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w3" + ] + }, + { + "type": "gemm", + "args": { + "m": 2696, + "n": 7168, + "k": 384, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w2" + ] + }, + { + "type": "gemm", + "args": { + "m": 2696, + "n": 7168, + "k": 2048, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 2711, + "n": 24, + "k": 28672, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/hc_attn", + "deepseek-ai/DeepSeek-V4-Pro-0813/hc_ffn", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.hc_attn", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.hc_ffn", + "deepseek-ai/DeepSeek-V4-Pro/hc_attn", + "deepseek-ai/DeepSeek-V4-Pro/hc_ffn", + "deepseek-ai/DeepSeek-V4-Pro/mtp.hc_attn", + "deepseek-ai/DeepSeek-V4-Pro/mtp.hc_ffn", + "sgl-project/DeepSeek-V4-Pro-FP8/hc_attn", + "sgl-project/DeepSeek-V4-Pro-FP8/hc_ffn", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.hc_attn", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.hc_ffn" + ] + }, + { + "type": "gemm", + "args": { + "m": 2711, + "n": 64, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.weights_proj", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.weights_proj", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.weights_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 2711, + "n": 256, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.compressor.wkv", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.compressor.wkv", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.compressor.wgate", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.compressor.wkv" + ] + }, + { + "type": "gemm", + "args": { + "m": 2711, + "n": 512, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wkv", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wkv", + "deepseek-ai/DeepSeek-V4-Pro/attn.wkv", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wkv", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wkv", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wkv" + ] + }, + { + "type": "gemm", + "args": { + "m": 2711, + "n": 512, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.compressor.wkv+wgate", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.compressor.wkv+wgate", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.compressor.wkv+wgate" + ] + }, + { + "type": "gemm", + "args": { + "m": 2711, + "n": 1024, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.compressor.wkv", + "deepseek-ai/DeepSeek-V4-Pro/attn.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro/attn.compressor.wkv", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.compressor.wgate", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.compressor.wkv" + ] + }, + { + "type": "gemm", + "args": { + "m": 2711, + "n": 1536, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w3" + ] + }, + { + "type": "gemm", + "args": { + "m": 2711, + "n": 8192, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 2712, + "n": 24, + "k": 28672, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/hc_attn", + "deepseek-ai/DeepSeek-V4-Pro-0813/hc_ffn", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.hc_attn", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.hc_ffn", + "deepseek-ai/DeepSeek-V4-Pro/hc_attn", + "deepseek-ai/DeepSeek-V4-Pro/hc_ffn", + "deepseek-ai/DeepSeek-V4-Pro/mtp.hc_attn", + "deepseek-ai/DeepSeek-V4-Pro/mtp.hc_ffn", + "sgl-project/DeepSeek-V4-Pro-FP8/hc_attn", + "sgl-project/DeepSeek-V4-Pro-FP8/hc_ffn", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.hc_attn", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.hc_ffn" + ] + }, + { + "type": "gemm", + "args": { + "m": 2712, + "n": 64, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.weights_proj", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.weights_proj", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.weights_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 2712, + "n": 256, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.compressor.wkv", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.compressor.wkv", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.compressor.wgate", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.compressor.wkv" + ] + }, + { + "type": "gemm", + "args": { + "m": 2712, + "n": 512, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wkv", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wkv", + "deepseek-ai/DeepSeek-V4-Pro/attn.wkv", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wkv", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wkv", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wkv" + ] + }, + { + "type": "gemm", + "args": { + "m": 2712, + "n": 512, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.compressor.wkv+wgate", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.compressor.wkv+wgate", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.compressor.wkv+wgate" + ] + }, + { + "type": "gemm", + "args": { + "m": 2712, + "n": 1024, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.compressor.wkv", + "deepseek-ai/DeepSeek-V4-Pro/attn.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro/attn.compressor.wkv", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.compressor.wgate", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.compressor.wkv" + ] + }, + { + "type": "gemm", + "args": { + "m": 2712, + "n": 1536, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w3" + ] + }, + { + "type": "gemm", + "args": { + "m": 2712, + "n": 8192, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 2718, + "n": 384, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.gate", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.gate", + "deepseek-ai/DeepSeek-V4-Pro/ffn.gate", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.gate", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.gate", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.gate" + ] + }, + { + "type": "gemm", + "args": { + "m": 2720, + "n": 24, + "k": 28672, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/hc_attn", + "deepseek-ai/DeepSeek-V4-Pro-0813/hc_ffn", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.hc_attn", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.hc_ffn", + "deepseek-ai/DeepSeek-V4-Pro/hc_attn", + "deepseek-ai/DeepSeek-V4-Pro/hc_ffn", + "deepseek-ai/DeepSeek-V4-Pro/mtp.hc_attn", + "deepseek-ai/DeepSeek-V4-Pro/mtp.hc_ffn", + "sgl-project/DeepSeek-V4-Pro-FP8/hc_attn", + "sgl-project/DeepSeek-V4-Pro-FP8/hc_ffn", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.hc_attn", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.hc_ffn" + ] + }, + { + "type": "gemm", + "args": { + "m": 2720, + "n": 64, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.weights_proj", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.weights_proj", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.weights_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 2720, + "n": 256, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.compressor.wkv", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.compressor.wkv", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.compressor.wgate", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.compressor.wkv" + ] + }, + { + "type": "gemm", + "args": { + "m": 2720, + "n": 384, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.gate", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.gate", + "deepseek-ai/DeepSeek-V4-Pro/ffn.gate", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.gate", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.gate", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.gate" + ] + }, + { + "type": "gemm", + "args": { + "m": 2720, + "n": 512, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wkv", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wkv", + "deepseek-ai/DeepSeek-V4-Pro/attn.wkv", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wkv", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wkv", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wkv" + ] + }, + { + "type": "gemm", + "args": { + "m": 2720, + "n": 512, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.compressor.wkv+wgate", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.compressor.wkv+wgate", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.compressor.wkv+wgate" + ] + }, + { + "type": "gemm", + "args": { + "m": 2720, + "n": 1024, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.compressor.wkv", + "deepseek-ai/DeepSeek-V4-Pro/attn.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro/attn.compressor.wkv", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.compressor.wgate", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.compressor.wkv" + ] + }, + { + "type": "gemm", + "args": { + "m": 2720, + "n": 1536, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w3" + ] + }, + { + "type": "gemm", + "args": { + "m": 2720, + "n": 8192, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 2728, + "n": 384, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.gate", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.gate", + "deepseek-ai/DeepSeek-V4-Pro/ffn.gate", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.gate", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.gate", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.gate" + ] + }, + { + "type": "gemm", + "args": { + "m": 2730, + "n": 384, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w3" + ] + }, + { + "type": "gemm", + "args": { + "m": 2730, + "n": 7168, + "k": 192, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w2" + ] + }, + { + "type": "gemm", + "args": { + "m": 2731, + "n": 1536, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w3" + ] + }, + { + "type": "gemm", + "args": { + "m": 2731, + "n": 7168, + "k": 768, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w2" + ] + }, + { + "type": "gemm", + "args": { + "m": 2731, + "n": 7168, + "k": 4096, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 2731, + "n": 16384, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 2732, + "n": 2048, + "k": 10240, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 2732, + "n": 10240, + "k": 1024, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 2732, + "n": 10240, + "k": 4096, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 2736, + "n": 24, + "k": 28672, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/hc_attn", + "deepseek-ai/DeepSeek-V4-Pro-0813/hc_ffn", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.hc_attn", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.hc_ffn", + "deepseek-ai/DeepSeek-V4-Pro/hc_attn", + "deepseek-ai/DeepSeek-V4-Pro/hc_ffn", + "deepseek-ai/DeepSeek-V4-Pro/mtp.hc_attn", + "deepseek-ai/DeepSeek-V4-Pro/mtp.hc_ffn", + "sgl-project/DeepSeek-V4-Pro-FP8/hc_attn", + "sgl-project/DeepSeek-V4-Pro-FP8/hc_ffn", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.hc_attn", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.hc_ffn" + ] + }, + { + "type": "gemm", + "args": { + "m": 2736, + "n": 64, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.weights_proj", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.weights_proj", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.weights_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 2736, + "n": 256, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.compressor.wkv", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.compressor.wkv", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.compressor.wgate", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.compressor.wkv" + ] + }, + { + "type": "gemm", + "args": { + "m": 2736, + "n": 384, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.gate", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.gate", + "deepseek-ai/DeepSeek-V4-Pro/ffn.gate", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.gate", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.gate", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.gate" + ] + }, + { + "type": "gemm", + "args": { + "m": 2736, + "n": 512, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wkv", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wkv", + "deepseek-ai/DeepSeek-V4-Pro/attn.wkv", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wkv", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wkv", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wkv" + ] + }, + { + "type": "gemm", + "args": { + "m": 2736, + "n": 512, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.compressor.wkv+wgate", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.compressor.wkv+wgate", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.compressor.wkv+wgate" + ] + }, + { + "type": "gemm", + "args": { + "m": 2736, + "n": 1024, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.compressor.wkv", + "deepseek-ai/DeepSeek-V4-Pro/attn.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro/attn.compressor.wkv", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.compressor.wgate", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.compressor.wkv" + ] + }, + { + "type": "gemm", + "args": { + "m": 2736, + "n": 1536, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w3" + ] + }, + { + "type": "gemm", + "args": { + "m": 2736, + "n": 7168, + "k": 768, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w2" + ] + }, + { + "type": "gemm", + "args": { + "m": 2736, + "n": 7168, + "k": 4096, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 2736, + "n": 8192, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 2736, + "n": 16384, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 2744, + "n": 1536, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w3" + ] + }, + { + "type": "gemm", + "args": { + "m": 2744, + "n": 7168, + "k": 768, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w2" + ] + }, + { + "type": "gemm", + "args": { + "m": 2744, + "n": 7168, + "k": 4096, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 2744, + "n": 16384, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 2752, + "n": 24, + "k": 28672, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/hc_attn", + "deepseek-ai/DeepSeek-V4-Pro-0813/hc_ffn", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.hc_attn", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.hc_ffn", + "deepseek-ai/DeepSeek-V4-Pro/hc_attn", + "deepseek-ai/DeepSeek-V4-Pro/hc_ffn", + "deepseek-ai/DeepSeek-V4-Pro/mtp.hc_attn", + "deepseek-ai/DeepSeek-V4-Pro/mtp.hc_ffn", + "sgl-project/DeepSeek-V4-Pro-FP8/hc_attn", + "sgl-project/DeepSeek-V4-Pro-FP8/hc_ffn", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.hc_attn", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.hc_ffn" + ] + }, + { + "type": "gemm", + "args": { + "m": 2752, + "n": 64, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.weights_proj", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.weights_proj", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.weights_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 2752, + "n": 256, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.compressor.wkv", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.compressor.wkv", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.compressor.wgate", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.compressor.wkv" + ] + }, + { + "type": "gemm", + "args": { + "m": 2752, + "n": 384, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.gate", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.gate", + "deepseek-ai/DeepSeek-V4-Pro/ffn.gate", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.gate", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.gate", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.gate" + ] + }, + { + "type": "gemm", + "args": { + "m": 2752, + "n": 384, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w3" + ] + }, + { + "type": "gemm", + "args": { + "m": 2752, + "n": 512, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wkv", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wkv", + "deepseek-ai/DeepSeek-V4-Pro/attn.wkv", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wkv", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wkv", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wkv" + ] + }, + { + "type": "gemm", + "args": { + "m": 2752, + "n": 512, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.compressor.wkv+wgate", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.compressor.wkv+wgate", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.compressor.wkv+wgate" + ] + }, + { + "type": "gemm", + "args": { + "m": 2752, + "n": 1024, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.compressor.wkv", + "deepseek-ai/DeepSeek-V4-Pro/attn.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro/attn.compressor.wkv", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.compressor.wgate", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.compressor.wkv" + ] + }, + { + "type": "gemm", + "args": { + "m": 2752, + "n": 1536, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w3" + ] + }, + { + "type": "gemm", + "args": { + "m": 2752, + "n": 1792, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 2752, + "n": 7168, + "k": 192, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w2" + ] + }, + { + "type": "gemm", + "args": { + "m": 2752, + "n": 7168, + "k": 768, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w2" + ] + }, + { + "type": "gemm", + "args": { + "m": 2752, + "n": 7168, + "k": 896, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 2752, + "n": 7168, + "k": 2048, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 2752, + "n": 7168, + "k": 4096, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 2752, + "n": 8192, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 2752, + "n": 16384, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 2754, + "n": 1792, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 2754, + "n": 7168, + "k": 896, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 2760, + "n": 384, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w3" + ] + }, + { + "type": "gemm", + "args": { + "m": 2760, + "n": 1792, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 2760, + "n": 7168, + "k": 192, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w2" + ] + }, + { + "type": "gemm", + "args": { + "m": 2760, + "n": 7168, + "k": 896, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 2768, + "n": 768, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w3" + ] + }, + { + "type": "gemm", + "args": { + "m": 2768, + "n": 1792, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 2768, + "n": 7168, + "k": 384, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w2" + ] + }, + { + "type": "gemm", + "args": { + "m": 2768, + "n": 7168, + "k": 896, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 2768, + "n": 7168, + "k": 2048, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 2784, + "n": 1536, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w3" + ] + }, + { + "type": "gemm", + "args": { + "m": 2784, + "n": 7168, + "k": 768, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w2" + ] + }, + { + "type": "gemm", + "args": { + "m": 2784, + "n": 7168, + "k": 4096, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 2784, + "n": 16384, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 2816, + "n": 24, + "k": 28672, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/hc_attn", + "deepseek-ai/DeepSeek-V4-Pro-0813/hc_ffn", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.hc_attn", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.hc_ffn", + "deepseek-ai/DeepSeek-V4-Pro/hc_attn", + "deepseek-ai/DeepSeek-V4-Pro/hc_ffn", + "deepseek-ai/DeepSeek-V4-Pro/mtp.hc_attn", + "deepseek-ai/DeepSeek-V4-Pro/mtp.hc_ffn", + "sgl-project/DeepSeek-V4-Pro-FP8/hc_attn", + "sgl-project/DeepSeek-V4-Pro-FP8/hc_ffn", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.hc_attn", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.hc_ffn" + ] + }, + { + "type": "gemm", + "args": { + "m": 2816, + "n": 64, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.weights_proj", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.weights_proj", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.weights_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 2816, + "n": 256, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.compressor.wkv", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.compressor.wkv", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.compressor.wgate", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.compressor.wkv" + ] + }, + { + "type": "gemm", + "args": { + "m": 2816, + "n": 384, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.gate", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.gate", + "deepseek-ai/DeepSeek-V4-Pro/ffn.gate", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.gate", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.gate", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.gate" + ] + }, + { + "type": "gemm", + "args": { + "m": 2816, + "n": 384, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w3" + ] + }, + { + "type": "gemm", + "args": { + "m": 2816, + "n": 512, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wkv", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wkv", + "deepseek-ai/DeepSeek-V4-Pro/attn.wkv", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wkv", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wkv", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wkv" + ] + }, + { + "type": "gemm", + "args": { + "m": 2816, + "n": 512, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.compressor.wkv+wgate", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.compressor.wkv+wgate", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.compressor.wkv+wgate" + ] + }, + { + "type": "gemm", + "args": { + "m": 2816, + "n": 1024, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.compressor.wkv", + "deepseek-ai/DeepSeek-V4-Pro/attn.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro/attn.compressor.wkv", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.compressor.wgate", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.compressor.wkv" + ] + }, + { + "type": "gemm", + "args": { + "m": 2816, + "n": 1536, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w3" + ] + }, + { + "type": "gemm", + "args": { + "m": 2816, + "n": 1792, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 2816, + "n": 7168, + "k": 192, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w2" + ] + }, + { + "type": "gemm", + "args": { + "m": 2816, + "n": 7168, + "k": 768, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w2" + ] + }, + { + "type": "gemm", + "args": { + "m": 2816, + "n": 7168, + "k": 896, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 2816, + "n": 7168, + "k": 2048, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 2816, + "n": 7168, + "k": 4096, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 2816, + "n": 8192, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 2816, + "n": 16384, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 2880, + "n": 768, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w3" + ] + }, + { + "type": "gemm", + "args": { + "m": 2880, + "n": 896, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 2880, + "n": 7168, + "k": 384, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w2" + ] + }, + { + "type": "gemm", + "args": { + "m": 2880, + "n": 7168, + "k": 448, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 2896, + "n": 1024, + "k": 10240, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 2896, + "n": 10240, + "k": 512, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 2921, + "n": 24, + "k": 49152, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 2921, + "n": 64, + "k": 12288, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 2921, + "n": 120, + "k": 12288, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 2921, + "n": 256, + "k": 12288, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 2921, + "n": 512, + "k": 12288, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 2921, + "n": 512, + "k": 12288, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 2921, + "n": 1024, + "k": 12288, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 2921, + "n": 1536, + "k": 12288, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 2921, + "n": 5120, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 2921, + "n": 12288, + "k": 1024, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 2944, + "n": 24, + "k": 28672, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/hc_attn", + "deepseek-ai/DeepSeek-V4-Pro-0813/hc_ffn", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.hc_attn", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.hc_ffn", + "deepseek-ai/DeepSeek-V4-Pro/hc_attn", + "deepseek-ai/DeepSeek-V4-Pro/hc_ffn", + "deepseek-ai/DeepSeek-V4-Pro/mtp.hc_attn", + "deepseek-ai/DeepSeek-V4-Pro/mtp.hc_ffn", + "sgl-project/DeepSeek-V4-Pro-FP8/hc_attn", + "sgl-project/DeepSeek-V4-Pro-FP8/hc_ffn", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.hc_attn", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.hc_ffn" + ] + }, + { + "type": "gemm", + "args": { + "m": 2944, + "n": 24, + "k": 49152, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 2944, + "n": 64, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.weights_proj", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.weights_proj", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.weights_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 2944, + "n": 64, + "k": 12288, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 2944, + "n": 120, + "k": 12288, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 2944, + "n": 256, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.compressor.wkv", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.compressor.wkv", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.compressor.wgate", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.compressor.wkv" + ] + }, + { + "type": "gemm", + "args": { + "m": 2944, + "n": 256, + "k": 12288, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 2944, + "n": 384, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.gate", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.gate", + "deepseek-ai/DeepSeek-V4-Pro/ffn.gate", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.gate", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.gate", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.gate" + ] + }, + { + "type": "gemm", + "args": { + "m": 2944, + "n": 384, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w3" + ] + }, + { + "type": "gemm", + "args": { + "m": 2944, + "n": 512, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wkv", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wkv", + "deepseek-ai/DeepSeek-V4-Pro/attn.wkv", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wkv", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wkv", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wkv" + ] + }, + { + "type": "gemm", + "args": { + "m": 2944, + "n": 512, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.compressor.wkv+wgate", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.compressor.wkv+wgate", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.compressor.wkv+wgate" + ] + }, + { + "type": "gemm", + "args": { + "m": 2944, + "n": 512, + "k": 12288, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 2944, + "n": 512, + "k": 12288, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 2944, + "n": 1024, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.compressor.wkv", + "deepseek-ai/DeepSeek-V4-Pro/attn.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro/attn.compressor.wkv", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.compressor.wgate", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.compressor.wkv" + ] + }, + { + "type": "gemm", + "args": { + "m": 2944, + "n": 1024, + "k": 12288, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 2944, + "n": 1536, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w3" + ] + }, + { + "type": "gemm", + "args": { + "m": 2944, + "n": 1536, + "k": 12288, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 2944, + "n": 1792, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 2944, + "n": 5120, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 2944, + "n": 6144, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3" + ] + }, + { + "type": "gemm", + "args": { + "m": 2944, + "n": 7168, + "k": 192, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w2" + ] + }, + { + "type": "gemm", + "args": { + "m": 2944, + "n": 7168, + "k": 768, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w2" + ] + }, + { + "type": "gemm", + "args": { + "m": 2944, + "n": 7168, + "k": 896, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 2944, + "n": 7168, + "k": 3072, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w2" + ] + }, + { + "type": "gemm", + "args": { + "m": 2944, + "n": 7168, + "k": 4096, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 2944, + "n": 7168, + "k": 16384, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 2944, + "n": 8192, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 2944, + "n": 10240, + "k": 16384, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 2944, + "n": 12288, + "k": 1024, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 2944, + "n": 16384, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 2944, + "n": 65536, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 2952, + "n": 24, + "k": 49152, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 2952, + "n": 64, + "k": 12288, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 2952, + "n": 120, + "k": 12288, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 2952, + "n": 256, + "k": 12288, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 2952, + "n": 512, + "k": 12288, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 2952, + "n": 512, + "k": 12288, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 2952, + "n": 1024, + "k": 12288, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 2952, + "n": 1536, + "k": 12288, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 2952, + "n": 5120, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 2952, + "n": 12288, + "k": 1024, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 3008, + "n": 24, + "k": 49152, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 3008, + "n": 64, + "k": 12288, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 3008, + "n": 120, + "k": 12288, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 3008, + "n": 256, + "k": 12288, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 3008, + "n": 512, + "k": 12288, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 3008, + "n": 512, + "k": 12288, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 3008, + "n": 1024, + "k": 12288, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 3008, + "n": 1536, + "k": 12288, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 3008, + "n": 2048, + "k": 12288, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 3008, + "n": 5120, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 3008, + "n": 6144, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3" + ] + }, + { + "type": "gemm", + "args": { + "m": 3008, + "n": 7168, + "k": 3072, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w2" + ] + }, + { + "type": "gemm", + "args": { + "m": 3008, + "n": 7168, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.e_proj", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.h_proj", + "deepseek-ai/DeepSeek-V4-Pro/mtp.e_proj", + "deepseek-ai/DeepSeek-V4-Pro/mtp.h_proj", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.e_proj", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.h_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 3008, + "n": 7168, + "k": 16384, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 3008, + "n": 10240, + "k": 16384, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 3008, + "n": 12288, + "k": 1024, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 3008, + "n": 12288, + "k": 8192, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 3008, + "n": 12288, + "k": 16384, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 3008, + "n": 14336, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 3008, + "n": 16384, + "k": 12288, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 3008, + "n": 65536, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 3008, + "n": 81920, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 3024, + "n": 24, + "k": 49152, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 3024, + "n": 64, + "k": 12288, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 3024, + "n": 120, + "k": 12288, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 3024, + "n": 256, + "k": 12288, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 3024, + "n": 512, + "k": 12288, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 3024, + "n": 512, + "k": 12288, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 3024, + "n": 1024, + "k": 12288, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 3024, + "n": 1536, + "k": 12288, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 3024, + "n": 5120, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 3024, + "n": 12288, + "k": 1024, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 3072, + "n": 24, + "k": 49152, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 3072, + "n": 64, + "k": 12288, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 3072, + "n": 120, + "k": 12288, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 3072, + "n": 256, + "k": 12288, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 3072, + "n": 512, + "k": 12288, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 3072, + "n": 512, + "k": 12288, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 3072, + "n": 1024, + "k": 10240, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 3072, + "n": 1024, + "k": 12288, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 3072, + "n": 1536, + "k": 12288, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 3072, + "n": 2048, + "k": 12288, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 3072, + "n": 5120, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 3072, + "n": 6144, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3" + ] + }, + { + "type": "gemm", + "args": { + "m": 3072, + "n": 7168, + "k": 3072, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w2" + ] + }, + { + "type": "gemm", + "args": { + "m": 3072, + "n": 7168, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.e_proj", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.h_proj", + "deepseek-ai/DeepSeek-V4-Pro/mtp.e_proj", + "deepseek-ai/DeepSeek-V4-Pro/mtp.h_proj", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.e_proj", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.h_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 3072, + "n": 7168, + "k": 16384, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 3072, + "n": 10240, + "k": 512, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 3072, + "n": 10240, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 3072, + "n": 10240, + "k": 16384, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 3072, + "n": 12288, + "k": 1024, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 3072, + "n": 12288, + "k": 2048, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 3072, + "n": 12288, + "k": 8192, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 3072, + "n": 12288, + "k": 16384, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 3072, + "n": 14336, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 3072, + "n": 16384, + "k": 12288, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 3072, + "n": 65536, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 3072, + "n": 81920, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 3073, + "n": 1024, + "k": 10240, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 3073, + "n": 10240, + "k": 512, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 3080, + "n": 1024, + "k": 10240, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 3080, + "n": 10240, + "k": 512, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 3080, + "n": 10240, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 3080, + "n": 12288, + "k": 2048, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 3081, + "n": 10240, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 3081, + "n": 12288, + "k": 2048, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 3104, + "n": 24, + "k": 40960, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 3104, + "n": 64, + "k": 10240, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 3104, + "n": 192, + "k": 10240, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 3104, + "n": 256, + "k": 10240, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 3104, + "n": 512, + "k": 10240, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 3104, + "n": 512, + "k": 10240, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 3104, + "n": 1024, + "k": 10240, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 3104, + "n": 1536, + "k": 10240, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 3104, + "n": 10240, + "k": 1024, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 3106, + "n": 4096, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 3106, + "n": 7168, + "k": 1024, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 3112, + "n": 4096, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 3136, + "n": 24, + "k": 40960, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 3136, + "n": 64, + "k": 10240, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 3136, + "n": 192, + "k": 10240, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 3136, + "n": 256, + "k": 10240, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 3136, + "n": 320, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 3136, + "n": 512, + "k": 10240, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 3136, + "n": 512, + "k": 10240, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 3136, + "n": 1024, + "k": 10240, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 3136, + "n": 1024, + "k": 10240, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 3136, + "n": 1536, + "k": 10240, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 3136, + "n": 4096, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 3136, + "n": 7168, + "k": 1024, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 3136, + "n": 10240, + "k": 512, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 3136, + "n": 10240, + "k": 1024, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 3136, + "n": 10240, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 3136, + "n": 12288, + "k": 2048, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 3137, + "n": 24, + "k": 40960, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 3137, + "n": 64, + "k": 10240, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 3137, + "n": 192, + "k": 10240, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 3137, + "n": 256, + "k": 10240, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 3137, + "n": 320, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 3137, + "n": 512, + "k": 10240, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 3137, + "n": 512, + "k": 10240, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 3137, + "n": 1024, + "k": 10240, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 3137, + "n": 1536, + "k": 10240, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 3137, + "n": 10240, + "k": 1024, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 3137, + "n": 10240, + "k": 2048, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 3138, + "n": 896, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 3138, + "n": 7168, + "k": 448, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 3138, + "n": 7168, + "k": 16384, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 3138, + "n": 12288, + "k": 8192, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 3138, + "n": 12288, + "k": 16384, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 3138, + "n": 16384, + "k": 12288, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 3138, + "n": 65536, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 3138, + "n": 81920, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 3144, + "n": 24, + "k": 40960, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 3144, + "n": 64, + "k": 10240, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 3144, + "n": 192, + "k": 10240, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 3144, + "n": 256, + "k": 10240, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 3144, + "n": 320, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 3144, + "n": 512, + "k": 10240, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 3144, + "n": 512, + "k": 10240, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 3144, + "n": 1024, + "k": 10240, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 3144, + "n": 1536, + "k": 10240, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 3144, + "n": 7168, + "k": 1024, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 3144, + "n": 10240, + "k": 1024, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 3144, + "n": 10240, + "k": 2048, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 3145, + "n": 6144, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3" + ] + }, + { + "type": "gemm", + "args": { + "m": 3145, + "n": 7168, + "k": 3072, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w2" + ] + }, + { + "type": "gemm", + "args": { + "m": 3152, + "n": 24, + "k": 40960, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 3152, + "n": 64, + "k": 10240, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 3152, + "n": 192, + "k": 10240, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 3152, + "n": 256, + "k": 10240, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 3152, + "n": 512, + "k": 10240, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 3152, + "n": 512, + "k": 10240, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 3152, + "n": 1024, + "k": 10240, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 3152, + "n": 1536, + "k": 10240, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 3152, + "n": 4096, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 3152, + "n": 10240, + "k": 1024, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 3168, + "n": 320, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 3168, + "n": 896, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 3168, + "n": 4096, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 3168, + "n": 7168, + "k": 448, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 3168, + "n": 7168, + "k": 1024, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 3184, + "n": 320, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 3184, + "n": 896, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 3184, + "n": 7168, + "k": 448, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 3184, + "n": 7168, + "k": 1024, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 3200, + "n": 24, + "k": 40960, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 3200, + "n": 64, + "k": 10240, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 3200, + "n": 192, + "k": 10240, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 3200, + "n": 256, + "k": 10240, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 3200, + "n": 320, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 3200, + "n": 512, + "k": 10240, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 3200, + "n": 512, + "k": 10240, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 3200, + "n": 1024, + "k": 10240, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 3200, + "n": 1024, + "k": 10240, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 3200, + "n": 1536, + "k": 10240, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 3200, + "n": 2048, + "k": 12288, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 3200, + "n": 4096, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 3200, + "n": 6144, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3" + ] + }, + { + "type": "gemm", + "args": { + "m": 3200, + "n": 7168, + "k": 1024, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 3200, + "n": 7168, + "k": 3072, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w2" + ] + }, + { + "type": "gemm", + "args": { + "m": 3200, + "n": 7168, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.e_proj", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.h_proj", + "deepseek-ai/DeepSeek-V4-Pro/mtp.e_proj", + "deepseek-ai/DeepSeek-V4-Pro/mtp.h_proj", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.e_proj", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.h_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 3200, + "n": 10240, + "k": 512, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 3200, + "n": 10240, + "k": 1024, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 3200, + "n": 10240, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 3200, + "n": 10240, + "k": 2048, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 3200, + "n": 12288, + "k": 1024, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 3200, + "n": 12288, + "k": 2048, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 3200, + "n": 12288, + "k": 8192, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 3200, + "n": 12288, + "k": 16384, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 3200, + "n": 14336, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 3200, + "n": 16384, + "k": 12288, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 3200, + "n": 81920, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 3208, + "n": 896, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 3208, + "n": 7168, + "k": 448, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 3264, + "n": 896, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 3264, + "n": 7168, + "k": 448, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 3264, + "n": 10240, + "k": 2048, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 3312, + "n": 10240, + "k": 2048, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 3313, + "n": 768, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w3" + ] + }, + { + "type": "gemm", + "args": { + "m": 3313, + "n": 7168, + "k": 384, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w2" + ] + }, + { + "type": "gemm", + "args": { + "m": 3313, + "n": 7168, + "k": 2048, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 3328, + "n": 24, + "k": 40960, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 3328, + "n": 64, + "k": 10240, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 3328, + "n": 192, + "k": 10240, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 3328, + "n": 256, + "k": 10240, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 3328, + "n": 320, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 3328, + "n": 512, + "k": 10240, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 3328, + "n": 512, + "k": 10240, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 3328, + "n": 1024, + "k": 10240, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 3328, + "n": 1536, + "k": 10240, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 3328, + "n": 4096, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 3328, + "n": 7168, + "k": 1024, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 3328, + "n": 7168, + "k": 2048, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 3328, + "n": 10240, + "k": 1024, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 3328, + "n": 10240, + "k": 2048, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 3336, + "n": 768, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w3" + ] + }, + { + "type": "gemm", + "args": { + "m": 3336, + "n": 7168, + "k": 384, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w2" + ] + }, + { + "type": "gemm", + "args": { + "m": 3336, + "n": 7168, + "k": 2048, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 3344, + "n": 768, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w3" + ] + }, + { + "type": "gemm", + "args": { + "m": 3344, + "n": 1024, + "k": 10240, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 3344, + "n": 7168, + "k": 384, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w2" + ] + }, + { + "type": "gemm", + "args": { + "m": 3344, + "n": 7168, + "k": 2048, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 3344, + "n": 10240, + "k": 512, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 3389, + "n": 24, + "k": 28672, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/hc_attn", + "deepseek-ai/DeepSeek-V4-Pro-0813/hc_ffn", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.hc_attn", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.hc_ffn", + "deepseek-ai/DeepSeek-V4-Pro/hc_attn", + "deepseek-ai/DeepSeek-V4-Pro/hc_ffn", + "deepseek-ai/DeepSeek-V4-Pro/mtp.hc_attn", + "deepseek-ai/DeepSeek-V4-Pro/mtp.hc_ffn", + "sgl-project/DeepSeek-V4-Pro-FP8/hc_attn", + "sgl-project/DeepSeek-V4-Pro-FP8/hc_ffn", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.hc_attn", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.hc_ffn" + ] + }, + { + "type": "gemm", + "args": { + "m": 3389, + "n": 64, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.weights_proj", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.weights_proj", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.weights_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 3389, + "n": 256, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.compressor.wkv", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.compressor.wkv", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.compressor.wgate", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.compressor.wkv" + ] + }, + { + "type": "gemm", + "args": { + "m": 3389, + "n": 512, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wkv", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wkv", + "deepseek-ai/DeepSeek-V4-Pro/attn.wkv", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wkv", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wkv", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wkv" + ] + }, + { + "type": "gemm", + "args": { + "m": 3389, + "n": 512, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.compressor.wkv+wgate", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.compressor.wkv+wgate", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.compressor.wkv+wgate" + ] + }, + { + "type": "gemm", + "args": { + "m": 3389, + "n": 1024, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.compressor.wkv", + "deepseek-ai/DeepSeek-V4-Pro/attn.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro/attn.compressor.wkv", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.compressor.wgate", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.compressor.wkv" + ] + }, + { + "type": "gemm", + "args": { + "m": 3389, + "n": 1536, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w3" + ] + }, + { + "type": "gemm", + "args": { + "m": 3389, + "n": 8192, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 3392, + "n": 24, + "k": 28672, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/hc_attn", + "deepseek-ai/DeepSeek-V4-Pro-0813/hc_ffn", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.hc_attn", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.hc_ffn", + "deepseek-ai/DeepSeek-V4-Pro/hc_attn", + "deepseek-ai/DeepSeek-V4-Pro/hc_ffn", + "deepseek-ai/DeepSeek-V4-Pro/mtp.hc_attn", + "deepseek-ai/DeepSeek-V4-Pro/mtp.hc_ffn", + "sgl-project/DeepSeek-V4-Pro-FP8/hc_attn", + "sgl-project/DeepSeek-V4-Pro-FP8/hc_ffn", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.hc_attn", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.hc_ffn" + ] + }, + { + "type": "gemm", + "args": { + "m": 3392, + "n": 64, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.weights_proj", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.weights_proj", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.weights_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 3392, + "n": 256, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.compressor.wkv", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.compressor.wkv", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.compressor.wgate", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.compressor.wkv" + ] + }, + { + "type": "gemm", + "args": { + "m": 3392, + "n": 512, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wkv", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wkv", + "deepseek-ai/DeepSeek-V4-Pro/attn.wkv", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wkv", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wkv", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wkv" + ] + }, + { + "type": "gemm", + "args": { + "m": 3392, + "n": 512, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.compressor.wkv+wgate", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.compressor.wkv+wgate", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.compressor.wkv+wgate" + ] + }, + { + "type": "gemm", + "args": { + "m": 3392, + "n": 768, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w3" + ] + }, + { + "type": "gemm", + "args": { + "m": 3392, + "n": 1024, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.compressor.wkv", + "deepseek-ai/DeepSeek-V4-Pro/attn.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro/attn.compressor.wkv", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.compressor.wgate", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.compressor.wkv" + ] + }, + { + "type": "gemm", + "args": { + "m": 3392, + "n": 1536, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w3" + ] + }, + { + "type": "gemm", + "args": { + "m": 3392, + "n": 7168, + "k": 384, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w2" + ] + }, + { + "type": "gemm", + "args": { + "m": 3392, + "n": 7168, + "k": 2048, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 3392, + "n": 8192, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 3393, + "n": 6144, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3" + ] + }, + { + "type": "gemm", + "args": { + "m": 3393, + "n": 7168, + "k": 3072, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w2" + ] + }, + { + "type": "gemm", + "args": { + "m": 3398, + "n": 384, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.gate", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.gate", + "deepseek-ai/DeepSeek-V4-Pro/ffn.gate", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.gate", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.gate", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.gate" + ] + }, + { + "type": "gemm", + "args": { + "m": 3400, + "n": 24, + "k": 28672, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/hc_attn", + "deepseek-ai/DeepSeek-V4-Pro-0813/hc_ffn", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.hc_attn", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.hc_ffn", + "deepseek-ai/DeepSeek-V4-Pro/hc_attn", + "deepseek-ai/DeepSeek-V4-Pro/hc_ffn", + "deepseek-ai/DeepSeek-V4-Pro/mtp.hc_attn", + "deepseek-ai/DeepSeek-V4-Pro/mtp.hc_ffn", + "sgl-project/DeepSeek-V4-Pro-FP8/hc_attn", + "sgl-project/DeepSeek-V4-Pro-FP8/hc_ffn", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.hc_attn", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.hc_ffn" + ] + }, + { + "type": "gemm", + "args": { + "m": 3400, + "n": 64, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.weights_proj", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.weights_proj", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.weights_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 3400, + "n": 256, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.compressor.wkv", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.compressor.wkv", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.compressor.wgate", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.compressor.wkv" + ] + }, + { + "type": "gemm", + "args": { + "m": 3400, + "n": 384, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.gate", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.gate", + "deepseek-ai/DeepSeek-V4-Pro/ffn.gate", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.gate", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.gate", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.gate" + ] + }, + { + "type": "gemm", + "args": { + "m": 3400, + "n": 512, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wkv", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wkv", + "deepseek-ai/DeepSeek-V4-Pro/attn.wkv", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wkv", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wkv", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wkv" + ] + }, + { + "type": "gemm", + "args": { + "m": 3400, + "n": 512, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.compressor.wkv+wgate", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.compressor.wkv+wgate", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.compressor.wkv+wgate" + ] + }, + { + "type": "gemm", + "args": { + "m": 3400, + "n": 1024, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.compressor.wkv", + "deepseek-ai/DeepSeek-V4-Pro/attn.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro/attn.compressor.wkv", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.compressor.wgate", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.compressor.wkv" + ] + }, + { + "type": "gemm", + "args": { + "m": 3400, + "n": 1536, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w3" + ] + }, + { + "type": "gemm", + "args": { + "m": 3400, + "n": 8192, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 3408, + "n": 24, + "k": 28672, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/hc_attn", + "deepseek-ai/DeepSeek-V4-Pro-0813/hc_ffn", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.hc_attn", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.hc_ffn", + "deepseek-ai/DeepSeek-V4-Pro/hc_attn", + "deepseek-ai/DeepSeek-V4-Pro/hc_ffn", + "deepseek-ai/DeepSeek-V4-Pro/mtp.hc_attn", + "deepseek-ai/DeepSeek-V4-Pro/mtp.hc_ffn", + "sgl-project/DeepSeek-V4-Pro-FP8/hc_attn", + "sgl-project/DeepSeek-V4-Pro-FP8/hc_ffn", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.hc_attn", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.hc_ffn" + ] + }, + { + "type": "gemm", + "args": { + "m": 3408, + "n": 64, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.weights_proj", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.weights_proj", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.weights_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 3408, + "n": 256, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.compressor.wkv", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.compressor.wkv", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.compressor.wgate", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.compressor.wkv" + ] + }, + { + "type": "gemm", + "args": { + "m": 3408, + "n": 384, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.gate", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.gate", + "deepseek-ai/DeepSeek-V4-Pro/ffn.gate", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.gate", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.gate", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.gate" + ] + }, + { + "type": "gemm", + "args": { + "m": 3408, + "n": 512, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wkv", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wkv", + "deepseek-ai/DeepSeek-V4-Pro/attn.wkv", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wkv", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wkv", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wkv" + ] + }, + { + "type": "gemm", + "args": { + "m": 3408, + "n": 512, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.compressor.wkv+wgate", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.compressor.wkv+wgate", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.compressor.wkv+wgate" + ] + }, + { + "type": "gemm", + "args": { + "m": 3408, + "n": 1024, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.compressor.wkv", + "deepseek-ai/DeepSeek-V4-Pro/attn.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro/attn.compressor.wkv", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.compressor.wgate", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.compressor.wkv" + ] + }, + { + "type": "gemm", + "args": { + "m": 3408, + "n": 1536, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w3" + ] + }, + { + "type": "gemm", + "args": { + "m": 3408, + "n": 8192, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 3413, + "n": 384, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w3" + ] + }, + { + "type": "gemm", + "args": { + "m": 3413, + "n": 7168, + "k": 192, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w2" + ] + }, + { + "type": "gemm", + "args": { + "m": 3415, + "n": 1536, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w3" + ] + }, + { + "type": "gemm", + "args": { + "m": 3415, + "n": 7168, + "k": 768, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w2" + ] + }, + { + "type": "gemm", + "args": { + "m": 3415, + "n": 7168, + "k": 4096, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 3415, + "n": 16384, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 3424, + "n": 24, + "k": 28672, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/hc_attn", + "deepseek-ai/DeepSeek-V4-Pro-0813/hc_ffn", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.hc_attn", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.hc_ffn", + "deepseek-ai/DeepSeek-V4-Pro/hc_attn", + "deepseek-ai/DeepSeek-V4-Pro/hc_ffn", + "deepseek-ai/DeepSeek-V4-Pro/mtp.hc_attn", + "deepseek-ai/DeepSeek-V4-Pro/mtp.hc_ffn", + "sgl-project/DeepSeek-V4-Pro-FP8/hc_attn", + "sgl-project/DeepSeek-V4-Pro-FP8/hc_ffn", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.hc_attn", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.hc_ffn" + ] + }, + { + "type": "gemm", + "args": { + "m": 3424, + "n": 64, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.weights_proj", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.weights_proj", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.weights_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 3424, + "n": 256, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.compressor.wkv", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.compressor.wkv", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.compressor.wgate", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.compressor.wkv" + ] + }, + { + "type": "gemm", + "args": { + "m": 3424, + "n": 384, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.gate", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.gate", + "deepseek-ai/DeepSeek-V4-Pro/ffn.gate", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.gate", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.gate", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.gate" + ] + }, + { + "type": "gemm", + "args": { + "m": 3424, + "n": 512, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wkv", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wkv", + "deepseek-ai/DeepSeek-V4-Pro/attn.wkv", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wkv", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wkv", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wkv" + ] + }, + { + "type": "gemm", + "args": { + "m": 3424, + "n": 512, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.compressor.wkv+wgate", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.compressor.wkv+wgate", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.compressor.wkv+wgate" + ] + }, + { + "type": "gemm", + "args": { + "m": 3424, + "n": 1024, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.compressor.wkv", + "deepseek-ai/DeepSeek-V4-Pro/attn.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro/attn.compressor.wkv", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.compressor.wgate", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.compressor.wkv" + ] + }, + { + "type": "gemm", + "args": { + "m": 3424, + "n": 1536, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w3" + ] + }, + { + "type": "gemm", + "args": { + "m": 3424, + "n": 7168, + "k": 768, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w2" + ] + }, + { + "type": "gemm", + "args": { + "m": 3424, + "n": 7168, + "k": 4096, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 3424, + "n": 8192, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 3424, + "n": 16384, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 3432, + "n": 1536, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w3" + ] + }, + { + "type": "gemm", + "args": { + "m": 3432, + "n": 7168, + "k": 768, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w2" + ] + }, + { + "type": "gemm", + "args": { + "m": 3432, + "n": 7168, + "k": 4096, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 3432, + "n": 16384, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 3440, + "n": 1536, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w3" + ] + }, + { + "type": "gemm", + "args": { + "m": 3440, + "n": 7168, + "k": 768, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w2" + ] + }, + { + "type": "gemm", + "args": { + "m": 3440, + "n": 7168, + "k": 4096, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 3440, + "n": 16384, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 3456, + "n": 24, + "k": 28672, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/hc_attn", + "deepseek-ai/DeepSeek-V4-Pro-0813/hc_ffn", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.hc_attn", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.hc_ffn", + "deepseek-ai/DeepSeek-V4-Pro/hc_attn", + "deepseek-ai/DeepSeek-V4-Pro/hc_ffn", + "deepseek-ai/DeepSeek-V4-Pro/mtp.hc_attn", + "deepseek-ai/DeepSeek-V4-Pro/mtp.hc_ffn", + "sgl-project/DeepSeek-V4-Pro-FP8/hc_attn", + "sgl-project/DeepSeek-V4-Pro-FP8/hc_ffn", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.hc_attn", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.hc_ffn" + ] + }, + { + "type": "gemm", + "args": { + "m": 3456, + "n": 64, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.weights_proj", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.weights_proj", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.weights_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 3456, + "n": 256, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.compressor.wkv", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.compressor.wkv", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.compressor.wgate", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.compressor.wkv" + ] + }, + { + "type": "gemm", + "args": { + "m": 3456, + "n": 384, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.gate", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.gate", + "deepseek-ai/DeepSeek-V4-Pro/ffn.gate", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.gate", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.gate", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.gate" + ] + }, + { + "type": "gemm", + "args": { + "m": 3456, + "n": 384, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w3" + ] + }, + { + "type": "gemm", + "args": { + "m": 3456, + "n": 512, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wkv", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wkv", + "deepseek-ai/DeepSeek-V4-Pro/attn.wkv", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wkv", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wkv", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wkv" + ] + }, + { + "type": "gemm", + "args": { + "m": 3456, + "n": 512, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.compressor.wkv+wgate", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.compressor.wkv+wgate", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.compressor.wkv+wgate" + ] + }, + { + "type": "gemm", + "args": { + "m": 3456, + "n": 1024, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.compressor.wkv", + "deepseek-ai/DeepSeek-V4-Pro/attn.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro/attn.compressor.wkv", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.compressor.wgate", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.compressor.wkv" + ] + }, + { + "type": "gemm", + "args": { + "m": 3456, + "n": 1536, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w3" + ] + }, + { + "type": "gemm", + "args": { + "m": 3456, + "n": 1792, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 3456, + "n": 7168, + "k": 192, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w2" + ] + }, + { + "type": "gemm", + "args": { + "m": 3456, + "n": 7168, + "k": 768, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w2" + ] + }, + { + "type": "gemm", + "args": { + "m": 3456, + "n": 7168, + "k": 896, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 3456, + "n": 7168, + "k": 2048, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 3456, + "n": 7168, + "k": 4096, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 3456, + "n": 8192, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 3456, + "n": 16384, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 3459, + "n": 1792, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 3459, + "n": 7168, + "k": 896, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 3464, + "n": 384, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w3" + ] + }, + { + "type": "gemm", + "args": { + "m": 3464, + "n": 1792, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 3464, + "n": 7168, + "k": 192, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w2" + ] + }, + { + "type": "gemm", + "args": { + "m": 3464, + "n": 7168, + "k": 896, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 3488, + "n": 24, + "k": 40960, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 3488, + "n": 64, + "k": 10240, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 3488, + "n": 192, + "k": 10240, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 3488, + "n": 256, + "k": 10240, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 3488, + "n": 512, + "k": 10240, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 3488, + "n": 512, + "k": 10240, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 3488, + "n": 1024, + "k": 10240, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 3488, + "n": 1536, + "k": 10240, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 3488, + "n": 10240, + "k": 1024, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 3520, + "n": 384, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.gate", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.gate", + "deepseek-ai/DeepSeek-V4-Pro/ffn.gate", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.gate", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.gate", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.gate" + ] + }, + { + "type": "gemm", + "args": { + "m": 3520, + "n": 384, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w3" + ] + }, + { + "type": "gemm", + "args": { + "m": 3520, + "n": 1536, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w3" + ] + }, + { + "type": "gemm", + "args": { + "m": 3520, + "n": 1792, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 3520, + "n": 7168, + "k": 192, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w2" + ] + }, + { + "type": "gemm", + "args": { + "m": 3520, + "n": 7168, + "k": 768, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w2" + ] + }, + { + "type": "gemm", + "args": { + "m": 3520, + "n": 7168, + "k": 896, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 3520, + "n": 7168, + "k": 4096, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 3520, + "n": 16384, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 3525, + "n": 2048, + "k": 12288, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 3525, + "n": 12288, + "k": 1024, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 3584, + "n": 24, + "k": 28672, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/hc_attn", + "deepseek-ai/DeepSeek-V4-Pro-0813/hc_ffn", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.hc_attn", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.hc_ffn", + "deepseek-ai/DeepSeek-V4-Pro/hc_attn", + "deepseek-ai/DeepSeek-V4-Pro/hc_ffn", + "deepseek-ai/DeepSeek-V4-Pro/mtp.hc_attn", + "deepseek-ai/DeepSeek-V4-Pro/mtp.hc_ffn", + "sgl-project/DeepSeek-V4-Pro-FP8/hc_attn", + "sgl-project/DeepSeek-V4-Pro-FP8/hc_ffn", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.hc_attn", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.hc_ffn" + ] + }, + { + "type": "gemm", + "args": { + "m": 3584, + "n": 64, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.weights_proj", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.weights_proj", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.weights_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 3584, + "n": 256, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.compressor.wkv", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.compressor.wkv", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.compressor.wgate", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.compressor.wkv" + ] + }, + { + "type": "gemm", + "args": { + "m": 3584, + "n": 384, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.gate", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.gate", + "deepseek-ai/DeepSeek-V4-Pro/ffn.gate", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.gate", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.gate", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.gate" + ] + }, + { + "type": "gemm", + "args": { + "m": 3584, + "n": 384, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w3" + ] + }, + { + "type": "gemm", + "args": { + "m": 3584, + "n": 512, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wkv", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wkv", + "deepseek-ai/DeepSeek-V4-Pro/attn.wkv", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wkv", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wkv", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wkv" + ] + }, + { + "type": "gemm", + "args": { + "m": 3584, + "n": 512, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.compressor.wkv+wgate", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.compressor.wkv+wgate", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.compressor.wkv+wgate" + ] + }, + { + "type": "gemm", + "args": { + "m": 3584, + "n": 768, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w3" + ] + }, + { + "type": "gemm", + "args": { + "m": 3584, + "n": 1024, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.compressor.wkv", + "deepseek-ai/DeepSeek-V4-Pro/attn.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro/attn.compressor.wkv", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.compressor.wgate", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.compressor.wkv" + ] + }, + { + "type": "gemm", + "args": { + "m": 3584, + "n": 1536, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w3" + ] + }, + { + "type": "gemm", + "args": { + "m": 3584, + "n": 1792, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 3584, + "n": 7168, + "k": 192, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w2" + ] + }, + { + "type": "gemm", + "args": { + "m": 3584, + "n": 7168, + "k": 384, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w2" + ] + }, + { + "type": "gemm", + "args": { + "m": 3584, + "n": 7168, + "k": 768, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w2" + ] + }, + { + "type": "gemm", + "args": { + "m": 3584, + "n": 7168, + "k": 896, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 3584, + "n": 7168, + "k": 4096, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 3584, + "n": 8192, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 3584, + "n": 16384, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 3652, + "n": 24, + "k": 49152, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 3652, + "n": 64, + "k": 12288, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 3652, + "n": 120, + "k": 12288, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 3652, + "n": 256, + "k": 12288, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 3652, + "n": 512, + "k": 12288, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 3652, + "n": 512, + "k": 12288, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 3652, + "n": 1024, + "k": 12288, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 3652, + "n": 1536, + "k": 12288, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 3652, + "n": 5120, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 3652, + "n": 12288, + "k": 1024, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 3656, + "n": 24, + "k": 49152, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 3656, + "n": 64, + "k": 12288, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 3656, + "n": 120, + "k": 12288, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 3656, + "n": 256, + "k": 12288, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 3656, + "n": 512, + "k": 12288, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 3656, + "n": 512, + "k": 12288, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 3656, + "n": 1024, + "k": 12288, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 3656, + "n": 1536, + "k": 12288, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 3656, + "n": 5120, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 3656, + "n": 12288, + "k": 1024, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 3664, + "n": 24, + "k": 49152, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 3664, + "n": 64, + "k": 12288, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 3664, + "n": 120, + "k": 12288, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 3664, + "n": 256, + "k": 12288, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 3664, + "n": 512, + "k": 12288, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 3664, + "n": 512, + "k": 12288, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 3664, + "n": 1024, + "k": 12288, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 3664, + "n": 1536, + "k": 12288, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 3664, + "n": 1792, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 3664, + "n": 5120, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 3664, + "n": 7168, + "k": 896, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 3664, + "n": 12288, + "k": 1024, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 3712, + "n": 24, + "k": 49152, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 3712, + "n": 64, + "k": 12288, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 3712, + "n": 120, + "k": 12288, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 3712, + "n": 256, + "k": 12288, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 3712, + "n": 512, + "k": 12288, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 3712, + "n": 512, + "k": 12288, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 3712, + "n": 1024, + "k": 12288, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 3712, + "n": 1536, + "k": 12288, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 3712, + "n": 5120, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 3712, + "n": 6144, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3" + ] + }, + { + "type": "gemm", + "args": { + "m": 3712, + "n": 7168, + "k": 3072, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w2" + ] + }, + { + "type": "gemm", + "args": { + "m": 3712, + "n": 7168, + "k": 16384, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 3712, + "n": 10240, + "k": 16384, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 3712, + "n": 12288, + "k": 1024, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 3712, + "n": 65536, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 3776, + "n": 24, + "k": 49152, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 3776, + "n": 64, + "k": 12288, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 3776, + "n": 120, + "k": 12288, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 3776, + "n": 256, + "k": 12288, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 3776, + "n": 512, + "k": 12288, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 3776, + "n": 512, + "k": 12288, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 3776, + "n": 1024, + "k": 12288, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 3776, + "n": 1536, + "k": 12288, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 3776, + "n": 5120, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 3776, + "n": 6144, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3" + ] + }, + { + "type": "gemm", + "args": { + "m": 3776, + "n": 7168, + "k": 3072, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w2" + ] + }, + { + "type": "gemm", + "args": { + "m": 3776, + "n": 7168, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.e_proj", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.h_proj", + "deepseek-ai/DeepSeek-V4-Pro/mtp.e_proj", + "deepseek-ai/DeepSeek-V4-Pro/mtp.h_proj", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.e_proj", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.h_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 3776, + "n": 7168, + "k": 16384, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 3776, + "n": 10240, + "k": 16384, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 3776, + "n": 12288, + "k": 1024, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 3776, + "n": 12288, + "k": 8192, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 3776, + "n": 12288, + "k": 16384, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 3776, + "n": 14336, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 3776, + "n": 16384, + "k": 12288, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 3776, + "n": 65536, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 3776, + "n": 81920, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 3778, + "n": 6144, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3" + ] + }, + { + "type": "gemm", + "args": { + "m": 3778, + "n": 7168, + "k": 3072, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w2" + ] + }, + { + "type": "gemm", + "args": { + "m": 3778, + "n": 7168, + "k": 16384, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 3778, + "n": 65536, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 3840, + "n": 24, + "k": 49152, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 3840, + "n": 64, + "k": 12288, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 3840, + "n": 120, + "k": 12288, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 3840, + "n": 256, + "k": 12288, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 3840, + "n": 512, + "k": 12288, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 3840, + "n": 512, + "k": 12288, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 3840, + "n": 1024, + "k": 12288, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 3840, + "n": 1536, + "k": 12288, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 3840, + "n": 2048, + "k": 12288, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 3840, + "n": 5120, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 3840, + "n": 6144, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3" + ] + }, + { + "type": "gemm", + "args": { + "m": 3840, + "n": 7168, + "k": 3072, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w2" + ] + }, + { + "type": "gemm", + "args": { + "m": 3840, + "n": 7168, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.e_proj", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.h_proj", + "deepseek-ai/DeepSeek-V4-Pro/mtp.e_proj", + "deepseek-ai/DeepSeek-V4-Pro/mtp.h_proj", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.e_proj", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.h_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 3840, + "n": 7168, + "k": 16384, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 3840, + "n": 10240, + "k": 16384, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 3840, + "n": 12288, + "k": 1024, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 3840, + "n": 12288, + "k": 8192, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 3840, + "n": 12288, + "k": 16384, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 3840, + "n": 14336, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 3840, + "n": 16384, + "k": 12288, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 3840, + "n": 65536, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 3840, + "n": 81920, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 3841, + "n": 1024, + "k": 10240, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 3841, + "n": 10240, + "k": 512, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 3841, + "n": 10240, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 3841, + "n": 12288, + "k": 2048, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 3888, + "n": 4096, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 3897, + "n": 4096, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 3904, + "n": 1024, + "k": 10240, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 3904, + "n": 2048, + "k": 12288, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 3904, + "n": 4096, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 3904, + "n": 7168, + "k": 1024, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 3904, + "n": 10240, + "k": 512, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 3904, + "n": 10240, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 3904, + "n": 12288, + "k": 1024, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 3904, + "n": 12288, + "k": 2048, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 3905, + "n": 7168, + "k": 1024, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 3912, + "n": 4096, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 3912, + "n": 7168, + "k": 1024, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 3920, + "n": 7168, + "k": 1024, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 3921, + "n": 320, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 3922, + "n": 24, + "k": 40960, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 3922, + "n": 64, + "k": 10240, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 3922, + "n": 192, + "k": 10240, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 3922, + "n": 256, + "k": 10240, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 3922, + "n": 512, + "k": 10240, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 3922, + "n": 512, + "k": 10240, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 3922, + "n": 1024, + "k": 10240, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 3922, + "n": 1536, + "k": 10240, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 3922, + "n": 10240, + "k": 1024, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 3922, + "n": 10240, + "k": 2048, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 3925, + "n": 896, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 3925, + "n": 7168, + "k": 448, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 3928, + "n": 320, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 3928, + "n": 896, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 3928, + "n": 7168, + "k": 448, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 3936, + "n": 320, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 3936, + "n": 768, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w3" + ] + }, + { + "type": "gemm", + "args": { + "m": 3936, + "n": 896, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 3936, + "n": 4096, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 3936, + "n": 7168, + "k": 384, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w2" + ] + }, + { + "type": "gemm", + "args": { + "m": 3936, + "n": 7168, + "k": 448, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 3936, + "n": 7168, + "k": 1024, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 3936, + "n": 7168, + "k": 2048, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 3944, + "n": 24, + "k": 40960, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 3944, + "n": 64, + "k": 10240, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 3944, + "n": 192, + "k": 10240, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 3944, + "n": 256, + "k": 10240, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 3944, + "n": 512, + "k": 10240, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 3944, + "n": 512, + "k": 10240, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 3944, + "n": 1024, + "k": 10240, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 3944, + "n": 1536, + "k": 10240, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 3944, + "n": 10240, + "k": 1024, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 3951, + "n": 2048, + "k": 10240, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 3951, + "n": 10240, + "k": 1024, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 3951, + "n": 10240, + "k": 4096, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 3968, + "n": 24, + "k": 40960, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 3968, + "n": 64, + "k": 10240, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 3968, + "n": 192, + "k": 10240, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 3968, + "n": 256, + "k": 10240, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 3968, + "n": 320, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 3968, + "n": 512, + "k": 10240, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 3968, + "n": 512, + "k": 10240, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 3968, + "n": 768, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w3" + ] + }, + { + "type": "gemm", + "args": { + "m": 3968, + "n": 1024, + "k": 10240, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 3968, + "n": 1024, + "k": 10240, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 3968, + "n": 1536, + "k": 10240, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 3968, + "n": 2048, + "k": 12288, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 3968, + "n": 4096, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 3968, + "n": 6144, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3" + ] + }, + { + "type": "gemm", + "args": { + "m": 3968, + "n": 7168, + "k": 384, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w2" + ] + }, + { + "type": "gemm", + "args": { + "m": 3968, + "n": 7168, + "k": 1024, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 3968, + "n": 7168, + "k": 3072, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w2" + ] + }, + { + "type": "gemm", + "args": { + "m": 3968, + "n": 7168, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.e_proj", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.h_proj", + "deepseek-ai/DeepSeek-V4-Pro/mtp.e_proj", + "deepseek-ai/DeepSeek-V4-Pro/mtp.h_proj", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.e_proj", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.h_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 3968, + "n": 10240, + "k": 512, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 3968, + "n": 10240, + "k": 1024, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 3968, + "n": 10240, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 3968, + "n": 10240, + "k": 2048, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 3968, + "n": 12288, + "k": 1024, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 3968, + "n": 12288, + "k": 2048, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 3968, + "n": 12288, + "k": 8192, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 3968, + "n": 12288, + "k": 16384, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 3968, + "n": 14336, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 3968, + "n": 16384, + "k": 12288, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 3968, + "n": 81920, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 4000, + "n": 24, + "k": 40960, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 4000, + "n": 64, + "k": 10240, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 4000, + "n": 192, + "k": 10240, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 4000, + "n": 256, + "k": 10240, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 4000, + "n": 512, + "k": 10240, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 4000, + "n": 512, + "k": 10240, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 4000, + "n": 1024, + "k": 10240, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 4000, + "n": 1536, + "k": 10240, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 4000, + "n": 10240, + "k": 1024, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 4032, + "n": 24, + "k": 40960, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 4032, + "n": 64, + "k": 10240, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 4032, + "n": 192, + "k": 10240, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 4032, + "n": 256, + "k": 10240, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 4032, + "n": 320, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 4032, + "n": 512, + "k": 10240, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 4032, + "n": 512, + "k": 10240, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 4032, + "n": 1024, + "k": 10240, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 4032, + "n": 1536, + "k": 10240, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 4032, + "n": 10240, + "k": 1024, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 4032, + "n": 10240, + "k": 2048, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 4040, + "n": 1024, + "k": 10240, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 4040, + "n": 10240, + "k": 512, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 4040, + "n": 10240, + "k": 2048, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 4048, + "n": 24, + "k": 40960, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 4048, + "n": 64, + "k": 10240, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 4048, + "n": 192, + "k": 10240, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 4048, + "n": 256, + "k": 10240, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 4048, + "n": 320, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 4048, + "n": 512, + "k": 10240, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 4048, + "n": 512, + "k": 10240, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 4048, + "n": 896, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 4048, + "n": 1024, + "k": 10240, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 4048, + "n": 1536, + "k": 10240, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 4048, + "n": 7168, + "k": 448, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 4048, + "n": 10240, + "k": 1024, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 4096, + "n": 24, + "k": 40960, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 4096, + "n": 64, + "k": 10240, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 4096, + "n": 192, + "k": 10240, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 4096, + "n": 256, + "k": 10240, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 4096, + "n": 320, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 4096, + "n": 512, + "k": 10240, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 4096, + "n": 512, + "k": 10240, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 4096, + "n": 1024, + "k": 10240, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 4096, + "n": 1024, + "k": 10240, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 4096, + "n": 1536, + "k": 10240, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 4096, + "n": 4096, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 4096, + "n": 7168, + "k": 1024, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 4096, + "n": 10240, + "k": 512, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 4096, + "n": 10240, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 4096, + "n": 10240, + "k": 2048, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 4096, + "n": 12288, + "k": 2048, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 4112, + "n": 10240, + "k": 2048, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 4160, + "n": 7168, + "k": 2048, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 4162, + "n": 768, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w3" + ] + }, + { + "type": "gemm", + "args": { + "m": 4162, + "n": 7168, + "k": 384, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w2" + ] + }, + { + "type": "gemm", + "args": { + "m": 4162, + "n": 7168, + "k": 2048, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 4168, + "n": 768, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w3" + ] + }, + { + "type": "gemm", + "args": { + "m": 4168, + "n": 7168, + "k": 384, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w2" + ] + }, + { + "type": "gemm", + "args": { + "m": 4168, + "n": 7168, + "k": 2048, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 4168, + "n": 10240, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 4168, + "n": 12288, + "k": 2048, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 4224, + "n": 7168, + "k": 2048, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 4237, + "n": 24, + "k": 28672, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/hc_attn", + "deepseek-ai/DeepSeek-V4-Pro-0813/hc_ffn", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.hc_attn", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.hc_ffn", + "deepseek-ai/DeepSeek-V4-Pro/hc_attn", + "deepseek-ai/DeepSeek-V4-Pro/hc_ffn", + "deepseek-ai/DeepSeek-V4-Pro/mtp.hc_attn", + "deepseek-ai/DeepSeek-V4-Pro/mtp.hc_ffn", + "sgl-project/DeepSeek-V4-Pro-FP8/hc_attn", + "sgl-project/DeepSeek-V4-Pro-FP8/hc_ffn", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.hc_attn", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.hc_ffn" + ] + }, + { + "type": "gemm", + "args": { + "m": 4237, + "n": 64, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.weights_proj", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.weights_proj", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.weights_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 4237, + "n": 256, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.compressor.wkv", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.compressor.wkv", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.compressor.wgate", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.compressor.wkv" + ] + }, + { + "type": "gemm", + "args": { + "m": 4237, + "n": 512, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wkv", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wkv", + "deepseek-ai/DeepSeek-V4-Pro/attn.wkv", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wkv", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wkv", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wkv" + ] + }, + { + "type": "gemm", + "args": { + "m": 4237, + "n": 512, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.compressor.wkv+wgate", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.compressor.wkv+wgate", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.compressor.wkv+wgate" + ] + }, + { + "type": "gemm", + "args": { + "m": 4237, + "n": 1024, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.compressor.wkv", + "deepseek-ai/DeepSeek-V4-Pro/attn.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro/attn.compressor.wkv", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.compressor.wgate", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.compressor.wkv" + ] + }, + { + "type": "gemm", + "args": { + "m": 4237, + "n": 1536, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w3" + ] + }, + { + "type": "gemm", + "args": { + "m": 4237, + "n": 8192, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 4240, + "n": 24, + "k": 28672, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/hc_attn", + "deepseek-ai/DeepSeek-V4-Pro-0813/hc_ffn", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.hc_attn", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.hc_ffn", + "deepseek-ai/DeepSeek-V4-Pro/hc_attn", + "deepseek-ai/DeepSeek-V4-Pro/hc_ffn", + "deepseek-ai/DeepSeek-V4-Pro/mtp.hc_attn", + "deepseek-ai/DeepSeek-V4-Pro/mtp.hc_ffn", + "sgl-project/DeepSeek-V4-Pro-FP8/hc_attn", + "sgl-project/DeepSeek-V4-Pro-FP8/hc_ffn", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.hc_attn", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.hc_ffn" + ] + }, + { + "type": "gemm", + "args": { + "m": 4240, + "n": 64, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.weights_proj", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.weights_proj", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.weights_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 4240, + "n": 256, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.compressor.wkv", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.compressor.wkv", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.compressor.wgate", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.compressor.wkv" + ] + }, + { + "type": "gemm", + "args": { + "m": 4240, + "n": 384, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w3" + ] + }, + { + "type": "gemm", + "args": { + "m": 4240, + "n": 512, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wkv", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wkv", + "deepseek-ai/DeepSeek-V4-Pro/attn.wkv", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wkv", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wkv", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wkv" + ] + }, + { + "type": "gemm", + "args": { + "m": 4240, + "n": 512, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.compressor.wkv+wgate", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.compressor.wkv+wgate", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.compressor.wkv+wgate" + ] + }, + { + "type": "gemm", + "args": { + "m": 4240, + "n": 1024, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.compressor.wkv", + "deepseek-ai/DeepSeek-V4-Pro/attn.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro/attn.compressor.wkv", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.compressor.wgate", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.compressor.wkv" + ] + }, + { + "type": "gemm", + "args": { + "m": 4240, + "n": 1536, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w3" + ] + }, + { + "type": "gemm", + "args": { + "m": 4240, + "n": 7168, + "k": 192, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w2" + ] + }, + { + "type": "gemm", + "args": { + "m": 4240, + "n": 7168, + "k": 2048, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 4240, + "n": 8192, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 4248, + "n": 24, + "k": 28672, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/hc_attn", + "deepseek-ai/DeepSeek-V4-Pro-0813/hc_ffn", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.hc_attn", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.hc_ffn", + "deepseek-ai/DeepSeek-V4-Pro/hc_attn", + "deepseek-ai/DeepSeek-V4-Pro/hc_ffn", + "deepseek-ai/DeepSeek-V4-Pro/mtp.hc_attn", + "deepseek-ai/DeepSeek-V4-Pro/mtp.hc_ffn", + "sgl-project/DeepSeek-V4-Pro-FP8/hc_attn", + "sgl-project/DeepSeek-V4-Pro-FP8/hc_ffn", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.hc_attn", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.hc_ffn" + ] + }, + { + "type": "gemm", + "args": { + "m": 4248, + "n": 64, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.weights_proj", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.weights_proj", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.weights_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 4248, + "n": 256, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.compressor.wkv", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.compressor.wkv", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.compressor.wgate", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.compressor.wkv" + ] + }, + { + "type": "gemm", + "args": { + "m": 4248, + "n": 512, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wkv", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wkv", + "deepseek-ai/DeepSeek-V4-Pro/attn.wkv", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wkv", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wkv", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wkv" + ] + }, + { + "type": "gemm", + "args": { + "m": 4248, + "n": 512, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.compressor.wkv+wgate", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.compressor.wkv+wgate", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.compressor.wkv+wgate" + ] + }, + { + "type": "gemm", + "args": { + "m": 4248, + "n": 1024, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.compressor.wkv", + "deepseek-ai/DeepSeek-V4-Pro/attn.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro/attn.compressor.wkv", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.compressor.wgate", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.compressor.wkv" + ] + }, + { + "type": "gemm", + "args": { + "m": 4248, + "n": 1536, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w3" + ] + }, + { + "type": "gemm", + "args": { + "m": 4248, + "n": 8192, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 4249, + "n": 384, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.gate", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.gate", + "deepseek-ai/DeepSeek-V4-Pro/ffn.gate", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.gate", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.gate", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.gate" + ] + }, + { + "type": "gemm", + "args": { + "m": 4256, + "n": 24, + "k": 28672, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/hc_attn", + "deepseek-ai/DeepSeek-V4-Pro-0813/hc_ffn", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.hc_attn", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.hc_ffn", + "deepseek-ai/DeepSeek-V4-Pro/hc_attn", + "deepseek-ai/DeepSeek-V4-Pro/hc_ffn", + "deepseek-ai/DeepSeek-V4-Pro/mtp.hc_attn", + "deepseek-ai/DeepSeek-V4-Pro/mtp.hc_ffn", + "sgl-project/DeepSeek-V4-Pro-FP8/hc_attn", + "sgl-project/DeepSeek-V4-Pro-FP8/hc_ffn", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.hc_attn", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.hc_ffn" + ] + }, + { + "type": "gemm", + "args": { + "m": 4256, + "n": 64, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.weights_proj", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.weights_proj", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.weights_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 4256, + "n": 256, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.compressor.wkv", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.compressor.wkv", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.compressor.wgate", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.compressor.wkv" + ] + }, + { + "type": "gemm", + "args": { + "m": 4256, + "n": 384, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.gate", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.gate", + "deepseek-ai/DeepSeek-V4-Pro/ffn.gate", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.gate", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.gate", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.gate" + ] + }, + { + "type": "gemm", + "args": { + "m": 4256, + "n": 512, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wkv", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wkv", + "deepseek-ai/DeepSeek-V4-Pro/attn.wkv", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wkv", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wkv", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wkv" + ] + }, + { + "type": "gemm", + "args": { + "m": 4256, + "n": 512, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.compressor.wkv+wgate", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.compressor.wkv+wgate", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.compressor.wkv+wgate" + ] + }, + { + "type": "gemm", + "args": { + "m": 4256, + "n": 1024, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.compressor.wkv", + "deepseek-ai/DeepSeek-V4-Pro/attn.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro/attn.compressor.wkv", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.compressor.wgate", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.compressor.wkv" + ] + }, + { + "type": "gemm", + "args": { + "m": 4256, + "n": 1536, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w3" + ] + }, + { + "type": "gemm", + "args": { + "m": 4256, + "n": 8192, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 4264, + "n": 384, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.gate", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.gate", + "deepseek-ai/DeepSeek-V4-Pro/ffn.gate", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.gate", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.gate", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.gate" + ] + }, + { + "type": "gemm", + "args": { + "m": 4269, + "n": 1536, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w3" + ] + }, + { + "type": "gemm", + "args": { + "m": 4269, + "n": 7168, + "k": 768, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w2" + ] + }, + { + "type": "gemm", + "args": { + "m": 4269, + "n": 7168, + "k": 4096, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 4269, + "n": 16384, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 4285, + "n": 384, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w3" + ] + }, + { + "type": "gemm", + "args": { + "m": 4285, + "n": 7168, + "k": 192, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w2" + ] + }, + { + "type": "gemm", + "args": { + "m": 4288, + "n": 24, + "k": 28672, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/hc_attn", + "deepseek-ai/DeepSeek-V4-Pro-0813/hc_ffn", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.hc_attn", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.hc_ffn", + "deepseek-ai/DeepSeek-V4-Pro/hc_attn", + "deepseek-ai/DeepSeek-V4-Pro/hc_ffn", + "deepseek-ai/DeepSeek-V4-Pro/mtp.hc_attn", + "deepseek-ai/DeepSeek-V4-Pro/mtp.hc_ffn", + "sgl-project/DeepSeek-V4-Pro-FP8/hc_attn", + "sgl-project/DeepSeek-V4-Pro-FP8/hc_ffn", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.hc_attn", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.hc_ffn" + ] + }, + { + "type": "gemm", + "args": { + "m": 4288, + "n": 64, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.weights_proj", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.weights_proj", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.weights_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 4288, + "n": 256, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.compressor.wkv", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.compressor.wkv", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.compressor.wgate", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.compressor.wkv" + ] + }, + { + "type": "gemm", + "args": { + "m": 4288, + "n": 384, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.gate", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.gate", + "deepseek-ai/DeepSeek-V4-Pro/ffn.gate", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.gate", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.gate", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.gate" + ] + }, + { + "type": "gemm", + "args": { + "m": 4288, + "n": 384, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w3" + ] + }, + { + "type": "gemm", + "args": { + "m": 4288, + "n": 512, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wkv", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wkv", + "deepseek-ai/DeepSeek-V4-Pro/attn.wkv", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wkv", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wkv", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wkv" + ] + }, + { + "type": "gemm", + "args": { + "m": 4288, + "n": 512, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.compressor.wkv+wgate", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.compressor.wkv+wgate", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.compressor.wkv+wgate" + ] + }, + { + "type": "gemm", + "args": { + "m": 4288, + "n": 1024, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.compressor.wkv", + "deepseek-ai/DeepSeek-V4-Pro/attn.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro/attn.compressor.wkv", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.compressor.wgate", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.compressor.wkv" + ] + }, + { + "type": "gemm", + "args": { + "m": 4288, + "n": 1536, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w3" + ] + }, + { + "type": "gemm", + "args": { + "m": 4288, + "n": 7168, + "k": 192, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w2" + ] + }, + { + "type": "gemm", + "args": { + "m": 4288, + "n": 7168, + "k": 768, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w2" + ] + }, + { + "type": "gemm", + "args": { + "m": 4288, + "n": 7168, + "k": 4096, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 4288, + "n": 8192, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 4288, + "n": 16384, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 4290, + "n": 384, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w3" + ] + }, + { + "type": "gemm", + "args": { + "m": 4290, + "n": 7168, + "k": 192, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w2" + ] + }, + { + "type": "gemm", + "args": { + "m": 4304, + "n": 384, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.gate", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.gate", + "deepseek-ai/DeepSeek-V4-Pro/ffn.gate", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.gate", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.gate", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.gate" + ] + }, + { + "type": "gemm", + "args": { + "m": 4304, + "n": 1536, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w3" + ] + }, + { + "type": "gemm", + "args": { + "m": 4304, + "n": 7168, + "k": 768, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w2" + ] + }, + { + "type": "gemm", + "args": { + "m": 4304, + "n": 7168, + "k": 4096, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 4304, + "n": 16384, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 4312, + "n": 1536, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w3" + ] + }, + { + "type": "gemm", + "args": { + "m": 4312, + "n": 7168, + "k": 768, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w2" + ] + }, + { + "type": "gemm", + "args": { + "m": 4312, + "n": 7168, + "k": 4096, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 4312, + "n": 16384, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 4320, + "n": 1536, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w3" + ] + }, + { + "type": "gemm", + "args": { + "m": 4320, + "n": 7168, + "k": 768, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w2" + ] + }, + { + "type": "gemm", + "args": { + "m": 4320, + "n": 7168, + "k": 4096, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 4320, + "n": 10240, + "k": 2048, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 4320, + "n": 16384, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 4352, + "n": 24, + "k": 28672, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/hc_attn", + "deepseek-ai/DeepSeek-V4-Pro-0813/hc_ffn", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.hc_attn", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.hc_ffn", + "deepseek-ai/DeepSeek-V4-Pro/hc_attn", + "deepseek-ai/DeepSeek-V4-Pro/hc_ffn", + "deepseek-ai/DeepSeek-V4-Pro/mtp.hc_attn", + "deepseek-ai/DeepSeek-V4-Pro/mtp.hc_ffn", + "sgl-project/DeepSeek-V4-Pro-FP8/hc_attn", + "sgl-project/DeepSeek-V4-Pro-FP8/hc_ffn", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.hc_attn", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.hc_ffn" + ] + }, + { + "type": "gemm", + "args": { + "m": 4352, + "n": 64, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.weights_proj", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.weights_proj", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.weights_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 4352, + "n": 256, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.compressor.wkv", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.compressor.wkv", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.compressor.wgate", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.compressor.wkv" + ] + }, + { + "type": "gemm", + "args": { + "m": 4352, + "n": 384, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.gate", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.gate", + "deepseek-ai/DeepSeek-V4-Pro/ffn.gate", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.gate", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.gate", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.gate" + ] + }, + { + "type": "gemm", + "args": { + "m": 4352, + "n": 384, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w3" + ] + }, + { + "type": "gemm", + "args": { + "m": 4352, + "n": 512, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wkv", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wkv", + "deepseek-ai/DeepSeek-V4-Pro/attn.wkv", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wkv", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wkv", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wkv" + ] + }, + { + "type": "gemm", + "args": { + "m": 4352, + "n": 512, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.compressor.wkv+wgate", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.compressor.wkv+wgate", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.compressor.wkv+wgate" + ] + }, + { + "type": "gemm", + "args": { + "m": 4352, + "n": 1024, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.compressor.wkv", + "deepseek-ai/DeepSeek-V4-Pro/attn.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro/attn.compressor.wkv", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.compressor.wgate", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.compressor.wkv" + ] + }, + { + "type": "gemm", + "args": { + "m": 4352, + "n": 1536, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w3" + ] + }, + { + "type": "gemm", + "args": { + "m": 4352, + "n": 1792, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 4352, + "n": 7168, + "k": 192, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w2" + ] + }, + { + "type": "gemm", + "args": { + "m": 4352, + "n": 7168, + "k": 768, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w2" + ] + }, + { + "type": "gemm", + "args": { + "m": 4352, + "n": 7168, + "k": 896, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 4352, + "n": 7168, + "k": 2048, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 4352, + "n": 7168, + "k": 4096, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 4352, + "n": 8192, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 4352, + "n": 10240, + "k": 1024, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 4352, + "n": 16384, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 4354, + "n": 1792, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 4354, + "n": 7168, + "k": 896, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 4400, + "n": 768, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w3" + ] + }, + { + "type": "gemm", + "args": { + "m": 4400, + "n": 7168, + "k": 384, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w2" + ] + }, + { + "type": "gemm", + "args": { + "m": 4416, + "n": 1792, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 4416, + "n": 7168, + "k": 896, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 4480, + "n": 24, + "k": 28672, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/hc_attn", + "deepseek-ai/DeepSeek-V4-Pro-0813/hc_ffn", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.hc_attn", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.hc_ffn", + "deepseek-ai/DeepSeek-V4-Pro/hc_attn", + "deepseek-ai/DeepSeek-V4-Pro/hc_ffn", + "deepseek-ai/DeepSeek-V4-Pro/mtp.hc_attn", + "deepseek-ai/DeepSeek-V4-Pro/mtp.hc_ffn", + "sgl-project/DeepSeek-V4-Pro-FP8/hc_attn", + "sgl-project/DeepSeek-V4-Pro-FP8/hc_ffn", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.hc_attn", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.hc_ffn" + ] + }, + { + "type": "gemm", + "args": { + "m": 4480, + "n": 64, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.weights_proj", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.weights_proj", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.weights_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 4480, + "n": 256, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.compressor.wkv", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.compressor.wkv", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.compressor.wgate", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.compressor.wkv" + ] + }, + { + "type": "gemm", + "args": { + "m": 4480, + "n": 384, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.gate", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.gate", + "deepseek-ai/DeepSeek-V4-Pro/ffn.gate", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.gate", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.gate", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.gate" + ] + }, + { + "type": "gemm", + "args": { + "m": 4480, + "n": 384, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w3" + ] + }, + { + "type": "gemm", + "args": { + "m": 4480, + "n": 512, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wkv", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wkv", + "deepseek-ai/DeepSeek-V4-Pro/attn.wkv", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wkv", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wkv", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wkv" + ] + }, + { + "type": "gemm", + "args": { + "m": 4480, + "n": 512, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.compressor.wkv+wgate", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.compressor.wkv+wgate", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.compressor.wkv+wgate" + ] + }, + { + "type": "gemm", + "args": { + "m": 4480, + "n": 896, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 4480, + "n": 1024, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.compressor.wkv", + "deepseek-ai/DeepSeek-V4-Pro/attn.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro/attn.compressor.wkv", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.compressor.wgate", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.compressor.wkv" + ] + }, + { + "type": "gemm", + "args": { + "m": 4480, + "n": 1536, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w3" + ] + }, + { + "type": "gemm", + "args": { + "m": 4480, + "n": 1792, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 4480, + "n": 7168, + "k": 192, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w2" + ] + }, + { + "type": "gemm", + "args": { + "m": 4480, + "n": 7168, + "k": 448, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 4480, + "n": 7168, + "k": 768, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w2" + ] + }, + { + "type": "gemm", + "args": { + "m": 4480, + "n": 7168, + "k": 896, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 4480, + "n": 7168, + "k": 4096, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 4480, + "n": 8192, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 4480, + "n": 16384, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 4488, + "n": 384, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w3" + ] + }, + { + "type": "gemm", + "args": { + "m": 4488, + "n": 7168, + "k": 192, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w2" + ] + }, + { + "type": "gemm", + "args": { + "m": 4496, + "n": 1792, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 4496, + "n": 7168, + "k": 896, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 4544, + "n": 768, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w3" + ] + }, + { + "type": "gemm", + "args": { + "m": 4544, + "n": 7168, + "k": 384, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w2" + ] + }, + { + "type": "gemm", + "args": { + "m": 4545, + "n": 12288, + "k": 16384, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 4545, + "n": 81920, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 4552, + "n": 1792, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 4552, + "n": 7168, + "k": 896, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 4566, + "n": 24, + "k": 49152, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 4566, + "n": 64, + "k": 12288, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 4566, + "n": 120, + "k": 12288, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 4566, + "n": 256, + "k": 12288, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 4566, + "n": 512, + "k": 12288, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 4566, + "n": 512, + "k": 12288, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 4566, + "n": 1024, + "k": 12288, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 4566, + "n": 1536, + "k": 12288, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 4566, + "n": 5120, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 4566, + "n": 12288, + "k": 1024, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 4608, + "n": 24, + "k": 49152, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 4608, + "n": 64, + "k": 12288, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 4608, + "n": 120, + "k": 12288, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 4608, + "n": 256, + "k": 12288, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 4608, + "n": 512, + "k": 12288, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 4608, + "n": 512, + "k": 12288, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 4608, + "n": 1024, + "k": 12288, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 4608, + "n": 1536, + "k": 12288, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 4608, + "n": 5120, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 4608, + "n": 12288, + "k": 1024, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 4672, + "n": 24, + "k": 49152, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 4672, + "n": 64, + "k": 12288, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 4672, + "n": 120, + "k": 12288, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 4672, + "n": 256, + "k": 12288, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 4672, + "n": 512, + "k": 12288, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 4672, + "n": 512, + "k": 12288, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 4672, + "n": 1024, + "k": 12288, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 4672, + "n": 1536, + "k": 12288, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 4672, + "n": 5120, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 4672, + "n": 6144, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3" + ] + }, + { + "type": "gemm", + "args": { + "m": 4672, + "n": 7168, + "k": 3072, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w2" + ] + }, + { + "type": "gemm", + "args": { + "m": 4672, + "n": 7168, + "k": 16384, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 4672, + "n": 10240, + "k": 16384, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 4672, + "n": 12288, + "k": 1024, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 4672, + "n": 65536, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 4688, + "n": 1024, + "k": 10240, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 4688, + "n": 10240, + "k": 512, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 4704, + "n": 384, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w3" + ] + }, + { + "type": "gemm", + "args": { + "m": 4704, + "n": 7168, + "k": 192, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w2" + ] + }, + { + "type": "gemm", + "args": { + "m": 4717, + "n": 10240, + "k": 16384, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 4717, + "n": 65536, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 4734, + "n": 7168, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.e_proj", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.h_proj", + "deepseek-ai/DeepSeek-V4-Pro/mtp.e_proj", + "deepseek-ai/DeepSeek-V4-Pro/mtp.h_proj", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.e_proj", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.h_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 4734, + "n": 7168, + "k": 16384, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 4734, + "n": 14336, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 4736, + "n": 24, + "k": 49152, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 4736, + "n": 64, + "k": 12288, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 4736, + "n": 120, + "k": 12288, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 4736, + "n": 256, + "k": 12288, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 4736, + "n": 512, + "k": 12288, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 4736, + "n": 512, + "k": 12288, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 4736, + "n": 1024, + "k": 12288, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 4736, + "n": 1536, + "k": 12288, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 4736, + "n": 5120, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 4736, + "n": 6144, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3" + ] + }, + { + "type": "gemm", + "args": { + "m": 4736, + "n": 7168, + "k": 3072, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w2" + ] + }, + { + "type": "gemm", + "args": { + "m": 4736, + "n": 7168, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.e_proj", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.h_proj", + "deepseek-ai/DeepSeek-V4-Pro/mtp.e_proj", + "deepseek-ai/DeepSeek-V4-Pro/mtp.h_proj", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.e_proj", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.h_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 4736, + "n": 7168, + "k": 16384, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 4736, + "n": 10240, + "k": 16384, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 4736, + "n": 12288, + "k": 1024, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 4736, + "n": 12288, + "k": 8192, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 4736, + "n": 12288, + "k": 16384, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 4736, + "n": 14336, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 4736, + "n": 16384, + "k": 12288, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 4736, + "n": 65536, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 4736, + "n": 81920, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 4800, + "n": 6144, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3" + ] + }, + { + "type": "gemm", + "args": { + "m": 4800, + "n": 7168, + "k": 3072, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w2" + ] + }, + { + "type": "gemm", + "args": { + "m": 4800, + "n": 7168, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.e_proj", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.h_proj", + "deepseek-ai/DeepSeek-V4-Pro/mtp.e_proj", + "deepseek-ai/DeepSeek-V4-Pro/mtp.h_proj", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.e_proj", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.h_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 4800, + "n": 12288, + "k": 8192, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 4800, + "n": 12288, + "k": 16384, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 4800, + "n": 14336, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 4800, + "n": 16384, + "k": 12288, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 4800, + "n": 81920, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 4802, + "n": 1024, + "k": 10240, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 4802, + "n": 10240, + "k": 512, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 4802, + "n": 10240, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 4802, + "n": 12288, + "k": 2048, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 4808, + "n": 24, + "k": 49152, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 4808, + "n": 64, + "k": 12288, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 4808, + "n": 120, + "k": 12288, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 4808, + "n": 256, + "k": 12288, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 4808, + "n": 512, + "k": 12288, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 4808, + "n": 512, + "k": 12288, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 4808, + "n": 1024, + "k": 12288, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 4808, + "n": 1536, + "k": 12288, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 4808, + "n": 5120, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 4808, + "n": 12288, + "k": 1024, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 4810, + "n": 8192, + "k": 10240, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 4810, + "n": 10240, + "k": 4096, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 4816, + "n": 1024, + "k": 10240, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 4816, + "n": 10240, + "k": 512, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 4862, + "n": 4096, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 4864, + "n": 768, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w3" + ] + }, + { + "type": "gemm", + "args": { + "m": 4864, + "n": 1024, + "k": 10240, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 4864, + "n": 2048, + "k": 12288, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 4864, + "n": 4096, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 4864, + "n": 6144, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3" + ] + }, + { + "type": "gemm", + "args": { + "m": 4864, + "n": 7168, + "k": 384, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w2" + ] + }, + { + "type": "gemm", + "args": { + "m": 4864, + "n": 7168, + "k": 3072, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w2" + ] + }, + { + "type": "gemm", + "args": { + "m": 4864, + "n": 7168, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.e_proj", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.h_proj", + "deepseek-ai/DeepSeek-V4-Pro/mtp.e_proj", + "deepseek-ai/DeepSeek-V4-Pro/mtp.h_proj", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.e_proj", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.h_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 4864, + "n": 7168, + "k": 16384, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 4864, + "n": 10240, + "k": 512, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 4864, + "n": 10240, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 4864, + "n": 10240, + "k": 16384, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 4864, + "n": 12288, + "k": 1024, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 4864, + "n": 12288, + "k": 2048, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 4864, + "n": 12288, + "k": 8192, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 4864, + "n": 12288, + "k": 16384, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 4864, + "n": 14336, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 4864, + "n": 16384, + "k": 12288, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 4864, + "n": 65536, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 4864, + "n": 81920, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 4868, + "n": 1024, + "k": 12288, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 4868, + "n": 12288, + "k": 512, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 4872, + "n": 4096, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 4880, + "n": 24, + "k": 49152, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 4880, + "n": 64, + "k": 12288, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 4880, + "n": 120, + "k": 12288, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 4880, + "n": 256, + "k": 12288, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 4880, + "n": 512, + "k": 12288, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 4880, + "n": 512, + "k": 12288, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 4880, + "n": 1024, + "k": 12288, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 4880, + "n": 1536, + "k": 12288, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 4880, + "n": 4096, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 4880, + "n": 5120, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 4880, + "n": 12288, + "k": 1024, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 4881, + "n": 7168, + "k": 1024, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 4888, + "n": 7168, + "k": 1024, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 4903, + "n": 320, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 4906, + "n": 24, + "k": 40960, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 4906, + "n": 64, + "k": 10240, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 4906, + "n": 192, + "k": 10240, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 4906, + "n": 256, + "k": 10240, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 4906, + "n": 512, + "k": 10240, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 4906, + "n": 512, + "k": 10240, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 4906, + "n": 1024, + "k": 10240, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 4906, + "n": 1536, + "k": 10240, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 4906, + "n": 10240, + "k": 2048, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 4928, + "n": 24, + "k": 40960, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 4928, + "n": 64, + "k": 10240, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 4928, + "n": 192, + "k": 10240, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 4928, + "n": 256, + "k": 10240, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 4928, + "n": 320, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 4928, + "n": 512, + "k": 10240, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 4928, + "n": 512, + "k": 10240, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 4928, + "n": 1024, + "k": 10240, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 4928, + "n": 1536, + "k": 10240, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 4928, + "n": 4096, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 4928, + "n": 7168, + "k": 1024, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 4928, + "n": 10240, + "k": 2048, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 4936, + "n": 24, + "k": 40960, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 4936, + "n": 64, + "k": 10240, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 4936, + "n": 192, + "k": 10240, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 4936, + "n": 256, + "k": 10240, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 4936, + "n": 320, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 4936, + "n": 512, + "k": 10240, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 4936, + "n": 512, + "k": 10240, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 4936, + "n": 1024, + "k": 10240, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 4936, + "n": 1536, + "k": 10240, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 4936, + "n": 10240, + "k": 2048, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 4937, + "n": 10240, + "k": 1024, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 4938, + "n": 896, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 4938, + "n": 7168, + "k": 448, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 4944, + "n": 7168, + "k": 1024, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 4992, + "n": 24, + "k": 40960, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 4992, + "n": 64, + "k": 10240, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 4992, + "n": 192, + "k": 10240, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 4992, + "n": 256, + "k": 10240, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 4992, + "n": 320, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 4992, + "n": 512, + "k": 10240, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 4992, + "n": 512, + "k": 10240, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 4992, + "n": 1024, + "k": 10240, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 4992, + "n": 1024, + "k": 10240, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 4992, + "n": 1536, + "k": 10240, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 4992, + "n": 4096, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 4992, + "n": 7168, + "k": 1024, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 4992, + "n": 10240, + "k": 512, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 4992, + "n": 10240, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 4992, + "n": 10240, + "k": 2048, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 4992, + "n": 12288, + "k": 2048, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 5000, + "n": 10240, + "k": 1024, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 5024, + "n": 24, + "k": 40960, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 5024, + "n": 64, + "k": 10240, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 5024, + "n": 192, + "k": 10240, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 5024, + "n": 256, + "k": 10240, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 5024, + "n": 320, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 5024, + "n": 512, + "k": 10240, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 5024, + "n": 512, + "k": 10240, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 5024, + "n": 896, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 5024, + "n": 1024, + "k": 10240, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 5024, + "n": 1536, + "k": 10240, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 5024, + "n": 4096, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 5024, + "n": 7168, + "k": 448, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 5024, + "n": 7168, + "k": 1024, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 5024, + "n": 10240, + "k": 1024, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 5056, + "n": 1024, + "k": 10240, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 5056, + "n": 10240, + "k": 512, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 5056, + "n": 10240, + "k": 1024, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 5064, + "n": 1024, + "k": 10240, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 5064, + "n": 10240, + "k": 512, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 5120, + "n": 24, + "k": 40960, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 5120, + "n": 64, + "k": 10240, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 5120, + "n": 192, + "k": 10240, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 5120, + "n": 256, + "k": 10240, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 5120, + "n": 320, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 5120, + "n": 512, + "k": 10240, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 5120, + "n": 512, + "k": 10240, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 5120, + "n": 1024, + "k": 10240, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 5120, + "n": 1536, + "k": 10240, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 5120, + "n": 7168, + "k": 1024, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 5120, + "n": 10240, + "k": 1024, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 5120, + "n": 10240, + "k": 2048, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 5136, + "n": 24, + "k": 40960, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 5136, + "n": 64, + "k": 10240, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 5136, + "n": 192, + "k": 10240, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 5136, + "n": 256, + "k": 10240, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 5136, + "n": 320, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 5136, + "n": 512, + "k": 10240, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 5136, + "n": 512, + "k": 10240, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 5136, + "n": 1024, + "k": 10240, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 5136, + "n": 1536, + "k": 10240, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 5136, + "n": 10240, + "k": 1024, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 5144, + "n": 6144, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3" + ] + }, + { + "type": "gemm", + "args": { + "m": 5144, + "n": 7168, + "k": 3072, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w2" + ] + }, + { + "type": "gemm", + "args": { + "m": 5144, + "n": 7168, + "k": 16384, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 5144, + "n": 65536, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 5168, + "n": 10240, + "k": 2048, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 5186, + "n": 6144, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3" + ] + }, + { + "type": "gemm", + "args": { + "m": 5186, + "n": 7168, + "k": 3072, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w2" + ] + }, + { + "type": "gemm", + "args": { + "m": 5192, + "n": 896, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 5192, + "n": 7168, + "k": 448, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 5202, + "n": 7168, + "k": 2048, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 5216, + "n": 10240, + "k": 2048, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 5228, + "n": 768, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w3" + ] + }, + { + "type": "gemm", + "args": { + "m": 5228, + "n": 7168, + "k": 384, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w2" + ] + }, + { + "type": "gemm", + "args": { + "m": 5248, + "n": 768, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w3" + ] + }, + { + "type": "gemm", + "args": { + "m": 5248, + "n": 2048, + "k": 12288, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 5248, + "n": 7168, + "k": 384, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w2" + ] + }, + { + "type": "gemm", + "args": { + "m": 5248, + "n": 7168, + "k": 2048, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 5248, + "n": 10240, + "k": 1024, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 5248, + "n": 12288, + "k": 1024, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 5256, + "n": 10240, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 5256, + "n": 12288, + "k": 2048, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 5264, + "n": 896, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 5264, + "n": 7168, + "k": 448, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 5297, + "n": 8192, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 5299, + "n": 24, + "k": 28672, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/hc_attn", + "deepseek-ai/DeepSeek-V4-Pro-0813/hc_ffn", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.hc_attn", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.hc_ffn", + "deepseek-ai/DeepSeek-V4-Pro/hc_attn", + "deepseek-ai/DeepSeek-V4-Pro/hc_ffn", + "deepseek-ai/DeepSeek-V4-Pro/mtp.hc_attn", + "deepseek-ai/DeepSeek-V4-Pro/mtp.hc_ffn", + "sgl-project/DeepSeek-V4-Pro-FP8/hc_attn", + "sgl-project/DeepSeek-V4-Pro-FP8/hc_ffn", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.hc_attn", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.hc_ffn" + ] + }, + { + "type": "gemm", + "args": { + "m": 5299, + "n": 64, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.weights_proj", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.weights_proj", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.weights_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 5299, + "n": 256, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.compressor.wkv", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.compressor.wkv", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.compressor.wgate", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.compressor.wkv" + ] + }, + { + "type": "gemm", + "args": { + "m": 5299, + "n": 512, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wkv", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wkv", + "deepseek-ai/DeepSeek-V4-Pro/attn.wkv", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wkv", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wkv", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wkv" + ] + }, + { + "type": "gemm", + "args": { + "m": 5299, + "n": 512, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.compressor.wkv+wgate", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.compressor.wkv+wgate", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.compressor.wkv+wgate" + ] + }, + { + "type": "gemm", + "args": { + "m": 5299, + "n": 1024, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.compressor.wkv", + "deepseek-ai/DeepSeek-V4-Pro/attn.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro/attn.compressor.wkv", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.compressor.wgate", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.compressor.wkv" + ] + }, + { + "type": "gemm", + "args": { + "m": 5299, + "n": 1536, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w3" + ] + }, + { + "type": "gemm", + "args": { + "m": 5312, + "n": 24, + "k": 28672, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/hc_attn", + "deepseek-ai/DeepSeek-V4-Pro-0813/hc_ffn", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.hc_attn", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.hc_ffn", + "deepseek-ai/DeepSeek-V4-Pro/hc_attn", + "deepseek-ai/DeepSeek-V4-Pro/hc_ffn", + "deepseek-ai/DeepSeek-V4-Pro/mtp.hc_attn", + "deepseek-ai/DeepSeek-V4-Pro/mtp.hc_ffn", + "sgl-project/DeepSeek-V4-Pro-FP8/hc_attn", + "sgl-project/DeepSeek-V4-Pro-FP8/hc_ffn", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.hc_attn", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.hc_ffn" + ] + }, + { + "type": "gemm", + "args": { + "m": 5312, + "n": 64, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.weights_proj", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.weights_proj", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.weights_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 5312, + "n": 256, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.compressor.wkv", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.compressor.wkv", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.compressor.wgate", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.compressor.wkv" + ] + }, + { + "type": "gemm", + "args": { + "m": 5312, + "n": 384, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.gate", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.gate", + "deepseek-ai/DeepSeek-V4-Pro/ffn.gate", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.gate", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.gate", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.gate" + ] + }, + { + "type": "gemm", + "args": { + "m": 5312, + "n": 512, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wkv", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wkv", + "deepseek-ai/DeepSeek-V4-Pro/attn.wkv", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wkv", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wkv", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wkv" + ] + }, + { + "type": "gemm", + "args": { + "m": 5312, + "n": 512, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.compressor.wkv+wgate", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.compressor.wkv+wgate", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.compressor.wkv+wgate" + ] + }, + { + "type": "gemm", + "args": { + "m": 5312, + "n": 1024, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.compressor.wkv", + "deepseek-ai/DeepSeek-V4-Pro/attn.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro/attn.compressor.wkv", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.compressor.wgate", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.compressor.wkv" + ] + }, + { + "type": "gemm", + "args": { + "m": 5312, + "n": 1536, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w3" + ] + }, + { + "type": "gemm", + "args": { + "m": 5312, + "n": 7168, + "k": 2048, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 5312, + "n": 8192, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 5312, + "n": 10240, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 5312, + "n": 12288, + "k": 2048, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 5313, + "n": 384, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.gate", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.gate", + "deepseek-ai/DeepSeek-V4-Pro/ffn.gate", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.gate", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.gate", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.gate" + ] + }, + { + "type": "gemm", + "args": { + "m": 5320, + "n": 24, + "k": 28672, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/hc_attn", + "deepseek-ai/DeepSeek-V4-Pro-0813/hc_ffn", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.hc_attn", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.hc_ffn", + "deepseek-ai/DeepSeek-V4-Pro/hc_attn", + "deepseek-ai/DeepSeek-V4-Pro/hc_ffn", + "deepseek-ai/DeepSeek-V4-Pro/mtp.hc_attn", + "deepseek-ai/DeepSeek-V4-Pro/mtp.hc_ffn", + "sgl-project/DeepSeek-V4-Pro-FP8/hc_attn", + "sgl-project/DeepSeek-V4-Pro-FP8/hc_ffn", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.hc_attn", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.hc_ffn" + ] + }, + { + "type": "gemm", + "args": { + "m": 5320, + "n": 64, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.weights_proj", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.weights_proj", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.weights_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 5320, + "n": 256, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.compressor.wkv", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.compressor.wkv", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.compressor.wgate", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.compressor.wkv" + ] + }, + { + "type": "gemm", + "args": { + "m": 5320, + "n": 384, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.gate", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.gate", + "deepseek-ai/DeepSeek-V4-Pro/ffn.gate", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.gate", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.gate", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.gate" + ] + }, + { + "type": "gemm", + "args": { + "m": 5320, + "n": 512, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wkv", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wkv", + "deepseek-ai/DeepSeek-V4-Pro/attn.wkv", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wkv", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wkv", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wkv" + ] + }, + { + "type": "gemm", + "args": { + "m": 5320, + "n": 512, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.compressor.wkv+wgate", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.compressor.wkv+wgate", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.compressor.wkv+wgate" + ] + }, + { + "type": "gemm", + "args": { + "m": 5320, + "n": 1024, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.compressor.wkv", + "deepseek-ai/DeepSeek-V4-Pro/attn.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro/attn.compressor.wkv", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.compressor.wgate", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.compressor.wkv" + ] + }, + { + "type": "gemm", + "args": { + "m": 5320, + "n": 1536, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w3" + ] + }, + { + "type": "gemm", + "args": { + "m": 5320, + "n": 8192, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 5328, + "n": 24, + "k": 28672, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/hc_attn", + "deepseek-ai/DeepSeek-V4-Pro-0813/hc_ffn", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.hc_attn", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.hc_ffn", + "deepseek-ai/DeepSeek-V4-Pro/hc_attn", + "deepseek-ai/DeepSeek-V4-Pro/hc_ffn", + "deepseek-ai/DeepSeek-V4-Pro/mtp.hc_attn", + "deepseek-ai/DeepSeek-V4-Pro/mtp.hc_ffn", + "sgl-project/DeepSeek-V4-Pro-FP8/hc_attn", + "sgl-project/DeepSeek-V4-Pro-FP8/hc_ffn", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.hc_attn", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.hc_ffn" + ] + }, + { + "type": "gemm", + "args": { + "m": 5328, + "n": 64, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.weights_proj", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.weights_proj", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.weights_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 5328, + "n": 256, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.compressor.wkv", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.compressor.wkv", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.compressor.wgate", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.compressor.wkv" + ] + }, + { + "type": "gemm", + "args": { + "m": 5328, + "n": 384, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.gate", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.gate", + "deepseek-ai/DeepSeek-V4-Pro/ffn.gate", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.gate", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.gate", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.gate" + ] + }, + { + "type": "gemm", + "args": { + "m": 5328, + "n": 512, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wkv", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wkv", + "deepseek-ai/DeepSeek-V4-Pro/attn.wkv", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wkv", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wkv", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wkv" + ] + }, + { + "type": "gemm", + "args": { + "m": 5328, + "n": 512, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.compressor.wkv+wgate", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.compressor.wkv+wgate", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.compressor.wkv+wgate" + ] + }, + { + "type": "gemm", + "args": { + "m": 5328, + "n": 768, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w3" + ] + }, + { + "type": "gemm", + "args": { + "m": 5328, + "n": 1024, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.compressor.wkv", + "deepseek-ai/DeepSeek-V4-Pro/attn.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro/attn.compressor.wkv", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.compressor.wgate", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.compressor.wkv" + ] + }, + { + "type": "gemm", + "args": { + "m": 5328, + "n": 1536, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w3" + ] + }, + { + "type": "gemm", + "args": { + "m": 5328, + "n": 7168, + "k": 384, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w2" + ] + }, + { + "type": "gemm", + "args": { + "m": 5328, + "n": 7168, + "k": 2048, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 5328, + "n": 8192, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 5336, + "n": 768, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w3" + ] + }, + { + "type": "gemm", + "args": { + "m": 5336, + "n": 7168, + "k": 384, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w2" + ] + }, + { + "type": "gemm", + "args": { + "m": 5336, + "n": 7168, + "k": 2048, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 5340, + "n": 1536, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w3" + ] + }, + { + "type": "gemm", + "args": { + "m": 5340, + "n": 7168, + "k": 768, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w2" + ] + }, + { + "type": "gemm", + "args": { + "m": 5340, + "n": 7168, + "k": 4096, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 5340, + "n": 16384, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 5344, + "n": 24, + "k": 28672, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/hc_attn", + "deepseek-ai/DeepSeek-V4-Pro-0813/hc_ffn", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.hc_attn", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.hc_ffn", + "deepseek-ai/DeepSeek-V4-Pro/hc_attn", + "deepseek-ai/DeepSeek-V4-Pro/hc_ffn", + "deepseek-ai/DeepSeek-V4-Pro/mtp.hc_attn", + "deepseek-ai/DeepSeek-V4-Pro/mtp.hc_ffn", + "sgl-project/DeepSeek-V4-Pro-FP8/hc_attn", + "sgl-project/DeepSeek-V4-Pro-FP8/hc_ffn", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.hc_attn", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.hc_ffn" + ] + }, + { + "type": "gemm", + "args": { + "m": 5344, + "n": 64, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.weights_proj", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.weights_proj", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.weights_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 5344, + "n": 256, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.compressor.wkv", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.compressor.wkv", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.compressor.wgate", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.compressor.wkv" + ] + }, + { + "type": "gemm", + "args": { + "m": 5344, + "n": 384, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.gate", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.gate", + "deepseek-ai/DeepSeek-V4-Pro/ffn.gate", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.gate", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.gate", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.gate" + ] + }, + { + "type": "gemm", + "args": { + "m": 5344, + "n": 512, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wkv", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wkv", + "deepseek-ai/DeepSeek-V4-Pro/attn.wkv", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wkv", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wkv", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wkv" + ] + }, + { + "type": "gemm", + "args": { + "m": 5344, + "n": 512, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.compressor.wkv+wgate", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.compressor.wkv+wgate", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.compressor.wkv+wgate" + ] + }, + { + "type": "gemm", + "args": { + "m": 5344, + "n": 1024, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.compressor.wkv", + "deepseek-ai/DeepSeek-V4-Pro/attn.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro/attn.compressor.wkv", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.compressor.wgate", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.compressor.wkv" + ] + }, + { + "type": "gemm", + "args": { + "m": 5344, + "n": 1536, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w3" + ] + }, + { + "type": "gemm", + "args": { + "m": 5344, + "n": 7168, + "k": 768, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w2" + ] + }, + { + "type": "gemm", + "args": { + "m": 5344, + "n": 7168, + "k": 4096, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 5344, + "n": 8192, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 5344, + "n": 16384, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 5352, + "n": 1536, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w3" + ] + }, + { + "type": "gemm", + "args": { + "m": 5352, + "n": 7168, + "k": 768, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w2" + ] + }, + { + "type": "gemm", + "args": { + "m": 5352, + "n": 7168, + "k": 4096, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 5352, + "n": 16384, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 5376, + "n": 24, + "k": 28672, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/hc_attn", + "deepseek-ai/DeepSeek-V4-Pro-0813/hc_ffn", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.hc_attn", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.hc_ffn", + "deepseek-ai/DeepSeek-V4-Pro/hc_attn", + "deepseek-ai/DeepSeek-V4-Pro/hc_ffn", + "deepseek-ai/DeepSeek-V4-Pro/mtp.hc_attn", + "deepseek-ai/DeepSeek-V4-Pro/mtp.hc_ffn", + "sgl-project/DeepSeek-V4-Pro-FP8/hc_attn", + "sgl-project/DeepSeek-V4-Pro-FP8/hc_ffn", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.hc_attn", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.hc_ffn" + ] + }, + { + "type": "gemm", + "args": { + "m": 5376, + "n": 64, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.weights_proj", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.weights_proj", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.weights_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 5376, + "n": 256, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.compressor.wkv", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.compressor.wkv", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.compressor.wgate", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.compressor.wkv" + ] + }, + { + "type": "gemm", + "args": { + "m": 5376, + "n": 384, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.gate", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.gate", + "deepseek-ai/DeepSeek-V4-Pro/ffn.gate", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.gate", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.gate", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.gate" + ] + }, + { + "type": "gemm", + "args": { + "m": 5376, + "n": 384, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w3" + ] + }, + { + "type": "gemm", + "args": { + "m": 5376, + "n": 512, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wkv", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wkv", + "deepseek-ai/DeepSeek-V4-Pro/attn.wkv", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wkv", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wkv", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wkv" + ] + }, + { + "type": "gemm", + "args": { + "m": 5376, + "n": 512, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.compressor.wkv+wgate", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.compressor.wkv+wgate", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.compressor.wkv+wgate" + ] + }, + { + "type": "gemm", + "args": { + "m": 5376, + "n": 1024, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.compressor.wkv", + "deepseek-ai/DeepSeek-V4-Pro/attn.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro/attn.compressor.wkv", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.compressor.wgate", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.compressor.wkv" + ] + }, + { + "type": "gemm", + "args": { + "m": 5376, + "n": 1536, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w3" + ] + }, + { + "type": "gemm", + "args": { + "m": 5376, + "n": 7168, + "k": 192, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w2" + ] + }, + { + "type": "gemm", + "args": { + "m": 5376, + "n": 7168, + "k": 768, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w2" + ] + }, + { + "type": "gemm", + "args": { + "m": 5376, + "n": 7168, + "k": 2048, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 5376, + "n": 7168, + "k": 4096, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 5376, + "n": 8192, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 5376, + "n": 16384, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 5380, + "n": 384, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w3" + ] + }, + { + "type": "gemm", + "args": { + "m": 5380, + "n": 7168, + "k": 192, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w2" + ] + }, + { + "type": "gemm", + "args": { + "m": 5392, + "n": 1536, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w3" + ] + }, + { + "type": "gemm", + "args": { + "m": 5392, + "n": 7168, + "k": 768, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w2" + ] + }, + { + "type": "gemm", + "args": { + "m": 5392, + "n": 7168, + "k": 4096, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 5392, + "n": 16384, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 5408, + "n": 24, + "k": 49152, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 5408, + "n": 64, + "k": 12288, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 5408, + "n": 120, + "k": 12288, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 5408, + "n": 256, + "k": 12288, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 5408, + "n": 512, + "k": 12288, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 5408, + "n": 512, + "k": 12288, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 5408, + "n": 1024, + "k": 12288, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 5408, + "n": 1536, + "k": 12288, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 5408, + "n": 5120, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 5408, + "n": 12288, + "k": 1024, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 5440, + "n": 384, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w3" + ] + }, + { + "type": "gemm", + "args": { + "m": 5440, + "n": 768, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w3" + ] + }, + { + "type": "gemm", + "args": { + "m": 5440, + "n": 896, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 5440, + "n": 1536, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w3" + ] + }, + { + "type": "gemm", + "args": { + "m": 5440, + "n": 2048, + "k": 12288, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 5440, + "n": 7168, + "k": 192, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w2" + ] + }, + { + "type": "gemm", + "args": { + "m": 5440, + "n": 7168, + "k": 384, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w2" + ] + }, + { + "type": "gemm", + "args": { + "m": 5440, + "n": 7168, + "k": 448, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 5440, + "n": 7168, + "k": 768, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w2" + ] + }, + { + "type": "gemm", + "args": { + "m": 5440, + "n": 7168, + "k": 4096, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 5440, + "n": 12288, + "k": 1024, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 5440, + "n": 16384, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 5441, + "n": 1792, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 5441, + "n": 7168, + "k": 896, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 5504, + "n": 24, + "k": 28672, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/hc_attn", + "deepseek-ai/DeepSeek-V4-Pro-0813/hc_ffn", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.hc_attn", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.hc_ffn", + "deepseek-ai/DeepSeek-V4-Pro/hc_attn", + "deepseek-ai/DeepSeek-V4-Pro/hc_ffn", + "deepseek-ai/DeepSeek-V4-Pro/mtp.hc_attn", + "deepseek-ai/DeepSeek-V4-Pro/mtp.hc_ffn", + "sgl-project/DeepSeek-V4-Pro-FP8/hc_attn", + "sgl-project/DeepSeek-V4-Pro-FP8/hc_ffn", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.hc_attn", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.hc_ffn" + ] + }, + { + "type": "gemm", + "args": { + "m": 5504, + "n": 64, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.weights_proj", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.weights_proj", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.weights_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 5504, + "n": 256, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.compressor.wkv", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.compressor.wkv", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.compressor.wgate", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.compressor.wkv" + ] + }, + { + "type": "gemm", + "args": { + "m": 5504, + "n": 384, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.gate", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.gate", + "deepseek-ai/DeepSeek-V4-Pro/ffn.gate", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.gate", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.gate", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.gate" + ] + }, + { + "type": "gemm", + "args": { + "m": 5504, + "n": 384, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w3" + ] + }, + { + "type": "gemm", + "args": { + "m": 5504, + "n": 512, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wkv", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wkv", + "deepseek-ai/DeepSeek-V4-Pro/attn.wkv", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wkv", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wkv", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wkv" + ] + }, + { + "type": "gemm", + "args": { + "m": 5504, + "n": 512, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.compressor.wkv+wgate", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.compressor.wkv+wgate", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.compressor.wkv+wgate" + ] + }, + { + "type": "gemm", + "args": { + "m": 5504, + "n": 1024, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.compressor.wkv", + "deepseek-ai/DeepSeek-V4-Pro/attn.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro/attn.compressor.wkv", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.compressor.wgate", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.compressor.wkv" + ] + }, + { + "type": "gemm", + "args": { + "m": 5504, + "n": 1536, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w3" + ] + }, + { + "type": "gemm", + "args": { + "m": 5504, + "n": 1792, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 5504, + "n": 7168, + "k": 192, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w2" + ] + }, + { + "type": "gemm", + "args": { + "m": 5504, + "n": 7168, + "k": 768, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w2" + ] + }, + { + "type": "gemm", + "args": { + "m": 5504, + "n": 7168, + "k": 896, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 5504, + "n": 7168, + "k": 4096, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 5504, + "n": 8192, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 5504, + "n": 16384, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 5520, + "n": 384, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w3" + ] + }, + { + "type": "gemm", + "args": { + "m": 5520, + "n": 7168, + "k": 192, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w2" + ] + }, + { + "type": "gemm", + "args": { + "m": 5568, + "n": 1792, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 5568, + "n": 7168, + "k": 896, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 5632, + "n": 1792, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 5632, + "n": 7168, + "k": 896, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 5640, + "n": 1792, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 5640, + "n": 7168, + "k": 896, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 5709, + "n": 24, + "k": 49152, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 5709, + "n": 64, + "k": 12288, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 5709, + "n": 120, + "k": 12288, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 5709, + "n": 256, + "k": 12288, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 5709, + "n": 512, + "k": 12288, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 5709, + "n": 512, + "k": 12288, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 5709, + "n": 1024, + "k": 12288, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 5709, + "n": 1536, + "k": 12288, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 5709, + "n": 5120, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 5709, + "n": 12288, + "k": 1024, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 5720, + "n": 24, + "k": 49152, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 5720, + "n": 64, + "k": 12288, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 5720, + "n": 120, + "k": 12288, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 5720, + "n": 256, + "k": 12288, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 5720, + "n": 512, + "k": 12288, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 5720, + "n": 512, + "k": 12288, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 5720, + "n": 1024, + "k": 12288, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 5720, + "n": 1536, + "k": 12288, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 5720, + "n": 5120, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 5720, + "n": 12288, + "k": 1024, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 5760, + "n": 24, + "k": 49152, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 5760, + "n": 64, + "k": 12288, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 5760, + "n": 120, + "k": 12288, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 5760, + "n": 256, + "k": 12288, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 5760, + "n": 512, + "k": 12288, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 5760, + "n": 512, + "k": 12288, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 5760, + "n": 1024, + "k": 12288, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 5760, + "n": 1536, + "k": 12288, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 5760, + "n": 5120, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 5760, + "n": 12288, + "k": 1024, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 5768, + "n": 384, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w3" + ] + }, + { + "type": "gemm", + "args": { + "m": 5768, + "n": 7168, + "k": 192, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w2" + ] + }, + { + "type": "gemm", + "args": { + "m": 5792, + "n": 24, + "k": 49152, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 5792, + "n": 64, + "k": 12288, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 5792, + "n": 120, + "k": 12288, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 5792, + "n": 256, + "k": 12288, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 5792, + "n": 512, + "k": 12288, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 5792, + "n": 512, + "k": 12288, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 5792, + "n": 1024, + "k": 12288, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 5792, + "n": 1536, + "k": 12288, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 5792, + "n": 5120, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 5792, + "n": 12288, + "k": 1024, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 5824, + "n": 24, + "k": 49152, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 5824, + "n": 64, + "k": 12288, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 5824, + "n": 120, + "k": 12288, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 5824, + "n": 256, + "k": 12288, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 5824, + "n": 512, + "k": 12288, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 5824, + "n": 512, + "k": 12288, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 5824, + "n": 1024, + "k": 12288, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 5824, + "n": 1536, + "k": 12288, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 5824, + "n": 5120, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 5824, + "n": 12288, + "k": 1024, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 5888, + "n": 24, + "k": 49152, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 5888, + "n": 64, + "k": 12288, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 5888, + "n": 120, + "k": 12288, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 5888, + "n": 256, + "k": 12288, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 5888, + "n": 512, + "k": 12288, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 5888, + "n": 512, + "k": 12288, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 5888, + "n": 1024, + "k": 12288, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 5888, + "n": 1536, + "k": 12288, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 5888, + "n": 5120, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 5888, + "n": 6144, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3" + ] + }, + { + "type": "gemm", + "args": { + "m": 5888, + "n": 7168, + "k": 3072, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w2" + ] + }, + { + "type": "gemm", + "args": { + "m": 5888, + "n": 7168, + "k": 16384, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 5888, + "n": 10240, + "k": 16384, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 5888, + "n": 12288, + "k": 1024, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 5888, + "n": 65536, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 5952, + "n": 6144, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3" + ] + }, + { + "type": "gemm", + "args": { + "m": 5952, + "n": 7168, + "k": 3072, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w2" + ] + }, + { + "type": "gemm", + "args": { + "m": 5952, + "n": 7168, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.e_proj", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.h_proj", + "deepseek-ai/DeepSeek-V4-Pro/mtp.e_proj", + "deepseek-ai/DeepSeek-V4-Pro/mtp.h_proj", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.e_proj", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.h_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 5952, + "n": 7168, + "k": 16384, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 5952, + "n": 10240, + "k": 16384, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 5952, + "n": 12288, + "k": 8192, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 5952, + "n": 12288, + "k": 16384, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 5952, + "n": 14336, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 5952, + "n": 16384, + "k": 12288, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 5952, + "n": 65536, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 5952, + "n": 81920, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 5968, + "n": 24, + "k": 49152, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 5968, + "n": 64, + "k": 12288, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 5968, + "n": 120, + "k": 12288, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 5968, + "n": 256, + "k": 12288, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 5968, + "n": 512, + "k": 12288, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 5968, + "n": 512, + "k": 12288, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 5968, + "n": 1024, + "k": 12288, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 5968, + "n": 1536, + "k": 12288, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 5968, + "n": 5120, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 5968, + "n": 12288, + "k": 1024, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 6016, + "n": 1024, + "k": 10240, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 6016, + "n": 6144, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3" + ] + }, + { + "type": "gemm", + "args": { + "m": 6016, + "n": 7168, + "k": 3072, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w2" + ] + }, + { + "type": "gemm", + "args": { + "m": 6016, + "n": 7168, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.e_proj", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.h_proj", + "deepseek-ai/DeepSeek-V4-Pro/mtp.e_proj", + "deepseek-ai/DeepSeek-V4-Pro/mtp.h_proj", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.e_proj", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.h_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 6016, + "n": 7168, + "k": 16384, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 6016, + "n": 10240, + "k": 512, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 6016, + "n": 10240, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 6016, + "n": 10240, + "k": 16384, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 6016, + "n": 12288, + "k": 2048, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 6016, + "n": 12288, + "k": 8192, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 6016, + "n": 12288, + "k": 16384, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 6016, + "n": 14336, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 6016, + "n": 16384, + "k": 12288, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 6016, + "n": 65536, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 6016, + "n": 81920, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 6017, + "n": 1024, + "k": 10240, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 6017, + "n": 10240, + "k": 512, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 6018, + "n": 10240, + "k": 16384, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 6018, + "n": 65536, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 6027, + "n": 10240, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 6027, + "n": 12288, + "k": 2048, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 6042, + "n": 8192, + "k": 10240, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 6042, + "n": 10240, + "k": 4096, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 6069, + "n": 7168, + "k": 16384, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 6078, + "n": 2048, + "k": 10240, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 6078, + "n": 10240, + "k": 1024, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 6078, + "n": 10240, + "k": 4096, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 6080, + "n": 1024, + "k": 10240, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 6080, + "n": 4096, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 6080, + "n": 10240, + "k": 512, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 6081, + "n": 4096, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 6088, + "n": 1024, + "k": 10240, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 6088, + "n": 4096, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 6088, + "n": 10240, + "k": 512, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 6103, + "n": 7168, + "k": 1024, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 6104, + "n": 7168, + "k": 1024, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 6134, + "n": 320, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 6144, + "n": 24, + "k": 40960, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 6144, + "n": 64, + "k": 10240, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 6144, + "n": 192, + "k": 10240, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 6144, + "n": 256, + "k": 10240, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 6144, + "n": 320, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 6144, + "n": 512, + "k": 10240, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 6144, + "n": 512, + "k": 10240, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 6144, + "n": 1024, + "k": 10240, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 6144, + "n": 1024, + "k": 10240, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 6144, + "n": 1536, + "k": 10240, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 6144, + "n": 2048, + "k": 12288, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 6144, + "n": 4096, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 6144, + "n": 6144, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3" + ] + }, + { + "type": "gemm", + "args": { + "m": 6144, + "n": 7168, + "k": 1024, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 6144, + "n": 7168, + "k": 3072, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w2" + ] + }, + { + "type": "gemm", + "args": { + "m": 6144, + "n": 7168, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.e_proj", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.h_proj", + "deepseek-ai/DeepSeek-V4-Pro/mtp.e_proj", + "deepseek-ai/DeepSeek-V4-Pro/mtp.h_proj", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.e_proj", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.h_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 6144, + "n": 10240, + "k": 512, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 6144, + "n": 10240, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 6144, + "n": 10240, + "k": 2048, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 6144, + "n": 12288, + "k": 1024, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 6144, + "n": 12288, + "k": 2048, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 6144, + "n": 12288, + "k": 8192, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 6144, + "n": 12288, + "k": 16384, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 6144, + "n": 14336, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 6144, + "n": 16384, + "k": 12288, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 6144, + "n": 81920, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 6148, + "n": 24, + "k": 40960, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 6148, + "n": 64, + "k": 10240, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 6148, + "n": 192, + "k": 10240, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 6148, + "n": 256, + "k": 10240, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 6148, + "n": 512, + "k": 10240, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 6148, + "n": 512, + "k": 10240, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 6148, + "n": 1024, + "k": 10240, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 6148, + "n": 1536, + "k": 10240, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 6148, + "n": 10240, + "k": 2048, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 6160, + "n": 24, + "k": 40960, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 6160, + "n": 64, + "k": 10240, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 6160, + "n": 192, + "k": 10240, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 6160, + "n": 256, + "k": 10240, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 6160, + "n": 320, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 6160, + "n": 512, + "k": 10240, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 6160, + "n": 512, + "k": 10240, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 6160, + "n": 1024, + "k": 10240, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 6160, + "n": 1024, + "k": 10240, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 6160, + "n": 1536, + "k": 10240, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 6160, + "n": 4096, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 6160, + "n": 7168, + "k": 1024, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 6160, + "n": 10240, + "k": 512, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 6160, + "n": 10240, + "k": 2048, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 6176, + "n": 4096, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 6176, + "n": 7168, + "k": 1024, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 6181, + "n": 10240, + "k": 1024, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 6200, + "n": 320, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 6200, + "n": 896, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 6200, + "n": 7168, + "k": 448, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 6208, + "n": 24, + "k": 40960, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 6208, + "n": 64, + "k": 10240, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 6208, + "n": 192, + "k": 10240, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 6208, + "n": 256, + "k": 10240, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 6208, + "n": 320, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 6208, + "n": 512, + "k": 10240, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 6208, + "n": 512, + "k": 10240, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 6208, + "n": 1024, + "k": 10240, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 6208, + "n": 1536, + "k": 10240, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 6208, + "n": 7168, + "k": 1024, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 6208, + "n": 10240, + "k": 2048, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 6210, + "n": 6144, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3" + ] + }, + { + "type": "gemm", + "args": { + "m": 6210, + "n": 7168, + "k": 3072, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w2" + ] + }, + { + "type": "gemm", + "args": { + "m": 6214, + "n": 896, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 6214, + "n": 7168, + "k": 448, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 6216, + "n": 24, + "k": 40960, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 6216, + "n": 64, + "k": 10240, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 6216, + "n": 192, + "k": 10240, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 6216, + "n": 256, + "k": 10240, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 6216, + "n": 512, + "k": 10240, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 6216, + "n": 512, + "k": 10240, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 6216, + "n": 1024, + "k": 10240, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 6216, + "n": 1536, + "k": 10240, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 6216, + "n": 10240, + "k": 1024, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 6272, + "n": 24, + "k": 40960, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 6272, + "n": 64, + "k": 10240, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 6272, + "n": 192, + "k": 10240, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 6272, + "n": 256, + "k": 10240, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 6272, + "n": 320, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 6272, + "n": 512, + "k": 10240, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 6272, + "n": 512, + "k": 10240, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 6272, + "n": 896, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 6272, + "n": 1024, + "k": 10240, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 6272, + "n": 1536, + "k": 10240, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 6272, + "n": 2048, + "k": 12288, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 6272, + "n": 4096, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 6272, + "n": 7168, + "k": 448, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 6272, + "n": 7168, + "k": 1024, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 6272, + "n": 10240, + "k": 1024, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 6272, + "n": 10240, + "k": 2048, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 6272, + "n": 12288, + "k": 1024, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 6275, + "n": 2048, + "k": 12288, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 6275, + "n": 12288, + "k": 1024, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 6336, + "n": 2048, + "k": 12288, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 6336, + "n": 10240, + "k": 1024, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 6336, + "n": 10240, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 6336, + "n": 12288, + "k": 1024, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 6336, + "n": 12288, + "k": 2048, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 6344, + "n": 10240, + "k": 2048, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 6384, + "n": 896, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 6384, + "n": 7168, + "k": 448, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 6400, + "n": 10240, + "k": 1024, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 6416, + "n": 10240, + "k": 1024, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 6432, + "n": 24, + "k": 40960, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 6432, + "n": 64, + "k": 10240, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 6432, + "n": 192, + "k": 10240, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 6432, + "n": 256, + "k": 10240, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 6432, + "n": 320, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 6432, + "n": 512, + "k": 10240, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 6432, + "n": 512, + "k": 10240, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 6432, + "n": 896, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 6432, + "n": 1024, + "k": 10240, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 6432, + "n": 1024, + "k": 10240, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 6432, + "n": 1536, + "k": 10240, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 6432, + "n": 7168, + "k": 448, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 6432, + "n": 10240, + "k": 512, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 6432, + "n": 10240, + "k": 2048, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 6528, + "n": 6144, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3" + ] + }, + { + "type": "gemm", + "args": { + "m": 6528, + "n": 7168, + "k": 2048, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 6528, + "n": 7168, + "k": 3072, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w2" + ] + }, + { + "type": "gemm", + "args": { + "m": 6531, + "n": 7168, + "k": 2048, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 6540, + "n": 768, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w3" + ] + }, + { + "type": "gemm", + "args": { + "m": 6540, + "n": 7168, + "k": 384, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w2" + ] + }, + { + "type": "gemm", + "args": { + "m": 6560, + "n": 384, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w3" + ] + }, + { + "type": "gemm", + "args": { + "m": 6560, + "n": 7168, + "k": 192, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w2" + ] + }, + { + "type": "gemm", + "args": { + "m": 6592, + "n": 7168, + "k": 2048, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 6622, + "n": 8192, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 6624, + "n": 24, + "k": 28672, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/hc_attn", + "deepseek-ai/DeepSeek-V4-Pro-0813/hc_ffn", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.hc_attn", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.hc_ffn", + "deepseek-ai/DeepSeek-V4-Pro/hc_attn", + "deepseek-ai/DeepSeek-V4-Pro/hc_ffn", + "deepseek-ai/DeepSeek-V4-Pro/mtp.hc_attn", + "deepseek-ai/DeepSeek-V4-Pro/mtp.hc_ffn", + "sgl-project/DeepSeek-V4-Pro-FP8/hc_attn", + "sgl-project/DeepSeek-V4-Pro-FP8/hc_ffn", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.hc_attn", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.hc_ffn" + ] + }, + { + "type": "gemm", + "args": { + "m": 6624, + "n": 64, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.weights_proj", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.weights_proj", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.weights_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 6624, + "n": 256, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.compressor.wkv", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.compressor.wkv", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.compressor.wgate", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.compressor.wkv" + ] + }, + { + "type": "gemm", + "args": { + "m": 6624, + "n": 512, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wkv", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wkv", + "deepseek-ai/DeepSeek-V4-Pro/attn.wkv", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wkv", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wkv", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wkv" + ] + }, + { + "type": "gemm", + "args": { + "m": 6624, + "n": 512, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.compressor.wkv+wgate", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.compressor.wkv+wgate", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.compressor.wkv+wgate" + ] + }, + { + "type": "gemm", + "args": { + "m": 6624, + "n": 1024, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.compressor.wkv", + "deepseek-ai/DeepSeek-V4-Pro/attn.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro/attn.compressor.wkv", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.compressor.wgate", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.compressor.wkv" + ] + }, + { + "type": "gemm", + "args": { + "m": 6624, + "n": 1536, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w3" + ] + }, + { + "type": "gemm", + "args": { + "m": 6624, + "n": 8192, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 6629, + "n": 24, + "k": 28672, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/hc_attn", + "deepseek-ai/DeepSeek-V4-Pro-0813/hc_ffn", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.hc_attn", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.hc_ffn", + "deepseek-ai/DeepSeek-V4-Pro/hc_attn", + "deepseek-ai/DeepSeek-V4-Pro/hc_ffn", + "deepseek-ai/DeepSeek-V4-Pro/mtp.hc_attn", + "deepseek-ai/DeepSeek-V4-Pro/mtp.hc_ffn", + "sgl-project/DeepSeek-V4-Pro-FP8/hc_attn", + "sgl-project/DeepSeek-V4-Pro-FP8/hc_ffn", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.hc_attn", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.hc_ffn" + ] + }, + { + "type": "gemm", + "args": { + "m": 6629, + "n": 64, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.weights_proj", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.weights_proj", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.weights_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 6629, + "n": 256, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.compressor.wkv", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.compressor.wkv", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.compressor.wgate", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.compressor.wkv" + ] + }, + { + "type": "gemm", + "args": { + "m": 6629, + "n": 512, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wkv", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wkv", + "deepseek-ai/DeepSeek-V4-Pro/attn.wkv", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wkv", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wkv", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wkv" + ] + }, + { + "type": "gemm", + "args": { + "m": 6629, + "n": 512, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.compressor.wkv+wgate", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.compressor.wkv+wgate", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.compressor.wkv+wgate" + ] + }, + { + "type": "gemm", + "args": { + "m": 6629, + "n": 1024, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.compressor.wkv", + "deepseek-ai/DeepSeek-V4-Pro/attn.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro/attn.compressor.wkv", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.compressor.wgate", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.compressor.wkv" + ] + }, + { + "type": "gemm", + "args": { + "m": 6629, + "n": 1536, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w3" + ] + }, + { + "type": "gemm", + "args": { + "m": 6632, + "n": 24, + "k": 28672, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/hc_attn", + "deepseek-ai/DeepSeek-V4-Pro-0813/hc_ffn", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.hc_attn", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.hc_ffn", + "deepseek-ai/DeepSeek-V4-Pro/hc_attn", + "deepseek-ai/DeepSeek-V4-Pro/hc_ffn", + "deepseek-ai/DeepSeek-V4-Pro/mtp.hc_attn", + "deepseek-ai/DeepSeek-V4-Pro/mtp.hc_ffn", + "sgl-project/DeepSeek-V4-Pro-FP8/hc_attn", + "sgl-project/DeepSeek-V4-Pro-FP8/hc_ffn", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.hc_attn", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.hc_ffn" + ] + }, + { + "type": "gemm", + "args": { + "m": 6632, + "n": 64, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.weights_proj", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.weights_proj", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.weights_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 6632, + "n": 256, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.compressor.wkv", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.compressor.wkv", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.compressor.wgate", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.compressor.wkv" + ] + }, + { + "type": "gemm", + "args": { + "m": 6632, + "n": 512, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wkv", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wkv", + "deepseek-ai/DeepSeek-V4-Pro/attn.wkv", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wkv", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wkv", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wkv" + ] + }, + { + "type": "gemm", + "args": { + "m": 6632, + "n": 512, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.compressor.wkv+wgate", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.compressor.wkv+wgate", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.compressor.wkv+wgate" + ] + }, + { + "type": "gemm", + "args": { + "m": 6632, + "n": 768, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w3" + ] + }, + { + "type": "gemm", + "args": { + "m": 6632, + "n": 1024, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.compressor.wkv", + "deepseek-ai/DeepSeek-V4-Pro/attn.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro/attn.compressor.wkv", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.compressor.wgate", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.compressor.wkv" + ] + }, + { + "type": "gemm", + "args": { + "m": 6632, + "n": 1536, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w3" + ] + }, + { + "type": "gemm", + "args": { + "m": 6632, + "n": 7168, + "k": 384, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w2" + ] + }, + { + "type": "gemm", + "args": { + "m": 6632, + "n": 7168, + "k": 2048, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 6632, + "n": 8192, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 6641, + "n": 384, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.gate", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.gate", + "deepseek-ai/DeepSeek-V4-Pro/ffn.gate", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.gate", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.gate", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.gate" + ] + }, + { + "type": "gemm", + "args": { + "m": 6656, + "n": 24, + "k": 28672, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/hc_attn", + "deepseek-ai/DeepSeek-V4-Pro-0813/hc_ffn", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.hc_attn", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.hc_ffn", + "deepseek-ai/DeepSeek-V4-Pro/hc_attn", + "deepseek-ai/DeepSeek-V4-Pro/hc_ffn", + "deepseek-ai/DeepSeek-V4-Pro/mtp.hc_attn", + "deepseek-ai/DeepSeek-V4-Pro/mtp.hc_ffn", + "sgl-project/DeepSeek-V4-Pro-FP8/hc_attn", + "sgl-project/DeepSeek-V4-Pro-FP8/hc_ffn", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.hc_attn", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.hc_ffn" + ] + }, + { + "type": "gemm", + "args": { + "m": 6656, + "n": 64, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.weights_proj", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.weights_proj", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.weights_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 6656, + "n": 256, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.compressor.wkv", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.compressor.wkv", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.compressor.wgate", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.compressor.wkv" + ] + }, + { + "type": "gemm", + "args": { + "m": 6656, + "n": 384, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.gate", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.gate", + "deepseek-ai/DeepSeek-V4-Pro/ffn.gate", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.gate", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.gate", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.gate" + ] + }, + { + "type": "gemm", + "args": { + "m": 6656, + "n": 512, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wkv", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wkv", + "deepseek-ai/DeepSeek-V4-Pro/attn.wkv", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wkv", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wkv", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wkv" + ] + }, + { + "type": "gemm", + "args": { + "m": 6656, + "n": 512, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.compressor.wkv+wgate", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.compressor.wkv+wgate", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.compressor.wkv+wgate" + ] + }, + { + "type": "gemm", + "args": { + "m": 6656, + "n": 1024, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.compressor.wkv", + "deepseek-ai/DeepSeek-V4-Pro/attn.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro/attn.compressor.wkv", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.compressor.wgate", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.compressor.wkv" + ] + }, + { + "type": "gemm", + "args": { + "m": 6656, + "n": 1536, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w3" + ] + }, + { + "type": "gemm", + "args": { + "m": 6656, + "n": 7168, + "k": 2048, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 6656, + "n": 8192, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 6664, + "n": 384, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.gate", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.gate", + "deepseek-ai/DeepSeek-V4-Pro/ffn.gate", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.gate", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.gate", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.gate" + ] + }, + { + "type": "gemm", + "args": { + "m": 6672, + "n": 24, + "k": 28672, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/hc_attn", + "deepseek-ai/DeepSeek-V4-Pro-0813/hc_ffn", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.hc_attn", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.hc_ffn", + "deepseek-ai/DeepSeek-V4-Pro/hc_attn", + "deepseek-ai/DeepSeek-V4-Pro/hc_ffn", + "deepseek-ai/DeepSeek-V4-Pro/mtp.hc_attn", + "deepseek-ai/DeepSeek-V4-Pro/mtp.hc_ffn", + "sgl-project/DeepSeek-V4-Pro-FP8/hc_attn", + "sgl-project/DeepSeek-V4-Pro-FP8/hc_ffn", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.hc_attn", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.hc_ffn" + ] + }, + { + "type": "gemm", + "args": { + "m": 6672, + "n": 64, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.weights_proj", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.weights_proj", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.weights_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 6672, + "n": 256, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.compressor.wkv", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.compressor.wkv", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.compressor.wgate", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.compressor.wkv" + ] + }, + { + "type": "gemm", + "args": { + "m": 6672, + "n": 384, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.gate", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.gate", + "deepseek-ai/DeepSeek-V4-Pro/ffn.gate", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.gate", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.gate", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.gate" + ] + }, + { + "type": "gemm", + "args": { + "m": 6672, + "n": 512, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wkv", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wkv", + "deepseek-ai/DeepSeek-V4-Pro/attn.wkv", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wkv", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wkv", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wkv" + ] + }, + { + "type": "gemm", + "args": { + "m": 6672, + "n": 512, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.compressor.wkv+wgate", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.compressor.wkv+wgate", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.compressor.wkv+wgate" + ] + }, + { + "type": "gemm", + "args": { + "m": 6672, + "n": 1024, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.compressor.wkv", + "deepseek-ai/DeepSeek-V4-Pro/attn.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro/attn.compressor.wkv", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.compressor.wgate", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.compressor.wkv" + ] + }, + { + "type": "gemm", + "args": { + "m": 6672, + "n": 1536, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w3" + ] + }, + { + "type": "gemm", + "args": { + "m": 6672, + "n": 8192, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 6677, + "n": 1536, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w3" + ] + }, + { + "type": "gemm", + "args": { + "m": 6677, + "n": 7168, + "k": 768, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w2" + ] + }, + { + "type": "gemm", + "args": { + "m": 6677, + "n": 7168, + "k": 4096, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 6677, + "n": 16384, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 6680, + "n": 1536, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w3" + ] + }, + { + "type": "gemm", + "args": { + "m": 6680, + "n": 7168, + "k": 768, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w2" + ] + }, + { + "type": "gemm", + "args": { + "m": 6680, + "n": 7168, + "k": 4096, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 6680, + "n": 16384, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 6688, + "n": 384, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.gate", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.gate", + "deepseek-ai/DeepSeek-V4-Pro/ffn.gate", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.gate", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.gate", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.gate" + ] + }, + { + "type": "gemm", + "args": { + "m": 6688, + "n": 1536, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w3" + ] + }, + { + "type": "gemm", + "args": { + "m": 6688, + "n": 7168, + "k": 768, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w2" + ] + }, + { + "type": "gemm", + "args": { + "m": 6688, + "n": 7168, + "k": 4096, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 6688, + "n": 16384, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 6720, + "n": 24, + "k": 28672, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/hc_attn", + "deepseek-ai/DeepSeek-V4-Pro-0813/hc_ffn", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.hc_attn", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.hc_ffn", + "deepseek-ai/DeepSeek-V4-Pro/hc_attn", + "deepseek-ai/DeepSeek-V4-Pro/hc_ffn", + "deepseek-ai/DeepSeek-V4-Pro/mtp.hc_attn", + "deepseek-ai/DeepSeek-V4-Pro/mtp.hc_ffn", + "sgl-project/DeepSeek-V4-Pro-FP8/hc_attn", + "sgl-project/DeepSeek-V4-Pro-FP8/hc_ffn", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.hc_attn", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.hc_ffn" + ] + }, + { + "type": "gemm", + "args": { + "m": 6720, + "n": 64, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.weights_proj", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.weights_proj", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.weights_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 6720, + "n": 256, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.compressor.wkv", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.compressor.wkv", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.compressor.wgate", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.compressor.wkv" + ] + }, + { + "type": "gemm", + "args": { + "m": 6720, + "n": 384, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.gate", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.gate", + "deepseek-ai/DeepSeek-V4-Pro/ffn.gate", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.gate", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.gate", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.gate" + ] + }, + { + "type": "gemm", + "args": { + "m": 6720, + "n": 512, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wkv", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wkv", + "deepseek-ai/DeepSeek-V4-Pro/attn.wkv", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wkv", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wkv", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wkv" + ] + }, + { + "type": "gemm", + "args": { + "m": 6720, + "n": 512, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.compressor.wkv+wgate", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.compressor.wkv+wgate", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.compressor.wkv+wgate" + ] + }, + { + "type": "gemm", + "args": { + "m": 6720, + "n": 768, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w3" + ] + }, + { + "type": "gemm", + "args": { + "m": 6720, + "n": 1024, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.compressor.wkv", + "deepseek-ai/DeepSeek-V4-Pro/attn.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro/attn.compressor.wkv", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.compressor.wgate", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.compressor.wkv" + ] + }, + { + "type": "gemm", + "args": { + "m": 6720, + "n": 1536, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w3" + ] + }, + { + "type": "gemm", + "args": { + "m": 6720, + "n": 7168, + "k": 384, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w2" + ] + }, + { + "type": "gemm", + "args": { + "m": 6720, + "n": 7168, + "k": 768, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w2" + ] + }, + { + "type": "gemm", + "args": { + "m": 6720, + "n": 7168, + "k": 4096, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 6720, + "n": 8192, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 6720, + "n": 16384, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 6723, + "n": 384, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w3" + ] + }, + { + "type": "gemm", + "args": { + "m": 6723, + "n": 7168, + "k": 192, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w2" + ] + }, + { + "type": "gemm", + "args": { + "m": 6736, + "n": 1536, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w3" + ] + }, + { + "type": "gemm", + "args": { + "m": 6736, + "n": 1792, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 6736, + "n": 6144, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3" + ] + }, + { + "type": "gemm", + "args": { + "m": 6736, + "n": 7168, + "k": 768, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w2" + ] + }, + { + "type": "gemm", + "args": { + "m": 6736, + "n": 7168, + "k": 896, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 6736, + "n": 7168, + "k": 2048, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 6736, + "n": 7168, + "k": 3072, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w2" + ] + }, + { + "type": "gemm", + "args": { + "m": 6736, + "n": 7168, + "k": 4096, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 6736, + "n": 7168, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.e_proj", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.h_proj", + "deepseek-ai/DeepSeek-V4-Pro/mtp.e_proj", + "deepseek-ai/DeepSeek-V4-Pro/mtp.h_proj", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.e_proj", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.h_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 6736, + "n": 7168, + "k": 16384, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 6736, + "n": 14336, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 6736, + "n": 16384, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 6736, + "n": 65536, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 6784, + "n": 24, + "k": 28672, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/hc_attn", + "deepseek-ai/DeepSeek-V4-Pro-0813/hc_ffn", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.hc_attn", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.hc_ffn", + "deepseek-ai/DeepSeek-V4-Pro/hc_attn", + "deepseek-ai/DeepSeek-V4-Pro/hc_ffn", + "deepseek-ai/DeepSeek-V4-Pro/mtp.hc_attn", + "deepseek-ai/DeepSeek-V4-Pro/mtp.hc_ffn", + "sgl-project/DeepSeek-V4-Pro-FP8/hc_attn", + "sgl-project/DeepSeek-V4-Pro-FP8/hc_ffn", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.hc_attn", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.hc_ffn" + ] + }, + { + "type": "gemm", + "args": { + "m": 6784, + "n": 64, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.weights_proj", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.weights_proj", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.weights_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 6784, + "n": 256, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.compressor.wkv", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.compressor.wkv", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.compressor.wgate", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.compressor.wkv" + ] + }, + { + "type": "gemm", + "args": { + "m": 6784, + "n": 384, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.gate", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.gate", + "deepseek-ai/DeepSeek-V4-Pro/ffn.gate", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.gate", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.gate", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.gate" + ] + }, + { + "type": "gemm", + "args": { + "m": 6784, + "n": 384, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w3" + ] + }, + { + "type": "gemm", + "args": { + "m": 6784, + "n": 512, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wkv", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wkv", + "deepseek-ai/DeepSeek-V4-Pro/attn.wkv", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wkv", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wkv", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wkv" + ] + }, + { + "type": "gemm", + "args": { + "m": 6784, + "n": 512, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.compressor.wkv+wgate", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.compressor.wkv+wgate", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.compressor.wkv+wgate" + ] + }, + { + "type": "gemm", + "args": { + "m": 6784, + "n": 1024, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.compressor.wkv", + "deepseek-ai/DeepSeek-V4-Pro/attn.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro/attn.compressor.wkv", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.compressor.wgate", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.compressor.wkv" + ] + }, + { + "type": "gemm", + "args": { + "m": 6784, + "n": 1536, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w3" + ] + }, + { + "type": "gemm", + "args": { + "m": 6784, + "n": 7168, + "k": 192, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w2" + ] + }, + { + "type": "gemm", + "args": { + "m": 6784, + "n": 7168, + "k": 768, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w2" + ] + }, + { + "type": "gemm", + "args": { + "m": 6784, + "n": 7168, + "k": 4096, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 6784, + "n": 8192, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 6784, + "n": 16384, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 6786, + "n": 12288, + "k": 8192, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 6786, + "n": 12288, + "k": 16384, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 6786, + "n": 16384, + "k": 12288, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 6786, + "n": 81920, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 6792, + "n": 10240, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 6792, + "n": 12288, + "k": 2048, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 6802, + "n": 1792, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 6802, + "n": 7168, + "k": 896, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 6808, + "n": 1792, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 6808, + "n": 7168, + "k": 896, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 6848, + "n": 384, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w3" + ] + }, + { + "type": "gemm", + "args": { + "m": 6848, + "n": 1792, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 6848, + "n": 7168, + "k": 192, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w2" + ] + }, + { + "type": "gemm", + "args": { + "m": 6848, + "n": 7168, + "k": 896, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 6864, + "n": 384, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w3" + ] + }, + { + "type": "gemm", + "args": { + "m": 6864, + "n": 768, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w3" + ] + }, + { + "type": "gemm", + "args": { + "m": 6864, + "n": 1792, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 6864, + "n": 7168, + "k": 192, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w2" + ] + }, + { + "type": "gemm", + "args": { + "m": 6864, + "n": 7168, + "k": 384, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w2" + ] + }, + { + "type": "gemm", + "args": { + "m": 6864, + "n": 7168, + "k": 896, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 6912, + "n": 1536, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w3" + ] + }, + { + "type": "gemm", + "args": { + "m": 6912, + "n": 1792, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 6912, + "n": 7168, + "k": 768, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w2" + ] + }, + { + "type": "gemm", + "args": { + "m": 6912, + "n": 7168, + "k": 896, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 6912, + "n": 7168, + "k": 4096, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 6912, + "n": 16384, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 7040, + "n": 1792, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 7040, + "n": 7168, + "k": 896, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 7112, + "n": 384, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w3" + ] + }, + { + "type": "gemm", + "args": { + "m": 7112, + "n": 1024, + "k": 12288, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 7112, + "n": 2048, + "k": 10240, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 7112, + "n": 6144, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3" + ] + }, + { + "type": "gemm", + "args": { + "m": 7112, + "n": 7168, + "k": 192, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w2" + ] + }, + { + "type": "gemm", + "args": { + "m": 7112, + "n": 7168, + "k": 3072, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w2" + ] + }, + { + "type": "gemm", + "args": { + "m": 7112, + "n": 7168, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.e_proj", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.h_proj", + "deepseek-ai/DeepSeek-V4-Pro/mtp.e_proj", + "deepseek-ai/DeepSeek-V4-Pro/mtp.h_proj", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.e_proj", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.h_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 7112, + "n": 7168, + "k": 16384, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 7112, + "n": 8192, + "k": 10240, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 7112, + "n": 10240, + "k": 1024, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 7112, + "n": 10240, + "k": 4096, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 7112, + "n": 10240, + "k": 16384, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 7112, + "n": 12288, + "k": 512, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 7112, + "n": 12288, + "k": 8192, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 7112, + "n": 12288, + "k": 16384, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 7112, + "n": 14336, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 7112, + "n": 16384, + "k": 12288, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 7112, + "n": 65536, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 7112, + "n": 81920, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 7141, + "n": 24, + "k": 49152, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 7141, + "n": 64, + "k": 12288, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 7141, + "n": 120, + "k": 12288, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 7141, + "n": 256, + "k": 12288, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 7141, + "n": 512, + "k": 12288, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 7141, + "n": 512, + "k": 12288, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 7141, + "n": 1024, + "k": 12288, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 7141, + "n": 1536, + "k": 12288, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 7141, + "n": 5120, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 7141, + "n": 12288, + "k": 1024, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 7152, + "n": 10240, + "k": 16384, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 7168, + "n": 24, + "k": 49152, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 7168, + "n": 64, + "k": 12288, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 7168, + "n": 120, + "k": 12288, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 7168, + "n": 256, + "k": 12288, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 7168, + "n": 512, + "k": 12288, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 7168, + "n": 512, + "k": 12288, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 7168, + "n": 1024, + "k": 12288, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 7168, + "n": 1536, + "k": 12288, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 7168, + "n": 5120, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 7168, + "n": 12288, + "k": 1024, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 7176, + "n": 24, + "k": 49152, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 7176, + "n": 64, + "k": 12288, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 7176, + "n": 120, + "k": 12288, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 7176, + "n": 256, + "k": 12288, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 7176, + "n": 512, + "k": 12288, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 7176, + "n": 512, + "k": 12288, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 7176, + "n": 1024, + "k": 12288, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 7176, + "n": 1536, + "k": 12288, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 7176, + "n": 5120, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 7176, + "n": 12288, + "k": 1024, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 7232, + "n": 24, + "k": 49152, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 7232, + "n": 64, + "k": 12288, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 7232, + "n": 120, + "k": 12288, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 7232, + "n": 256, + "k": 12288, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 7232, + "n": 512, + "k": 12288, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 7232, + "n": 512, + "k": 12288, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 7232, + "n": 1024, + "k": 12288, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 7232, + "n": 1536, + "k": 12288, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 7232, + "n": 5120, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 7232, + "n": 12288, + "k": 1024, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 7237, + "n": 384, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w3" + ] + }, + { + "type": "gemm", + "args": { + "m": 7237, + "n": 1024, + "k": 12288, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 7237, + "n": 6144, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3" + ] + }, + { + "type": "gemm", + "args": { + "m": 7237, + "n": 7168, + "k": 192, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w2" + ] + }, + { + "type": "gemm", + "args": { + "m": 7237, + "n": 7168, + "k": 3072, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w2" + ] + }, + { + "type": "gemm", + "args": { + "m": 7237, + "n": 7168, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.e_proj", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.h_proj", + "deepseek-ai/DeepSeek-V4-Pro/mtp.e_proj", + "deepseek-ai/DeepSeek-V4-Pro/mtp.h_proj", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.e_proj", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.h_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 7237, + "n": 12288, + "k": 512, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 7237, + "n": 14336, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 7296, + "n": 24, + "k": 49152, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 7296, + "n": 64, + "k": 12288, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 7296, + "n": 120, + "k": 12288, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 7296, + "n": 256, + "k": 12288, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 7296, + "n": 512, + "k": 12288, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 7296, + "n": 512, + "k": 12288, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 7296, + "n": 1024, + "k": 12288, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 7296, + "n": 1536, + "k": 12288, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 7296, + "n": 5120, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 7296, + "n": 12288, + "k": 1024, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 7328, + "n": 24, + "k": 49152, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 7328, + "n": 64, + "k": 12288, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 7328, + "n": 120, + "k": 12288, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 7328, + "n": 256, + "k": 12288, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 7328, + "n": 512, + "k": 12288, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 7328, + "n": 512, + "k": 12288, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 7328, + "n": 1024, + "k": 12288, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 7328, + "n": 1536, + "k": 12288, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 7328, + "n": 1792, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 7328, + "n": 5120, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 7328, + "n": 7168, + "k": 896, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 7328, + "n": 7168, + "k": 2048, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 7328, + "n": 10240, + "k": 1024, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 7328, + "n": 12288, + "k": 1024, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 7362, + "n": 6144, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3" + ] + }, + { + "type": "gemm", + "args": { + "m": 7362, + "n": 7168, + "k": 3072, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w2" + ] + }, + { + "type": "gemm", + "args": { + "m": 7362, + "n": 7168, + "k": 16384, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 7362, + "n": 65536, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 7388, + "n": 10240, + "k": 16384, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 7424, + "n": 384, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w3" + ] + }, + { + "type": "gemm", + "args": { + "m": 7424, + "n": 6144, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3" + ] + }, + { + "type": "gemm", + "args": { + "m": 7424, + "n": 7168, + "k": 192, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w2" + ] + }, + { + "type": "gemm", + "args": { + "m": 7424, + "n": 7168, + "k": 3072, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w2" + ] + }, + { + "type": "gemm", + "args": { + "m": 7424, + "n": 7168, + "k": 16384, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 7424, + "n": 10240, + "k": 16384, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 7424, + "n": 65536, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 7488, + "n": 6144, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3" + ] + }, + { + "type": "gemm", + "args": { + "m": 7488, + "n": 7168, + "k": 3072, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w2" + ] + }, + { + "type": "gemm", + "args": { + "m": 7488, + "n": 7168, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.e_proj", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.h_proj", + "deepseek-ai/DeepSeek-V4-Pro/mtp.e_proj", + "deepseek-ai/DeepSeek-V4-Pro/mtp.h_proj", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.e_proj", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.h_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 7488, + "n": 7168, + "k": 16384, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 7488, + "n": 10240, + "k": 16384, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 7488, + "n": 12288, + "k": 8192, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 7488, + "n": 12288, + "k": 16384, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 7488, + "n": 14336, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 7488, + "n": 16384, + "k": 12288, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 7488, + "n": 65536, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 7488, + "n": 81920, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 7496, + "n": 12288, + "k": 16384, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 7496, + "n": 81920, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 7520, + "n": 768, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w3" + ] + }, + { + "type": "gemm", + "args": { + "m": 7520, + "n": 7168, + "k": 384, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w2" + ] + }, + { + "type": "gemm", + "args": { + "m": 7530, + "n": 1024, + "k": 10240, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 7530, + "n": 10240, + "k": 512, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 7552, + "n": 1024, + "k": 10240, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 7552, + "n": 6144, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3" + ] + }, + { + "type": "gemm", + "args": { + "m": 7552, + "n": 7168, + "k": 3072, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w2" + ] + }, + { + "type": "gemm", + "args": { + "m": 7552, + "n": 7168, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.e_proj", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.h_proj", + "deepseek-ai/DeepSeek-V4-Pro/mtp.e_proj", + "deepseek-ai/DeepSeek-V4-Pro/mtp.h_proj", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.e_proj", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.h_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 7552, + "n": 7168, + "k": 16384, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 7552, + "n": 10240, + "k": 512, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 7552, + "n": 10240, + "k": 16384, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 7552, + "n": 12288, + "k": 16384, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 7552, + "n": 14336, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 7552, + "n": 65536, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 7552, + "n": 81920, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 7555, + "n": 10240, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 7555, + "n": 12288, + "k": 2048, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 7568, + "n": 24, + "k": 49152, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 7568, + "n": 64, + "k": 12288, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 7568, + "n": 120, + "k": 12288, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 7568, + "n": 256, + "k": 12288, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 7568, + "n": 512, + "k": 12288, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 7568, + "n": 512, + "k": 12288, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 7568, + "n": 1024, + "k": 12288, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 7568, + "n": 1536, + "k": 12288, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 7568, + "n": 5120, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 7568, + "n": 12288, + "k": 1024, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 7602, + "n": 4096, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 7616, + "n": 1024, + "k": 10240, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 7616, + "n": 4096, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 7616, + "n": 6144, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3" + ] + }, + { + "type": "gemm", + "args": { + "m": 7616, + "n": 7168, + "k": 3072, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w2" + ] + }, + { + "type": "gemm", + "args": { + "m": 7616, + "n": 10240, + "k": 512, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 7624, + "n": 4096, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 7629, + "n": 7168, + "k": 1024, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 7632, + "n": 1024, + "k": 10240, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 7632, + "n": 4096, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 7632, + "n": 7168, + "k": 1024, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 7632, + "n": 10240, + "k": 512, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 7640, + "n": 7168, + "k": 1024, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 7680, + "n": 320, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 7680, + "n": 1024, + "k": 10240, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 7680, + "n": 4096, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 7680, + "n": 6144, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3" + ] + }, + { + "type": "gemm", + "args": { + "m": 7680, + "n": 7168, + "k": 1024, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 7680, + "n": 7168, + "k": 3072, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w2" + ] + }, + { + "type": "gemm", + "args": { + "m": 7680, + "n": 7168, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.e_proj", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.h_proj", + "deepseek-ai/DeepSeek-V4-Pro/mtp.e_proj", + "deepseek-ai/DeepSeek-V4-Pro/mtp.h_proj", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.e_proj", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.h_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 7680, + "n": 10240, + "k": 512, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 7680, + "n": 10240, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 7680, + "n": 12288, + "k": 2048, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 7680, + "n": 12288, + "k": 8192, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 7680, + "n": 12288, + "k": 16384, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 7680, + "n": 14336, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 7680, + "n": 16384, + "k": 12288, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 7680, + "n": 81920, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 7681, + "n": 320, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 7682, + "n": 24, + "k": 40960, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 7682, + "n": 64, + "k": 10240, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 7682, + "n": 192, + "k": 10240, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 7682, + "n": 256, + "k": 10240, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 7682, + "n": 512, + "k": 10240, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 7682, + "n": 512, + "k": 10240, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 7682, + "n": 1024, + "k": 10240, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 7682, + "n": 1536, + "k": 10240, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 7683, + "n": 10240, + "k": 2048, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 7688, + "n": 24, + "k": 40960, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 7688, + "n": 64, + "k": 10240, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 7688, + "n": 192, + "k": 10240, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 7688, + "n": 256, + "k": 10240, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 7688, + "n": 320, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 7688, + "n": 512, + "k": 10240, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 7688, + "n": 512, + "k": 10240, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 7688, + "n": 1024, + "k": 10240, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 7688, + "n": 1536, + "k": 10240, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 7688, + "n": 10240, + "k": 2048, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 7720, + "n": 384, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w3" + ] + }, + { + "type": "gemm", + "args": { + "m": 7720, + "n": 7168, + "k": 192, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w2" + ] + }, + { + "type": "gemm", + "args": { + "m": 7744, + "n": 24, + "k": 40960, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 7744, + "n": 64, + "k": 10240, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 7744, + "n": 192, + "k": 10240, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 7744, + "n": 256, + "k": 10240, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 7744, + "n": 320, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 7744, + "n": 512, + "k": 10240, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 7744, + "n": 512, + "k": 10240, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 7744, + "n": 1024, + "k": 10240, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 7744, + "n": 1536, + "k": 10240, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 7744, + "n": 7168, + "k": 1024, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 7744, + "n": 10240, + "k": 2048, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 7752, + "n": 896, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 7752, + "n": 7168, + "k": 448, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 7756, + "n": 10240, + "k": 1024, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 7769, + "n": 896, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 7769, + "n": 7168, + "k": 448, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 7769, + "n": 7168, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.e_proj", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.h_proj", + "deepseek-ai/DeepSeek-V4-Pro/mtp.e_proj", + "deepseek-ai/DeepSeek-V4-Pro/mtp.h_proj", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.e_proj", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.h_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 7769, + "n": 8192, + "k": 10240, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 7769, + "n": 10240, + "k": 4096, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 7769, + "n": 12288, + "k": 8192, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 7769, + "n": 12288, + "k": 16384, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 7769, + "n": 14336, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 7769, + "n": 16384, + "k": 12288, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 7769, + "n": 81920, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 7792, + "n": 24, + "k": 40960, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 7792, + "n": 64, + "k": 10240, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 7792, + "n": 192, + "k": 10240, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 7792, + "n": 256, + "k": 10240, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 7792, + "n": 512, + "k": 10240, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 7792, + "n": 512, + "k": 10240, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 7792, + "n": 1024, + "k": 10240, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 7792, + "n": 1536, + "k": 10240, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 7792, + "n": 10240, + "k": 1024, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 7808, + "n": 24, + "k": 40960, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 7808, + "n": 64, + "k": 10240, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 7808, + "n": 192, + "k": 10240, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 7808, + "n": 256, + "k": 10240, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 7808, + "n": 320, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 7808, + "n": 512, + "k": 10240, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 7808, + "n": 512, + "k": 10240, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 7808, + "n": 768, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w3" + ] + }, + { + "type": "gemm", + "args": { + "m": 7808, + "n": 1024, + "k": 10240, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 7808, + "n": 1536, + "k": 10240, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 7808, + "n": 4096, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 7808, + "n": 7168, + "k": 384, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w2" + ] + }, + { + "type": "gemm", + "args": { + "m": 7808, + "n": 7168, + "k": 1024, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 7808, + "n": 10240, + "k": 1024, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 7808, + "n": 10240, + "k": 2048, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 7832, + "n": 10240, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 7832, + "n": 12288, + "k": 2048, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 7840, + "n": 320, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 7904, + "n": 896, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 7904, + "n": 4096, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 7904, + "n": 7168, + "k": 448, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 7904, + "n": 7168, + "k": 1024, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 7936, + "n": 24, + "k": 40960, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 7936, + "n": 64, + "k": 10240, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 7936, + "n": 192, + "k": 10240, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 7936, + "n": 256, + "k": 10240, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 7936, + "n": 512, + "k": 10240, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 7936, + "n": 512, + "k": 10240, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 7936, + "n": 768, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w3" + ] + }, + { + "type": "gemm", + "args": { + "m": 7936, + "n": 1024, + "k": 10240, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 7936, + "n": 1536, + "k": 10240, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 7936, + "n": 7168, + "k": 384, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w2" + ] + }, + { + "type": "gemm", + "args": { + "m": 7936, + "n": 8192, + "k": 10240, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 7936, + "n": 10240, + "k": 1024, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 7936, + "n": 10240, + "k": 2048, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 7936, + "n": 10240, + "k": 4096, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 8000, + "n": 2048, + "k": 12288, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 8000, + "n": 10240, + "k": 1024, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 8000, + "n": 10240, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 8000, + "n": 12288, + "k": 1024, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 8000, + "n": 12288, + "k": 2048, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 8064, + "n": 2048, + "k": 12288, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 8064, + "n": 10240, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 8064, + "n": 12288, + "k": 1024, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 8064, + "n": 12288, + "k": 2048, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 8064, + "n": 12288, + "k": 8192, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 8064, + "n": 16384, + "k": 12288, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 8080, + "n": 320, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 8080, + "n": 896, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 8080, + "n": 7168, + "k": 448, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 8080, + "n": 10240, + "k": 2048, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 8088, + "n": 1024, + "k": 10240, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 8088, + "n": 10240, + "k": 512, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 8096, + "n": 24, + "k": 40960, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 8096, + "n": 64, + "k": 10240, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 8096, + "n": 192, + "k": 10240, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 8096, + "n": 256, + "k": 10240, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 8096, + "n": 512, + "k": 10240, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 8096, + "n": 512, + "k": 10240, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 8096, + "n": 1024, + "k": 10240, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 8096, + "n": 1536, + "k": 10240, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 8096, + "n": 10240, + "k": 2048, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 8177, + "n": 6144, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3" + ] + }, + { + "type": "gemm", + "args": { + "m": 8177, + "n": 7168, + "k": 3072, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w2" + ] + }, + { + "type": "gemm", + "args": { + "m": 8192, + "n": 384, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w3" + ] + }, + { + "type": "gemm", + "args": { + "m": 8192, + "n": 768, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w3" + ] + }, + { + "type": "gemm", + "args": { + "m": 8192, + "n": 896, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 8192, + "n": 7168, + "k": 192, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w2" + ] + }, + { + "type": "gemm", + "args": { + "m": 8192, + "n": 7168, + "k": 384, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w2" + ] + }, + { + "type": "gemm", + "args": { + "m": 8192, + "n": 7168, + "k": 448, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 8192, + "n": 7168, + "k": 2048, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 8320, + "n": 24, + "k": 28672, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/hc_attn", + "deepseek-ai/DeepSeek-V4-Pro-0813/hc_ffn", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.hc_attn", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.hc_ffn", + "deepseek-ai/DeepSeek-V4-Pro/hc_attn", + "deepseek-ai/DeepSeek-V4-Pro/hc_ffn", + "deepseek-ai/DeepSeek-V4-Pro/mtp.hc_attn", + "deepseek-ai/DeepSeek-V4-Pro/mtp.hc_ffn", + "sgl-project/DeepSeek-V4-Pro-FP8/hc_attn", + "sgl-project/DeepSeek-V4-Pro-FP8/hc_ffn", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.hc_attn", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.hc_ffn" + ] + }, + { + "type": "gemm", + "args": { + "m": 8320, + "n": 64, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.weights_proj", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.weights_proj", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.weights_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 8320, + "n": 256, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.compressor.wkv", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.compressor.wkv", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.compressor.wgate", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.compressor.wkv" + ] + }, + { + "type": "gemm", + "args": { + "m": 8320, + "n": 384, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.gate", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.gate", + "deepseek-ai/DeepSeek-V4-Pro/ffn.gate", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.gate", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.gate", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.gate" + ] + }, + { + "type": "gemm", + "args": { + "m": 8320, + "n": 512, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wkv", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wkv", + "deepseek-ai/DeepSeek-V4-Pro/attn.wkv", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wkv", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wkv", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wkv" + ] + }, + { + "type": "gemm", + "args": { + "m": 8320, + "n": 512, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.compressor.wkv+wgate", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.compressor.wkv+wgate", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.compressor.wkv+wgate" + ] + }, + { + "type": "gemm", + "args": { + "m": 8320, + "n": 1024, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.compressor.wkv", + "deepseek-ai/DeepSeek-V4-Pro/attn.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro/attn.compressor.wkv", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.compressor.wgate", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.compressor.wkv" + ] + }, + { + "type": "gemm", + "args": { + "m": 8320, + "n": 1536, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w3" + ] + }, + { + "type": "gemm", + "args": { + "m": 8320, + "n": 7168, + "k": 2048, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 8320, + "n": 8192, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 8322, + "n": 24, + "k": 28672, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/hc_attn", + "deepseek-ai/DeepSeek-V4-Pro-0813/hc_ffn", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.hc_attn", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.hc_ffn", + "deepseek-ai/DeepSeek-V4-Pro/hc_attn", + "deepseek-ai/DeepSeek-V4-Pro/hc_ffn", + "deepseek-ai/DeepSeek-V4-Pro/mtp.hc_attn", + "deepseek-ai/DeepSeek-V4-Pro/mtp.hc_ffn", + "sgl-project/DeepSeek-V4-Pro-FP8/hc_attn", + "sgl-project/DeepSeek-V4-Pro-FP8/hc_ffn", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.hc_attn", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.hc_ffn" + ] + }, + { + "type": "gemm", + "args": { + "m": 8322, + "n": 64, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.weights_proj", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.weights_proj", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.weights_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 8322, + "n": 256, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.compressor.wkv", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.compressor.wkv", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.compressor.wgate", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.compressor.wkv" + ] + }, + { + "type": "gemm", + "args": { + "m": 8322, + "n": 512, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wkv", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wkv", + "deepseek-ai/DeepSeek-V4-Pro/attn.wkv", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wkv", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wkv", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wkv" + ] + }, + { + "type": "gemm", + "args": { + "m": 8322, + "n": 512, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.compressor.wkv+wgate", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.compressor.wkv+wgate", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.compressor.wkv+wgate" + ] + }, + { + "type": "gemm", + "args": { + "m": 8322, + "n": 1024, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.compressor.wkv", + "deepseek-ai/DeepSeek-V4-Pro/attn.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro/attn.compressor.wkv", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.compressor.wgate", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.compressor.wkv" + ] + }, + { + "type": "gemm", + "args": { + "m": 8322, + "n": 1536, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w3" + ] + }, + { + "type": "gemm", + "args": { + "m": 8322, + "n": 8192, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 8384, + "n": 24, + "k": 28672, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/hc_attn", + "deepseek-ai/DeepSeek-V4-Pro-0813/hc_ffn", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.hc_attn", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.hc_ffn", + "deepseek-ai/DeepSeek-V4-Pro/hc_attn", + "deepseek-ai/DeepSeek-V4-Pro/hc_ffn", + "deepseek-ai/DeepSeek-V4-Pro/mtp.hc_attn", + "deepseek-ai/DeepSeek-V4-Pro/mtp.hc_ffn", + "sgl-project/DeepSeek-V4-Pro-FP8/hc_attn", + "sgl-project/DeepSeek-V4-Pro-FP8/hc_ffn", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.hc_attn", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.hc_ffn" + ] + }, + { + "type": "gemm", + "args": { + "m": 8384, + "n": 64, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.weights_proj", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.weights_proj", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.weights_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 8384, + "n": 256, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.compressor.wkv", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.compressor.wkv", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.compressor.wgate", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.compressor.wkv" + ] + }, + { + "type": "gemm", + "args": { + "m": 8384, + "n": 384, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.gate", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.gate", + "deepseek-ai/DeepSeek-V4-Pro/ffn.gate", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.gate", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.gate", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.gate" + ] + }, + { + "type": "gemm", + "args": { + "m": 8384, + "n": 512, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wkv", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wkv", + "deepseek-ai/DeepSeek-V4-Pro/attn.wkv", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wkv", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wkv", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wkv" + ] + }, + { + "type": "gemm", + "args": { + "m": 8384, + "n": 512, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.compressor.wkv+wgate", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.compressor.wkv+wgate", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.compressor.wkv+wgate" + ] + }, + { + "type": "gemm", + "args": { + "m": 8384, + "n": 1024, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.compressor.wkv", + "deepseek-ai/DeepSeek-V4-Pro/attn.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro/attn.compressor.wkv", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.compressor.wgate", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.compressor.wkv" + ] + }, + { + "type": "gemm", + "args": { + "m": 8384, + "n": 1536, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w3" + ] + }, + { + "type": "gemm", + "args": { + "m": 8384, + "n": 7168, + "k": 768, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w2" + ] + }, + { + "type": "gemm", + "args": { + "m": 8384, + "n": 7168, + "k": 2048, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 8384, + "n": 7168, + "k": 4096, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 8384, + "n": 8192, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 8384, + "n": 16384, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 8448, + "n": 24, + "k": 28672, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/hc_attn", + "deepseek-ai/DeepSeek-V4-Pro-0813/hc_ffn", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.hc_attn", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.hc_ffn", + "deepseek-ai/DeepSeek-V4-Pro/hc_attn", + "deepseek-ai/DeepSeek-V4-Pro/hc_ffn", + "deepseek-ai/DeepSeek-V4-Pro/mtp.hc_attn", + "deepseek-ai/DeepSeek-V4-Pro/mtp.hc_ffn", + "sgl-project/DeepSeek-V4-Pro-FP8/hc_attn", + "sgl-project/DeepSeek-V4-Pro-FP8/hc_ffn", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.hc_attn", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.hc_ffn" + ] + }, + { + "type": "gemm", + "args": { + "m": 8448, + "n": 64, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.weights_proj", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.weights_proj", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.weights_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 8448, + "n": 256, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.compressor.wkv", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.compressor.wkv", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.compressor.wgate", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.compressor.wkv" + ] + }, + { + "type": "gemm", + "args": { + "m": 8448, + "n": 384, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.gate", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.gate", + "deepseek-ai/DeepSeek-V4-Pro/ffn.gate", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.gate", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.gate", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.gate" + ] + }, + { + "type": "gemm", + "args": { + "m": 8448, + "n": 384, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w3" + ] + }, + { + "type": "gemm", + "args": { + "m": 8448, + "n": 512, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wkv", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wkv", + "deepseek-ai/DeepSeek-V4-Pro/attn.wkv", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wkv", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wkv", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wkv" + ] + }, + { + "type": "gemm", + "args": { + "m": 8448, + "n": 512, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.compressor.wkv+wgate", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.compressor.wkv+wgate", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.compressor.wkv+wgate" + ] + }, + { + "type": "gemm", + "args": { + "m": 8448, + "n": 1024, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.compressor.wkv", + "deepseek-ai/DeepSeek-V4-Pro/attn.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro/attn.compressor.wkv", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.compressor.wgate", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.compressor.wkv" + ] + }, + { + "type": "gemm", + "args": { + "m": 8448, + "n": 1536, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w3" + ] + }, + { + "type": "gemm", + "args": { + "m": 8448, + "n": 7168, + "k": 192, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w2" + ] + }, + { + "type": "gemm", + "args": { + "m": 8448, + "n": 8192, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 8512, + "n": 1792, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 8512, + "n": 7168, + "k": 896, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 8576, + "n": 384, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w3" + ] + }, + { + "type": "gemm", + "args": { + "m": 8576, + "n": 1536, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w3" + ] + }, + { + "type": "gemm", + "args": { + "m": 8576, + "n": 1792, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 8576, + "n": 7168, + "k": 192, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w2" + ] + }, + { + "type": "gemm", + "args": { + "m": 8576, + "n": 7168, + "k": 768, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w2" + ] + }, + { + "type": "gemm", + "args": { + "m": 8576, + "n": 7168, + "k": 896, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 8576, + "n": 7168, + "k": 4096, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 8576, + "n": 16384, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 8640, + "n": 384, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w3" + ] + }, + { + "type": "gemm", + "args": { + "m": 8640, + "n": 7168, + "k": 192, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w2" + ] + }, + { + "type": "gemm", + "args": { + "m": 8704, + "n": 1792, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 8704, + "n": 7168, + "k": 896, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 8705, + "n": 384, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.gate", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.gate", + "deepseek-ai/DeepSeek-V4-Pro/ffn.gate", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.gate", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.gate", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.gate" + ] + }, + { + "type": "gemm", + "args": { + "m": 8705, + "n": 384, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w3" + ] + }, + { + "type": "gemm", + "args": { + "m": 8705, + "n": 7168, + "k": 192, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w2" + ] + }, + { + "type": "gemm", + "args": { + "m": 8772, + "n": 2048, + "k": 12288, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 8772, + "n": 12288, + "k": 1024, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 8835, + "n": 1792, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 8835, + "n": 7168, + "k": 896, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 8835, + "n": 7168, + "k": 2048, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 8960, + "n": 24, + "k": 49152, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 8960, + "n": 64, + "k": 12288, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 8960, + "n": 120, + "k": 12288, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 8960, + "n": 256, + "k": 12288, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 8960, + "n": 512, + "k": 12288, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 8960, + "n": 512, + "k": 12288, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 8960, + "n": 1024, + "k": 12288, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 8960, + "n": 1536, + "k": 12288, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 8960, + "n": 5120, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 8960, + "n": 12288, + "k": 1024, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 8961, + "n": 1536, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w3" + ] + }, + { + "type": "gemm", + "args": { + "m": 8961, + "n": 7168, + "k": 768, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w2" + ] + }, + { + "type": "gemm", + "args": { + "m": 8961, + "n": 7168, + "k": 4096, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 8961, + "n": 16384, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 9024, + "n": 24, + "k": 49152, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 9024, + "n": 64, + "k": 12288, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 9024, + "n": 120, + "k": 12288, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 9024, + "n": 256, + "k": 12288, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 9024, + "n": 512, + "k": 12288, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 9024, + "n": 512, + "k": 12288, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 9024, + "n": 1024, + "k": 12288, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 9024, + "n": 1536, + "k": 12288, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 9024, + "n": 5120, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 9024, + "n": 12288, + "k": 1024, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 9025, + "n": 24, + "k": 49152, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 9025, + "n": 64, + "k": 12288, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 9025, + "n": 120, + "k": 12288, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 9025, + "n": 256, + "k": 12288, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 9025, + "n": 512, + "k": 12288, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 9025, + "n": 512, + "k": 12288, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 9025, + "n": 1024, + "k": 12288, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 9025, + "n": 1536, + "k": 12288, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 9025, + "n": 5120, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 9025, + "n": 12288, + "k": 1024, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 9088, + "n": 24, + "k": 49152, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 9088, + "n": 64, + "k": 12288, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 9088, + "n": 120, + "k": 12288, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 9088, + "n": 256, + "k": 12288, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 9088, + "n": 512, + "k": 12288, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 9088, + "n": 512, + "k": 12288, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 9088, + "n": 1024, + "k": 12288, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 9088, + "n": 1536, + "k": 12288, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 9088, + "n": 5120, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 9088, + "n": 12288, + "k": 1024, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 9216, + "n": 1536, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w3" + ] + }, + { + "type": "gemm", + "args": { + "m": 9216, + "n": 2048, + "k": 12288, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 9216, + "n": 6144, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3" + ] + }, + { + "type": "gemm", + "args": { + "m": 9216, + "n": 7168, + "k": 768, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w2" + ] + }, + { + "type": "gemm", + "args": { + "m": 9216, + "n": 7168, + "k": 3072, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w2" + ] + }, + { + "type": "gemm", + "args": { + "m": 9216, + "n": 7168, + "k": 4096, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 9216, + "n": 7168, + "k": 16384, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 9216, + "n": 12288, + "k": 1024, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 9216, + "n": 16384, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 9216, + "n": 65536, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 9280, + "n": 6144, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3" + ] + }, + { + "type": "gemm", + "args": { + "m": 9280, + "n": 7168, + "k": 3072, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w2" + ] + }, + { + "type": "gemm", + "args": { + "m": 9280, + "n": 7168, + "k": 16384, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 9280, + "n": 10240, + "k": 16384, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 9280, + "n": 65536, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 9344, + "n": 6144, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3" + ] + }, + { + "type": "gemm", + "args": { + "m": 9344, + "n": 7168, + "k": 3072, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w2" + ] + }, + { + "type": "gemm", + "args": { + "m": 9344, + "n": 7168, + "k": 16384, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 9344, + "n": 10240, + "k": 16384, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 9344, + "n": 65536, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 9345, + "n": 6144, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3" + ] + }, + { + "type": "gemm", + "args": { + "m": 9345, + "n": 7168, + "k": 3072, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w2" + ] + }, + { + "type": "gemm", + "args": { + "m": 9345, + "n": 7168, + "k": 16384, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 9345, + "n": 65536, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 9408, + "n": 7168, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.e_proj", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.h_proj", + "deepseek-ai/DeepSeek-V4-Pro/mtp.e_proj", + "deepseek-ai/DeepSeek-V4-Pro/mtp.h_proj", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.e_proj", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.h_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 9408, + "n": 12288, + "k": 8192, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 9408, + "n": 12288, + "k": 16384, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 9408, + "n": 14336, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 9408, + "n": 16384, + "k": 12288, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 9408, + "n": 81920, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 9446, + "n": 12288, + "k": 16384, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 9446, + "n": 81920, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 9472, + "n": 7168, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.e_proj", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.h_proj", + "deepseek-ai/DeepSeek-V4-Pro/mtp.e_proj", + "deepseek-ai/DeepSeek-V4-Pro/mtp.h_proj", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.e_proj", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.h_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 9472, + "n": 10240, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 9472, + "n": 10240, + "k": 16384, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 9472, + "n": 12288, + "k": 2048, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 9472, + "n": 12288, + "k": 8192, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 9472, + "n": 12288, + "k": 16384, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 9472, + "n": 14336, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 9472, + "n": 16384, + "k": 12288, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 9472, + "n": 81920, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 9536, + "n": 4096, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 9536, + "n": 10240, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 9536, + "n": 12288, + "k": 2048, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 9600, + "n": 4096, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 9600, + "n": 7168, + "k": 1024, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 9600, + "n": 7168, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.e_proj", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.h_proj", + "deepseek-ai/DeepSeek-V4-Pro/mtp.e_proj", + "deepseek-ai/DeepSeek-V4-Pro/mtp.h_proj", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.e_proj", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.h_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 9600, + "n": 10240, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 9600, + "n": 12288, + "k": 2048, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 9600, + "n": 12288, + "k": 8192, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 9600, + "n": 12288, + "k": 16384, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 9600, + "n": 14336, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 9600, + "n": 16384, + "k": 12288, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 9600, + "n": 81920, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 9664, + "n": 24, + "k": 40960, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 9664, + "n": 64, + "k": 10240, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 9664, + "n": 192, + "k": 10240, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 9664, + "n": 256, + "k": 10240, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 9664, + "n": 320, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 9664, + "n": 512, + "k": 10240, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 9664, + "n": 512, + "k": 10240, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 9664, + "n": 1024, + "k": 10240, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 9664, + "n": 1536, + "k": 10240, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 9664, + "n": 7168, + "k": 1024, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 9728, + "n": 24, + "k": 40960, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 9728, + "n": 64, + "k": 10240, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 9728, + "n": 192, + "k": 10240, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 9728, + "n": 256, + "k": 10240, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 9728, + "n": 320, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 9728, + "n": 512, + "k": 10240, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 9728, + "n": 512, + "k": 10240, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 9728, + "n": 1024, + "k": 10240, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 9728, + "n": 1536, + "k": 10240, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 9728, + "n": 4096, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 9728, + "n": 7168, + "k": 1024, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 9728, + "n": 10240, + "k": 2048, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 9792, + "n": 10240, + "k": 2048, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 9856, + "n": 24, + "k": 40960, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 9856, + "n": 64, + "k": 10240, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 9856, + "n": 192, + "k": 10240, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 9856, + "n": 256, + "k": 10240, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 9856, + "n": 320, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 9856, + "n": 512, + "k": 10240, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 9856, + "n": 512, + "k": 10240, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 9856, + "n": 1024, + "k": 10240, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 9856, + "n": 1536, + "k": 10240, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 9864, + "n": 24, + "k": 28672, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/hc_attn", + "deepseek-ai/DeepSeek-V4-Pro-0813/hc_ffn", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.hc_attn", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.hc_ffn", + "deepseek-ai/DeepSeek-V4-Pro/hc_attn", + "deepseek-ai/DeepSeek-V4-Pro/hc_ffn", + "deepseek-ai/DeepSeek-V4-Pro/mtp.hc_attn", + "deepseek-ai/DeepSeek-V4-Pro/mtp.hc_ffn", + "sgl-project/DeepSeek-V4-Pro-FP8/hc_attn", + "sgl-project/DeepSeek-V4-Pro-FP8/hc_ffn", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.hc_attn", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.hc_ffn" + ] + }, + { + "type": "gemm", + "args": { + "m": 9864, + "n": 64, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.weights_proj", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.weights_proj", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.weights_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 9864, + "n": 256, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.compressor.wkv", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.compressor.wkv", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.compressor.wgate", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.compressor.wkv" + ] + }, + { + "type": "gemm", + "args": { + "m": 9864, + "n": 384, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.gate", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.gate", + "deepseek-ai/DeepSeek-V4-Pro/ffn.gate", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.gate", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.gate", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.gate" + ] + }, + { + "type": "gemm", + "args": { + "m": 9864, + "n": 512, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wkv", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wkv", + "deepseek-ai/DeepSeek-V4-Pro/attn.wkv", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wkv", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wkv", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wkv" + ] + }, + { + "type": "gemm", + "args": { + "m": 9864, + "n": 512, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.compressor.wkv+wgate", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.compressor.wkv+wgate", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.compressor.wkv+wgate" + ] + }, + { + "type": "gemm", + "args": { + "m": 9864, + "n": 1024, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.compressor.wkv", + "deepseek-ai/DeepSeek-V4-Pro/attn.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro/attn.compressor.wkv", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.compressor.wgate", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.compressor.wkv" + ] + }, + { + "type": "gemm", + "args": { + "m": 9864, + "n": 1536, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w3" + ] + }, + { + "type": "gemm", + "args": { + "m": 9864, + "n": 7168, + "k": 768, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w2" + ] + }, + { + "type": "gemm", + "args": { + "m": 9864, + "n": 7168, + "k": 4096, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 9864, + "n": 8192, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 9864, + "n": 16384, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 10048, + "n": 2048, + "k": 12288, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 10048, + "n": 12288, + "k": 1024, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 10112, + "n": 10240, + "k": 2048, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 10128, + "n": 24, + "k": 28672, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/hc_attn", + "deepseek-ai/DeepSeek-V4-Pro-0813/hc_ffn", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.hc_attn", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.hc_ffn", + "deepseek-ai/DeepSeek-V4-Pro/hc_attn", + "deepseek-ai/DeepSeek-V4-Pro/hc_ffn", + "deepseek-ai/DeepSeek-V4-Pro/mtp.hc_attn", + "deepseek-ai/DeepSeek-V4-Pro/mtp.hc_ffn", + "sgl-project/DeepSeek-V4-Pro-FP8/hc_attn", + "sgl-project/DeepSeek-V4-Pro-FP8/hc_ffn", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.hc_attn", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.hc_ffn" + ] + }, + { + "type": "gemm", + "args": { + "m": 10128, + "n": 64, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.weights_proj", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.weights_proj", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.weights_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 10128, + "n": 256, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.compressor.wkv", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.compressor.wkv", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.compressor.wgate", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.compressor.wkv" + ] + }, + { + "type": "gemm", + "args": { + "m": 10128, + "n": 320, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 10128, + "n": 512, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wkv", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wkv", + "deepseek-ai/DeepSeek-V4-Pro/attn.wkv", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wkv", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wkv", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wkv" + ] + }, + { + "type": "gemm", + "args": { + "m": 10128, + "n": 512, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.compressor.wkv+wgate", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.compressor.wkv+wgate", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.compressor.wkv+wgate" + ] + }, + { + "type": "gemm", + "args": { + "m": 10128, + "n": 1024, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.compressor.wkv", + "deepseek-ai/DeepSeek-V4-Pro/attn.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro/attn.compressor.wkv", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.compressor.wgate", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.compressor.wkv" + ] + }, + { + "type": "gemm", + "args": { + "m": 10128, + "n": 1536, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w3" + ] + }, + { + "type": "gemm", + "args": { + "m": 10128, + "n": 7168, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.e_proj", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.h_proj", + "deepseek-ai/DeepSeek-V4-Pro/mtp.e_proj", + "deepseek-ai/DeepSeek-V4-Pro/mtp.h_proj", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.e_proj", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.h_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 10128, + "n": 7168, + "k": 16384, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 10128, + "n": 8192, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 10128, + "n": 14336, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 10128, + "n": 65536, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 10236, + "n": 24, + "k": 40960, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 10236, + "n": 64, + "k": 10240, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 10236, + "n": 192, + "k": 10240, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 10236, + "n": 256, + "k": 10240, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 10236, + "n": 512, + "k": 10240, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 10236, + "n": 512, + "k": 10240, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 10236, + "n": 1024, + "k": 10240, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 10236, + "n": 1536, + "k": 10240, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 10236, + "n": 10240, + "k": 16384, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 10304, + "n": 7168, + "k": 2048, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 10368, + "n": 7168, + "k": 2048, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 10432, + "n": 24, + "k": 28672, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/hc_attn", + "deepseek-ai/DeepSeek-V4-Pro-0813/hc_ffn", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.hc_attn", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.hc_ffn", + "deepseek-ai/DeepSeek-V4-Pro/hc_attn", + "deepseek-ai/DeepSeek-V4-Pro/hc_ffn", + "deepseek-ai/DeepSeek-V4-Pro/mtp.hc_attn", + "deepseek-ai/DeepSeek-V4-Pro/mtp.hc_ffn", + "sgl-project/DeepSeek-V4-Pro-FP8/hc_attn", + "sgl-project/DeepSeek-V4-Pro-FP8/hc_ffn", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.hc_attn", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.hc_ffn" + ] + }, + { + "type": "gemm", + "args": { + "m": 10432, + "n": 64, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.weights_proj", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.weights_proj", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.weights_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 10432, + "n": 256, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.compressor.wkv", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.compressor.wkv", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.compressor.wgate", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.compressor.wkv" + ] + }, + { + "type": "gemm", + "args": { + "m": 10432, + "n": 384, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.gate", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.gate", + "deepseek-ai/DeepSeek-V4-Pro/ffn.gate", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.gate", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.gate", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.gate" + ] + }, + { + "type": "gemm", + "args": { + "m": 10432, + "n": 512, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wkv", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wkv", + "deepseek-ai/DeepSeek-V4-Pro/attn.wkv", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wkv", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wkv", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wkv" + ] + }, + { + "type": "gemm", + "args": { + "m": 10432, + "n": 512, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.compressor.wkv+wgate", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.compressor.wkv+wgate", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.compressor.wkv+wgate" + ] + }, + { + "type": "gemm", + "args": { + "m": 10432, + "n": 1024, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.compressor.wkv", + "deepseek-ai/DeepSeek-V4-Pro/attn.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro/attn.compressor.wkv", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.compressor.wgate", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.compressor.wkv" + ] + }, + { + "type": "gemm", + "args": { + "m": 10432, + "n": 1536, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w3" + ] + }, + { + "type": "gemm", + "args": { + "m": 10432, + "n": 8192, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 10434, + "n": 24, + "k": 28672, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/hc_attn", + "deepseek-ai/DeepSeek-V4-Pro-0813/hc_ffn", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.hc_attn", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.hc_ffn", + "deepseek-ai/DeepSeek-V4-Pro/hc_attn", + "deepseek-ai/DeepSeek-V4-Pro/hc_ffn", + "deepseek-ai/DeepSeek-V4-Pro/mtp.hc_attn", + "deepseek-ai/DeepSeek-V4-Pro/mtp.hc_ffn", + "sgl-project/DeepSeek-V4-Pro-FP8/hc_attn", + "sgl-project/DeepSeek-V4-Pro-FP8/hc_ffn", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.hc_attn", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.hc_ffn" + ] + }, + { + "type": "gemm", + "args": { + "m": 10434, + "n": 64, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.weights_proj", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.weights_proj", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.weights_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 10434, + "n": 256, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.compressor.wkv", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.compressor.wkv", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.compressor.wgate", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.compressor.wkv" + ] + }, + { + "type": "gemm", + "args": { + "m": 10434, + "n": 384, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.gate", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.gate", + "deepseek-ai/DeepSeek-V4-Pro/ffn.gate", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.gate", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.gate", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.gate" + ] + }, + { + "type": "gemm", + "args": { + "m": 10434, + "n": 512, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wkv", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wkv", + "deepseek-ai/DeepSeek-V4-Pro/attn.wkv", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wkv", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wkv", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wkv" + ] + }, + { + "type": "gemm", + "args": { + "m": 10434, + "n": 512, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.compressor.wkv+wgate", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.compressor.wkv+wgate", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.compressor.wkv+wgate" + ] + }, + { + "type": "gemm", + "args": { + "m": 10434, + "n": 1024, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.compressor.wkv", + "deepseek-ai/DeepSeek-V4-Pro/attn.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro/attn.compressor.wkv", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.compressor.wgate", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.compressor.wkv" + ] + }, + { + "type": "gemm", + "args": { + "m": 10434, + "n": 1536, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w3" + ] + }, + { + "type": "gemm", + "args": { + "m": 10434, + "n": 8192, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 10496, + "n": 24, + "k": 28672, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/hc_attn", + "deepseek-ai/DeepSeek-V4-Pro-0813/hc_ffn", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.hc_attn", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.hc_ffn", + "deepseek-ai/DeepSeek-V4-Pro/hc_attn", + "deepseek-ai/DeepSeek-V4-Pro/hc_ffn", + "deepseek-ai/DeepSeek-V4-Pro/mtp.hc_attn", + "deepseek-ai/DeepSeek-V4-Pro/mtp.hc_ffn", + "sgl-project/DeepSeek-V4-Pro-FP8/hc_attn", + "sgl-project/DeepSeek-V4-Pro-FP8/hc_ffn", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.hc_attn", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.hc_ffn" + ] + }, + { + "type": "gemm", + "args": { + "m": 10496, + "n": 64, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.weights_proj", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.weights_proj", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.weights_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 10496, + "n": 256, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.compressor.wkv", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.compressor.wkv", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.compressor.wgate", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.compressor.wkv" + ] + }, + { + "type": "gemm", + "args": { + "m": 10496, + "n": 384, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.gate", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.gate", + "deepseek-ai/DeepSeek-V4-Pro/ffn.gate", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.gate", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.gate", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.gate" + ] + }, + { + "type": "gemm", + "args": { + "m": 10496, + "n": 512, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wkv", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wkv", + "deepseek-ai/DeepSeek-V4-Pro/attn.wkv", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wkv", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wkv", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wkv" + ] + }, + { + "type": "gemm", + "args": { + "m": 10496, + "n": 512, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.compressor.wkv+wgate", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.compressor.wkv+wgate", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.compressor.wkv+wgate" + ] + }, + { + "type": "gemm", + "args": { + "m": 10496, + "n": 1024, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.compressor.wkv", + "deepseek-ai/DeepSeek-V4-Pro/attn.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro/attn.compressor.wkv", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.compressor.wgate", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.compressor.wkv" + ] + }, + { + "type": "gemm", + "args": { + "m": 10496, + "n": 1536, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w3" + ] + }, + { + "type": "gemm", + "args": { + "m": 10496, + "n": 7168, + "k": 2048, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 10496, + "n": 8192, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 10497, + "n": 1536, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w3" + ] + }, + { + "type": "gemm", + "args": { + "m": 10497, + "n": 4096, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 10497, + "n": 7168, + "k": 768, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w2" + ] + }, + { + "type": "gemm", + "args": { + "m": 10497, + "n": 7168, + "k": 1024, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 10497, + "n": 7168, + "k": 4096, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 10497, + "n": 16384, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 10560, + "n": 1536, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w3" + ] + }, + { + "type": "gemm", + "args": { + "m": 10560, + "n": 7168, + "k": 768, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w2" + ] + }, + { + "type": "gemm", + "args": { + "m": 10560, + "n": 7168, + "k": 4096, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 10560, + "n": 16384, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 10624, + "n": 24, + "k": 28672, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/hc_attn", + "deepseek-ai/DeepSeek-V4-Pro-0813/hc_ffn", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.hc_attn", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.hc_ffn", + "deepseek-ai/DeepSeek-V4-Pro/hc_attn", + "deepseek-ai/DeepSeek-V4-Pro/hc_ffn", + "deepseek-ai/DeepSeek-V4-Pro/mtp.hc_attn", + "deepseek-ai/DeepSeek-V4-Pro/mtp.hc_ffn", + "sgl-project/DeepSeek-V4-Pro-FP8/hc_attn", + "sgl-project/DeepSeek-V4-Pro-FP8/hc_ffn", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.hc_attn", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.hc_ffn" + ] + }, + { + "type": "gemm", + "args": { + "m": 10624, + "n": 64, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.weights_proj", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.weights_proj", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.weights_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 10624, + "n": 256, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.compressor.wkv", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.compressor.wkv", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.compressor.wgate", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.compressor.wkv" + ] + }, + { + "type": "gemm", + "args": { + "m": 10624, + "n": 384, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.gate", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.gate", + "deepseek-ai/DeepSeek-V4-Pro/ffn.gate", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.gate", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.gate", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.gate" + ] + }, + { + "type": "gemm", + "args": { + "m": 10624, + "n": 512, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wkv", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wkv", + "deepseek-ai/DeepSeek-V4-Pro/attn.wkv", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wkv", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wkv", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wkv" + ] + }, + { + "type": "gemm", + "args": { + "m": 10624, + "n": 512, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.compressor.wkv+wgate", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.compressor.wkv+wgate", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.compressor.wkv+wgate" + ] + }, + { + "type": "gemm", + "args": { + "m": 10624, + "n": 1024, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.compressor.wkv", + "deepseek-ai/DeepSeek-V4-Pro/attn.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro/attn.compressor.wkv", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.compressor.wgate", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.compressor.wkv" + ] + }, + { + "type": "gemm", + "args": { + "m": 10624, + "n": 1536, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w3" + ] + }, + { + "type": "gemm", + "args": { + "m": 10624, + "n": 8192, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 10689, + "n": 320, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 10689, + "n": 384, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w3" + ] + }, + { + "type": "gemm", + "args": { + "m": 10689, + "n": 7168, + "k": 192, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w2" + ] + }, + { + "type": "gemm", + "args": { + "m": 10752, + "n": 384, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w3" + ] + }, + { + "type": "gemm", + "args": { + "m": 10752, + "n": 1536, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w3" + ] + }, + { + "type": "gemm", + "args": { + "m": 10752, + "n": 1792, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 10752, + "n": 7168, + "k": 192, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w2" + ] + }, + { + "type": "gemm", + "args": { + "m": 10752, + "n": 7168, + "k": 768, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w2" + ] + }, + { + "type": "gemm", + "args": { + "m": 10752, + "n": 7168, + "k": 896, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 10752, + "n": 7168, + "k": 4096, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 10752, + "n": 16384, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 10776, + "n": 24, + "k": 28672, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/hc_attn", + "deepseek-ai/DeepSeek-V4-Pro-0813/hc_ffn", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.hc_attn", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.hc_ffn", + "deepseek-ai/DeepSeek-V4-Pro/hc_attn", + "deepseek-ai/DeepSeek-V4-Pro/hc_ffn", + "deepseek-ai/DeepSeek-V4-Pro/mtp.hc_attn", + "deepseek-ai/DeepSeek-V4-Pro/mtp.hc_ffn", + "sgl-project/DeepSeek-V4-Pro-FP8/hc_attn", + "sgl-project/DeepSeek-V4-Pro-FP8/hc_ffn", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.hc_attn", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.hc_ffn" + ] + }, + { + "type": "gemm", + "args": { + "m": 10776, + "n": 24, + "k": 49152, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 10776, + "n": 64, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.weights_proj", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.weights_proj", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.weights_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 10776, + "n": 64, + "k": 12288, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 10776, + "n": 120, + "k": 12288, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 10776, + "n": 256, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.compressor.wkv", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.compressor.wkv", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.compressor.wgate", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.compressor.wkv" + ] + }, + { + "type": "gemm", + "args": { + "m": 10776, + "n": 256, + "k": 12288, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 10776, + "n": 384, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.gate", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.gate", + "deepseek-ai/DeepSeek-V4-Pro/ffn.gate", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.gate", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.gate", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.gate" + ] + }, + { + "type": "gemm", + "args": { + "m": 10776, + "n": 512, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wkv", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wkv", + "deepseek-ai/DeepSeek-V4-Pro/attn.wkv", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wkv", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wkv", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wkv" + ] + }, + { + "type": "gemm", + "args": { + "m": 10776, + "n": 512, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.compressor.wkv+wgate", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.compressor.wkv+wgate", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.compressor.wkv+wgate" + ] + }, + { + "type": "gemm", + "args": { + "m": 10776, + "n": 512, + "k": 12288, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 10776, + "n": 512, + "k": 12288, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 10776, + "n": 1024, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.compressor.wkv", + "deepseek-ai/DeepSeek-V4-Pro/attn.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro/attn.compressor.wkv", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.compressor.wgate", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.compressor.wkv" + ] + }, + { + "type": "gemm", + "args": { + "m": 10776, + "n": 1024, + "k": 12288, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 10776, + "n": 1536, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w3" + ] + }, + { + "type": "gemm", + "args": { + "m": 10776, + "n": 1536, + "k": 12288, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 10776, + "n": 7168, + "k": 16384, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 10776, + "n": 8192, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 10776, + "n": 12288, + "k": 16384, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 10776, + "n": 65536, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 10776, + "n": 81920, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 10880, + "n": 384, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w3" + ] + }, + { + "type": "gemm", + "args": { + "m": 10880, + "n": 1792, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 10880, + "n": 2048, + "k": 12288, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 10880, + "n": 7168, + "k": 192, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w2" + ] + }, + { + "type": "gemm", + "args": { + "m": 10880, + "n": 7168, + "k": 896, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 10880, + "n": 12288, + "k": 1024, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 10881, + "n": 10240, + "k": 2048, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 10944, + "n": 384, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w3" + ] + }, + { + "type": "gemm", + "args": { + "m": 10944, + "n": 1792, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 10944, + "n": 7168, + "k": 192, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w2" + ] + }, + { + "type": "gemm", + "args": { + "m": 10944, + "n": 7168, + "k": 896, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 11068, + "n": 7168, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.e_proj", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.h_proj", + "deepseek-ai/DeepSeek-V4-Pro/mtp.e_proj", + "deepseek-ai/DeepSeek-V4-Pro/mtp.h_proj", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.e_proj", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.h_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 11068, + "n": 8192, + "k": 10240, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 11068, + "n": 10240, + "k": 4096, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 11068, + "n": 14336, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 11136, + "n": 1536, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w3" + ] + }, + { + "type": "gemm", + "args": { + "m": 11136, + "n": 7168, + "k": 768, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w2" + ] + }, + { + "type": "gemm", + "args": { + "m": 11136, + "n": 7168, + "k": 4096, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 11136, + "n": 16384, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 11202, + "n": 1792, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 11202, + "n": 7168, + "k": 896, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 11202, + "n": 7168, + "k": 2048, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 11264, + "n": 24, + "k": 49152, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 11264, + "n": 64, + "k": 12288, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 11264, + "n": 120, + "k": 12288, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 11264, + "n": 256, + "k": 12288, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 11264, + "n": 512, + "k": 12288, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 11264, + "n": 512, + "k": 12288, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 11264, + "n": 1024, + "k": 12288, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 11264, + "n": 1536, + "k": 12288, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 11264, + "n": 2048, + "k": 12288, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 11264, + "n": 5120, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 11264, + "n": 12288, + "k": 1024, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 11268, + "n": 24, + "k": 49152, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 11268, + "n": 64, + "k": 12288, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 11268, + "n": 120, + "k": 12288, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 11268, + "n": 256, + "k": 12288, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 11268, + "n": 512, + "k": 12288, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 11268, + "n": 512, + "k": 12288, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 11268, + "n": 1024, + "k": 12288, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 11268, + "n": 1536, + "k": 12288, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 11268, + "n": 5120, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 11268, + "n": 12288, + "k": 1024, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 11304, + "n": 24, + "k": 40960, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 11304, + "n": 64, + "k": 10240, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 11304, + "n": 192, + "k": 10240, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 11304, + "n": 256, + "k": 10240, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 11304, + "n": 512, + "k": 10240, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 11304, + "n": 512, + "k": 10240, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 11304, + "n": 1024, + "k": 10240, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 11304, + "n": 1536, + "k": 10240, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 11304, + "n": 10240, + "k": 16384, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 11328, + "n": 24, + "k": 49152, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 11328, + "n": 64, + "k": 12288, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 11328, + "n": 120, + "k": 12288, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 11328, + "n": 256, + "k": 12288, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 11328, + "n": 512, + "k": 12288, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 11328, + "n": 512, + "k": 12288, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 11328, + "n": 1024, + "k": 12288, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 11328, + "n": 1536, + "k": 12288, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 11328, + "n": 5120, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 11328, + "n": 12288, + "k": 1024, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 11392, + "n": 24, + "k": 49152, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 11392, + "n": 64, + "k": 12288, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 11392, + "n": 120, + "k": 12288, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 11392, + "n": 256, + "k": 12288, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 11392, + "n": 512, + "k": 12288, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 11392, + "n": 512, + "k": 12288, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 11392, + "n": 1024, + "k": 12288, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 11392, + "n": 1536, + "k": 12288, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 11392, + "n": 5120, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 11392, + "n": 12288, + "k": 1024, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 11400, + "n": 384, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w3" + ] + }, + { + "type": "gemm", + "args": { + "m": 11400, + "n": 4096, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 11400, + "n": 7168, + "k": 192, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w2" + ] + }, + { + "type": "gemm", + "args": { + "m": 11400, + "n": 7168, + "k": 1024, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 11584, + "n": 6144, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3" + ] + }, + { + "type": "gemm", + "args": { + "m": 11584, + "n": 7168, + "k": 3072, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w2" + ] + }, + { + "type": "gemm", + "args": { + "m": 11584, + "n": 7168, + "k": 16384, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 11584, + "n": 65536, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 11612, + "n": 7168, + "k": 16384, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 11612, + "n": 10240, + "k": 16384, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 11612, + "n": 65536, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 11648, + "n": 6144, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3" + ] + }, + { + "type": "gemm", + "args": { + "m": 11648, + "n": 7168, + "k": 3072, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w2" + ] + }, + { + "type": "gemm", + "args": { + "m": 11648, + "n": 7168, + "k": 16384, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 11648, + "n": 10240, + "k": 16384, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 11648, + "n": 65536, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 11710, + "n": 6144, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3" + ] + }, + { + "type": "gemm", + "args": { + "m": 11710, + "n": 7168, + "k": 3072, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w2" + ] + }, + { + "type": "gemm", + "args": { + "m": 11712, + "n": 10240, + "k": 16384, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 11724, + "n": 12288, + "k": 8192, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 11724, + "n": 16384, + "k": 12288, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 11776, + "n": 6144, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3" + ] + }, + { + "type": "gemm", + "args": { + "m": 11776, + "n": 7168, + "k": 3072, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w2" + ] + }, + { + "type": "gemm", + "args": { + "m": 11776, + "n": 7168, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.e_proj", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.h_proj", + "deepseek-ai/DeepSeek-V4-Pro/mtp.e_proj", + "deepseek-ai/DeepSeek-V4-Pro/mtp.h_proj", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.e_proj", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.h_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 11776, + "n": 7168, + "k": 16384, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 11776, + "n": 10240, + "k": 16384, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 11776, + "n": 12288, + "k": 8192, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 11776, + "n": 12288, + "k": 16384, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 11776, + "n": 14336, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 11776, + "n": 16384, + "k": 12288, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 11776, + "n": 65536, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 11776, + "n": 81920, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 11840, + "n": 7168, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.e_proj", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.h_proj", + "deepseek-ai/DeepSeek-V4-Pro/mtp.e_proj", + "deepseek-ai/DeepSeek-V4-Pro/mtp.h_proj", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.e_proj", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.h_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 11840, + "n": 12288, + "k": 8192, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 11840, + "n": 12288, + "k": 16384, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 11840, + "n": 14336, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 11840, + "n": 16384, + "k": 12288, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 11840, + "n": 81920, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 11872, + "n": 24, + "k": 28672, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/hc_attn", + "deepseek-ai/DeepSeek-V4-Pro-0813/hc_ffn", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.hc_attn", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.hc_ffn", + "deepseek-ai/DeepSeek-V4-Pro/hc_attn", + "deepseek-ai/DeepSeek-V4-Pro/hc_ffn", + "deepseek-ai/DeepSeek-V4-Pro/mtp.hc_attn", + "deepseek-ai/DeepSeek-V4-Pro/mtp.hc_ffn", + "sgl-project/DeepSeek-V4-Pro-FP8/hc_attn", + "sgl-project/DeepSeek-V4-Pro-FP8/hc_ffn", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.hc_attn", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.hc_ffn" + ] + }, + { + "type": "gemm", + "args": { + "m": 11872, + "n": 24, + "k": 40960, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 11872, + "n": 64, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.weights_proj", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.weights_proj", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.weights_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 11872, + "n": 64, + "k": 10240, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 11872, + "n": 192, + "k": 10240, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 11872, + "n": 256, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.compressor.wkv", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.compressor.wkv", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.compressor.wgate", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.compressor.wkv" + ] + }, + { + "type": "gemm", + "args": { + "m": 11872, + "n": 256, + "k": 10240, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 11872, + "n": 384, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.gate", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.gate", + "deepseek-ai/DeepSeek-V4-Pro/ffn.gate", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.gate", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.gate", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.gate" + ] + }, + { + "type": "gemm", + "args": { + "m": 11872, + "n": 512, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wkv", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wkv", + "deepseek-ai/DeepSeek-V4-Pro/attn.wkv", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wkv", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wkv", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wkv" + ] + }, + { + "type": "gemm", + "args": { + "m": 11872, + "n": 512, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.compressor.wkv+wgate", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.compressor.wkv+wgate", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.compressor.wkv+wgate" + ] + }, + { + "type": "gemm", + "args": { + "m": 11872, + "n": 512, + "k": 10240, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 11872, + "n": 512, + "k": 10240, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 11872, + "n": 1024, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.compressor.wkv", + "deepseek-ai/DeepSeek-V4-Pro/attn.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro/attn.compressor.wkv", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.compressor.wgate", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.compressor.wkv" + ] + }, + { + "type": "gemm", + "args": { + "m": 11872, + "n": 1024, + "k": 10240, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 11872, + "n": 1536, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w3" + ] + }, + { + "type": "gemm", + "args": { + "m": 11872, + "n": 1536, + "k": 10240, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 11872, + "n": 7168, + "k": 16384, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 11872, + "n": 8192, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 11872, + "n": 10240, + "k": 16384, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 11872, + "n": 65536, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 11904, + "n": 10240, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 11904, + "n": 12288, + "k": 2048, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 11904, + "n": 12288, + "k": 8192, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 11904, + "n": 12288, + "k": 16384, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 11904, + "n": 16384, + "k": 12288, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 11904, + "n": 81920, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 11968, + "n": 4096, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 12032, + "n": 4096, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 12032, + "n": 7168, + "k": 1024, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 12056, + "n": 7168, + "k": 16384, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 12056, + "n": 10240, + "k": 16384, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 12056, + "n": 65536, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 12084, + "n": 24, + "k": 40960, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 12084, + "n": 64, + "k": 10240, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 12084, + "n": 192, + "k": 10240, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 12084, + "n": 256, + "k": 10240, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 12084, + "n": 512, + "k": 10240, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 12084, + "n": 512, + "k": 10240, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 12084, + "n": 1024, + "k": 10240, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 12084, + "n": 1536, + "k": 10240, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 12096, + "n": 24, + "k": 40960, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 12096, + "n": 64, + "k": 10240, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 12096, + "n": 192, + "k": 10240, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 12096, + "n": 256, + "k": 10240, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 12096, + "n": 320, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 12096, + "n": 512, + "k": 10240, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 12096, + "n": 512, + "k": 10240, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 12096, + "n": 1024, + "k": 10240, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 12096, + "n": 1536, + "k": 10240, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 12096, + "n": 7168, + "k": 1024, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 12134, + "n": 12288, + "k": 16384, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 12134, + "n": 81920, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 12136, + "n": 320, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 12136, + "n": 6144, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3" + ] + }, + { + "type": "gemm", + "args": { + "m": 12136, + "n": 7168, + "k": 3072, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w2" + ] + }, + { + "type": "gemm", + "args": { + "m": 12160, + "n": 24, + "k": 40960, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 12160, + "n": 64, + "k": 10240, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 12160, + "n": 192, + "k": 10240, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 12160, + "n": 256, + "k": 10240, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 12160, + "n": 320, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 12160, + "n": 512, + "k": 10240, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 12160, + "n": 512, + "k": 10240, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 12160, + "n": 1024, + "k": 10240, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 12160, + "n": 1536, + "k": 10240, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 12160, + "n": 7168, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.e_proj", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.h_proj", + "deepseek-ai/DeepSeek-V4-Pro/mtp.e_proj", + "deepseek-ai/DeepSeek-V4-Pro/mtp.h_proj", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.e_proj", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.h_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 12160, + "n": 14336, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 12224, + "n": 10240, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 12224, + "n": 12288, + "k": 2048, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 12264, + "n": 24, + "k": 49152, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 12264, + "n": 64, + "k": 12288, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 12264, + "n": 120, + "k": 12288, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 12264, + "n": 256, + "k": 12288, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 12264, + "n": 512, + "k": 12288, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 12264, + "n": 512, + "k": 12288, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 12264, + "n": 1024, + "k": 12288, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 12264, + "n": 1536, + "k": 12288, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 12264, + "n": 12288, + "k": 16384, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 12264, + "n": 81920, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 12288, + "n": 24, + "k": 40960, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 12288, + "n": 64, + "k": 10240, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 12288, + "n": 192, + "k": 10240, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 12288, + "n": 256, + "k": 10240, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 12288, + "n": 320, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 12288, + "n": 512, + "k": 10240, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 12288, + "n": 512, + "k": 10240, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 12288, + "n": 1024, + "k": 10240, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 12288, + "n": 1536, + "k": 10240, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 12288, + "n": 10240, + "k": 2048, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 12352, + "n": 10240, + "k": 2048, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 12416, + "n": 4096, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 12416, + "n": 7168, + "k": 1024, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 12416, + "n": 10240, + "k": 2048, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 12417, + "n": 320, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 12417, + "n": 4096, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 12417, + "n": 7168, + "k": 1024, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 12546, + "n": 2048, + "k": 12288, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 12546, + "n": 10240, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 12546, + "n": 12288, + "k": 1024, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 12546, + "n": 12288, + "k": 2048, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 12560, + "n": 24, + "k": 28672, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/hc_attn", + "deepseek-ai/DeepSeek-V4-Pro-0813/hc_ffn", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.hc_attn", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.hc_ffn", + "deepseek-ai/DeepSeek-V4-Pro/hc_attn", + "deepseek-ai/DeepSeek-V4-Pro/hc_ffn", + "deepseek-ai/DeepSeek-V4-Pro/mtp.hc_attn", + "deepseek-ai/DeepSeek-V4-Pro/mtp.hc_ffn", + "sgl-project/DeepSeek-V4-Pro-FP8/hc_attn", + "sgl-project/DeepSeek-V4-Pro-FP8/hc_ffn", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.hc_attn", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.hc_ffn" + ] + }, + { + "type": "gemm", + "args": { + "m": 12560, + "n": 24, + "k": 40960, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 12560, + "n": 64, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.weights_proj", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.weights_proj", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.weights_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 12560, + "n": 64, + "k": 10240, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 12560, + "n": 192, + "k": 10240, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 12560, + "n": 256, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.compressor.wkv", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.compressor.wkv", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.compressor.wgate", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.compressor.wkv" + ] + }, + { + "type": "gemm", + "args": { + "m": 12560, + "n": 256, + "k": 10240, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 12560, + "n": 384, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.gate", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.gate", + "deepseek-ai/DeepSeek-V4-Pro/ffn.gate", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.gate", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.gate", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.gate" + ] + }, + { + "type": "gemm", + "args": { + "m": 12560, + "n": 512, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wkv", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wkv", + "deepseek-ai/DeepSeek-V4-Pro/attn.wkv", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wkv", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wkv", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wkv" + ] + }, + { + "type": "gemm", + "args": { + "m": 12560, + "n": 512, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.compressor.wkv+wgate", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.compressor.wkv+wgate", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.compressor.wkv+wgate" + ] + }, + { + "type": "gemm", + "args": { + "m": 12560, + "n": 512, + "k": 10240, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 12560, + "n": 512, + "k": 10240, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 12560, + "n": 1024, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.compressor.wkv", + "deepseek-ai/DeepSeek-V4-Pro/attn.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro/attn.compressor.wkv", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.compressor.wgate", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.compressor.wkv" + ] + }, + { + "type": "gemm", + "args": { + "m": 12560, + "n": 1024, + "k": 10240, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 12560, + "n": 1536, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w3" + ] + }, + { + "type": "gemm", + "args": { + "m": 12560, + "n": 1536, + "k": 10240, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 12560, + "n": 6144, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3" + ] + }, + { + "type": "gemm", + "args": { + "m": 12560, + "n": 7168, + "k": 3072, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w2" + ] + }, + { + "type": "gemm", + "args": { + "m": 12560, + "n": 7168, + "k": 16384, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 12560, + "n": 8192, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 12560, + "n": 10240, + "k": 16384, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 12560, + "n": 65536, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 12609, + "n": 10240, + "k": 2048, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 12672, + "n": 2048, + "k": 12288, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 12672, + "n": 12288, + "k": 1024, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 12804, + "n": 2048, + "k": 12288, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 12804, + "n": 12288, + "k": 1024, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 12928, + "n": 7168, + "k": 2048, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 12936, + "n": 24, + "k": 40960, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 12936, + "n": 64, + "k": 10240, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 12936, + "n": 192, + "k": 10240, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 12936, + "n": 256, + "k": 10240, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 12936, + "n": 512, + "k": 10240, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 12936, + "n": 512, + "k": 10240, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 12936, + "n": 1024, + "k": 10240, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 12936, + "n": 1536, + "k": 10240, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 12960, + "n": 24, + "k": 40960, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 12960, + "n": 64, + "k": 10240, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 12960, + "n": 192, + "k": 10240, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 12960, + "n": 256, + "k": 10240, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 12960, + "n": 512, + "k": 10240, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 12960, + "n": 512, + "k": 10240, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 12960, + "n": 1024, + "k": 10240, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 12960, + "n": 1536, + "k": 10240, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 12992, + "n": 2048, + "k": 12288, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 12992, + "n": 7168, + "k": 2048, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 12992, + "n": 12288, + "k": 1024, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 12993, + "n": 7168, + "k": 2048, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 13032, + "n": 12288, + "k": 8192, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 13032, + "n": 16384, + "k": 12288, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 13046, + "n": 8192, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 13056, + "n": 24, + "k": 28672, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/hc_attn", + "deepseek-ai/DeepSeek-V4-Pro-0813/hc_ffn", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.hc_attn", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.hc_ffn", + "deepseek-ai/DeepSeek-V4-Pro/hc_attn", + "deepseek-ai/DeepSeek-V4-Pro/hc_ffn", + "deepseek-ai/DeepSeek-V4-Pro/mtp.hc_attn", + "deepseek-ai/DeepSeek-V4-Pro/mtp.hc_ffn", + "sgl-project/DeepSeek-V4-Pro-FP8/hc_attn", + "sgl-project/DeepSeek-V4-Pro-FP8/hc_ffn", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.hc_attn", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.hc_ffn" + ] + }, + { + "type": "gemm", + "args": { + "m": 13056, + "n": 64, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.weights_proj", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.weights_proj", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.weights_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 13056, + "n": 256, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.compressor.wkv", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.compressor.wkv", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.compressor.wgate", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.compressor.wkv" + ] + }, + { + "type": "gemm", + "args": { + "m": 13056, + "n": 384, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.gate", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.gate", + "deepseek-ai/DeepSeek-V4-Pro/ffn.gate", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.gate", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.gate", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.gate" + ] + }, + { + "type": "gemm", + "args": { + "m": 13056, + "n": 512, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wkv", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wkv", + "deepseek-ai/DeepSeek-V4-Pro/attn.wkv", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wkv", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wkv", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wkv" + ] + }, + { + "type": "gemm", + "args": { + "m": 13056, + "n": 512, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.compressor.wkv+wgate", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.compressor.wkv+wgate", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.compressor.wkv+wgate" + ] + }, + { + "type": "gemm", + "args": { + "m": 13056, + "n": 1024, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.compressor.wkv", + "deepseek-ai/DeepSeek-V4-Pro/attn.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro/attn.compressor.wkv", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.compressor.wgate", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.compressor.wkv" + ] + }, + { + "type": "gemm", + "args": { + "m": 13056, + "n": 1536, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w3" + ] + }, + { + "type": "gemm", + "args": { + "m": 13056, + "n": 7168, + "k": 2048, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 13056, + "n": 8192, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 13056, + "n": 10240, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 13056, + "n": 12288, + "k": 2048, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 13072, + "n": 24, + "k": 49152, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 13072, + "n": 64, + "k": 12288, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 13072, + "n": 120, + "k": 12288, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 13072, + "n": 256, + "k": 12288, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 13072, + "n": 512, + "k": 12288, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 13072, + "n": 512, + "k": 12288, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 13072, + "n": 1024, + "k": 12288, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 13072, + "n": 1536, + "k": 12288, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 13072, + "n": 8192, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 13072, + "n": 12288, + "k": 8192, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 13072, + "n": 12288, + "k": 16384, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 13072, + "n": 16384, + "k": 12288, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 13072, + "n": 81920, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 13116, + "n": 24, + "k": 28672, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/hc_attn", + "deepseek-ai/DeepSeek-V4-Pro-0813/hc_ffn", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.hc_attn", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.hc_ffn", + "deepseek-ai/DeepSeek-V4-Pro/hc_attn", + "deepseek-ai/DeepSeek-V4-Pro/hc_ffn", + "deepseek-ai/DeepSeek-V4-Pro/mtp.hc_attn", + "deepseek-ai/DeepSeek-V4-Pro/mtp.hc_ffn", + "sgl-project/DeepSeek-V4-Pro-FP8/hc_attn", + "sgl-project/DeepSeek-V4-Pro-FP8/hc_ffn", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.hc_attn", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.hc_ffn" + ] + }, + { + "type": "gemm", + "args": { + "m": 13116, + "n": 64, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.weights_proj", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.weights_proj", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.weights_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 13116, + "n": 256, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.compressor.wkv", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.compressor.wkv", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.compressor.wgate", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.compressor.wkv" + ] + }, + { + "type": "gemm", + "args": { + "m": 13116, + "n": 384, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.gate", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.gate", + "deepseek-ai/DeepSeek-V4-Pro/ffn.gate", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.gate", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.gate", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.gate" + ] + }, + { + "type": "gemm", + "args": { + "m": 13116, + "n": 512, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wkv", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wkv", + "deepseek-ai/DeepSeek-V4-Pro/attn.wkv", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wkv", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wkv", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wkv" + ] + }, + { + "type": "gemm", + "args": { + "m": 13116, + "n": 512, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.compressor.wkv+wgate", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.compressor.wkv+wgate", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.compressor.wkv+wgate" + ] + }, + { + "type": "gemm", + "args": { + "m": 13116, + "n": 1024, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.compressor.wkv", + "deepseek-ai/DeepSeek-V4-Pro/attn.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro/attn.compressor.wkv", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.compressor.wgate", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.compressor.wkv" + ] + }, + { + "type": "gemm", + "args": { + "m": 13116, + "n": 1536, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w3" + ] + }, + { + "type": "gemm", + "args": { + "m": 13120, + "n": 24, + "k": 28672, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/hc_attn", + "deepseek-ai/DeepSeek-V4-Pro-0813/hc_ffn", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.hc_attn", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.hc_ffn", + "deepseek-ai/DeepSeek-V4-Pro/hc_attn", + "deepseek-ai/DeepSeek-V4-Pro/hc_ffn", + "deepseek-ai/DeepSeek-V4-Pro/mtp.hc_attn", + "deepseek-ai/DeepSeek-V4-Pro/mtp.hc_ffn", + "sgl-project/DeepSeek-V4-Pro-FP8/hc_attn", + "sgl-project/DeepSeek-V4-Pro-FP8/hc_ffn", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.hc_attn", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.hc_ffn" + ] + }, + { + "type": "gemm", + "args": { + "m": 13120, + "n": 64, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.weights_proj", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.weights_proj", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.weights_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 13120, + "n": 256, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.compressor.wkv", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.compressor.wkv", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.compressor.wgate", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.compressor.wkv" + ] + }, + { + "type": "gemm", + "args": { + "m": 13120, + "n": 384, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.gate", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.gate", + "deepseek-ai/DeepSeek-V4-Pro/ffn.gate", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.gate", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.gate", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.gate" + ] + }, + { + "type": "gemm", + "args": { + "m": 13120, + "n": 512, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wkv", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wkv", + "deepseek-ai/DeepSeek-V4-Pro/attn.wkv", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wkv", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wkv", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wkv" + ] + }, + { + "type": "gemm", + "args": { + "m": 13120, + "n": 512, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.compressor.wkv+wgate", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.compressor.wkv+wgate", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.compressor.wkv+wgate" + ] + }, + { + "type": "gemm", + "args": { + "m": 13120, + "n": 1024, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.compressor.wkv", + "deepseek-ai/DeepSeek-V4-Pro/attn.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro/attn.compressor.wkv", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.compressor.wgate", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.compressor.wkv" + ] + }, + { + "type": "gemm", + "args": { + "m": 13120, + "n": 1536, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w3" + ] + }, + { + "type": "gemm", + "args": { + "m": 13120, + "n": 8192, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 13122, + "n": 1536, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w3" + ] + }, + { + "type": "gemm", + "args": { + "m": 13122, + "n": 7168, + "k": 768, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w2" + ] + }, + { + "type": "gemm", + "args": { + "m": 13122, + "n": 7168, + "k": 4096, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 13122, + "n": 16384, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 13184, + "n": 24, + "k": 28672, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/hc_attn", + "deepseek-ai/DeepSeek-V4-Pro-0813/hc_ffn", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.hc_attn", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.hc_ffn", + "deepseek-ai/DeepSeek-V4-Pro/hc_attn", + "deepseek-ai/DeepSeek-V4-Pro/hc_ffn", + "deepseek-ai/DeepSeek-V4-Pro/mtp.hc_attn", + "deepseek-ai/DeepSeek-V4-Pro/mtp.hc_ffn", + "sgl-project/DeepSeek-V4-Pro-FP8/hc_attn", + "sgl-project/DeepSeek-V4-Pro-FP8/hc_ffn", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.hc_attn", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.hc_ffn" + ] + }, + { + "type": "gemm", + "args": { + "m": 13184, + "n": 64, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.weights_proj", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.weights_proj", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.weights_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 13184, + "n": 256, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.compressor.wkv", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.compressor.wkv", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.compressor.wgate", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.compressor.wkv" + ] + }, + { + "type": "gemm", + "args": { + "m": 13184, + "n": 384, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.gate", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.gate", + "deepseek-ai/DeepSeek-V4-Pro/ffn.gate", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.gate", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.gate", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.gate" + ] + }, + { + "type": "gemm", + "args": { + "m": 13184, + "n": 512, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wkv", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wkv", + "deepseek-ai/DeepSeek-V4-Pro/attn.wkv", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wkv", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wkv", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wkv" + ] + }, + { + "type": "gemm", + "args": { + "m": 13184, + "n": 512, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.compressor.wkv+wgate", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.compressor.wkv+wgate", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.compressor.wkv+wgate" + ] + }, + { + "type": "gemm", + "args": { + "m": 13184, + "n": 1024, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.compressor.wkv", + "deepseek-ai/DeepSeek-V4-Pro/attn.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro/attn.compressor.wkv", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.compressor.wgate", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.compressor.wkv" + ] + }, + { + "type": "gemm", + "args": { + "m": 13184, + "n": 1536, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w3" + ] + }, + { + "type": "gemm", + "args": { + "m": 13184, + "n": 8192, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 13312, + "n": 2048, + "k": 12288, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 13312, + "n": 12288, + "k": 1024, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 13320, + "n": 5120, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 13320, + "n": 8192, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 13320, + "n": 12288, + "k": 1024, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 13440, + "n": 384, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w3" + ] + }, + { + "type": "gemm", + "args": { + "m": 13440, + "n": 7168, + "k": 192, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w2" + ] + }, + { + "type": "gemm", + "args": { + "m": 13470, + "n": 12288, + "k": 8192, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 13470, + "n": 16384, + "k": 12288, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 13504, + "n": 1536, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w3" + ] + }, + { + "type": "gemm", + "args": { + "m": 13504, + "n": 7168, + "k": 768, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w2" + ] + }, + { + "type": "gemm", + "args": { + "m": 13504, + "n": 7168, + "k": 4096, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 13504, + "n": 16384, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 13506, + "n": 384, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w3" + ] + }, + { + "type": "gemm", + "args": { + "m": 13506, + "n": 7168, + "k": 192, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w2" + ] + }, + { + "type": "gemm", + "args": { + "m": 13568, + "n": 1792, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 13568, + "n": 7168, + "k": 896, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 13658, + "n": 2048, + "k": 10240, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 13658, + "n": 10240, + "k": 1024, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 13658, + "n": 10240, + "k": 4096, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 13696, + "n": 1792, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 13696, + "n": 7168, + "k": 896, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 13760, + "n": 1792, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 13760, + "n": 7168, + "k": 896, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 13790, + "n": 7168, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.e_proj", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.h_proj", + "deepseek-ai/DeepSeek-V4-Pro/mtp.e_proj", + "deepseek-ai/DeepSeek-V4-Pro/mtp.h_proj", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.e_proj", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.h_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 13790, + "n": 14336, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 13796, + "n": 768, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w3" + ] + }, + { + "type": "gemm", + "args": { + "m": 13796, + "n": 7168, + "k": 384, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w2" + ] + }, + { + "type": "gemm", + "args": { + "m": 13824, + "n": 384, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w3" + ] + }, + { + "type": "gemm", + "args": { + "m": 13824, + "n": 1536, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w3" + ] + }, + { + "type": "gemm", + "args": { + "m": 13824, + "n": 7168, + "k": 192, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w2" + ] + }, + { + "type": "gemm", + "args": { + "m": 13824, + "n": 7168, + "k": 768, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w2" + ] + }, + { + "type": "gemm", + "args": { + "m": 13824, + "n": 7168, + "k": 4096, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 13824, + "n": 16384, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 13888, + "n": 384, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w3" + ] + }, + { + "type": "gemm", + "args": { + "m": 13888, + "n": 7168, + "k": 192, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w2" + ] + }, + { + "type": "gemm", + "args": { + "m": 13952, + "n": 1536, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w3" + ] + }, + { + "type": "gemm", + "args": { + "m": 13952, + "n": 7168, + "k": 768, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w2" + ] + }, + { + "type": "gemm", + "args": { + "m": 13952, + "n": 7168, + "k": 4096, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 13952, + "n": 16384, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 14000, + "n": 24, + "k": 28672, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/hc_attn", + "deepseek-ai/DeepSeek-V4-Pro-0813/hc_ffn", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.hc_attn", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.hc_ffn", + "deepseek-ai/DeepSeek-V4-Pro/hc_attn", + "deepseek-ai/DeepSeek-V4-Pro/hc_ffn", + "deepseek-ai/DeepSeek-V4-Pro/mtp.hc_attn", + "deepseek-ai/DeepSeek-V4-Pro/mtp.hc_ffn", + "sgl-project/DeepSeek-V4-Pro-FP8/hc_attn", + "sgl-project/DeepSeek-V4-Pro-FP8/hc_ffn", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.hc_attn", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.hc_ffn" + ] + }, + { + "type": "gemm", + "args": { + "m": 14000, + "n": 64, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.weights_proj", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.weights_proj", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.weights_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 14000, + "n": 256, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.compressor.wkv", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.compressor.wkv", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.compressor.wgate", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.compressor.wkv" + ] + }, + { + "type": "gemm", + "args": { + "m": 14000, + "n": 384, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.gate", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.gate", + "deepseek-ai/DeepSeek-V4-Pro/ffn.gate", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.gate", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.gate", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.gate" + ] + }, + { + "type": "gemm", + "args": { + "m": 14000, + "n": 512, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wkv", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wkv", + "deepseek-ai/DeepSeek-V4-Pro/attn.wkv", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wkv", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wkv", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wkv" + ] + }, + { + "type": "gemm", + "args": { + "m": 14000, + "n": 512, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.compressor.wkv+wgate", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.compressor.wkv+wgate", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.compressor.wkv+wgate" + ] + }, + { + "type": "gemm", + "args": { + "m": 14000, + "n": 1024, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.compressor.wkv", + "deepseek-ai/DeepSeek-V4-Pro/attn.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro/attn.compressor.wkv", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.compressor.wgate", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.compressor.wkv" + ] + }, + { + "type": "gemm", + "args": { + "m": 14000, + "n": 1536, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w3" + ] + }, + { + "type": "gemm", + "args": { + "m": 14080, + "n": 384, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w3" + ] + }, + { + "type": "gemm", + "args": { + "m": 14080, + "n": 7168, + "k": 192, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w2" + ] + }, + { + "type": "gemm", + "args": { + "m": 14144, + "n": 24, + "k": 49152, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 14144, + "n": 64, + "k": 12288, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 14144, + "n": 120, + "k": 12288, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 14144, + "n": 256, + "k": 12288, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 14144, + "n": 512, + "k": 12288, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 14144, + "n": 512, + "k": 12288, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 14144, + "n": 1024, + "k": 12288, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 14144, + "n": 1536, + "k": 12288, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 14144, + "n": 5120, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 14144, + "n": 12288, + "k": 1024, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 14148, + "n": 24, + "k": 49152, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 14148, + "n": 64, + "k": 12288, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 14148, + "n": 120, + "k": 12288, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 14148, + "n": 256, + "k": 12288, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 14148, + "n": 512, + "k": 12288, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 14148, + "n": 512, + "k": 12288, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 14148, + "n": 1024, + "k": 12288, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 14148, + "n": 1536, + "k": 12288, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 14148, + "n": 5120, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 14148, + "n": 12288, + "k": 1024, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 14169, + "n": 1024, + "k": 10240, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 14169, + "n": 10240, + "k": 512, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 14208, + "n": 24, + "k": 49152, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 14208, + "n": 64, + "k": 12288, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 14208, + "n": 120, + "k": 12288, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 14208, + "n": 256, + "k": 12288, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 14208, + "n": 512, + "k": 12288, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 14208, + "n": 512, + "k": 12288, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 14208, + "n": 1024, + "k": 12288, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 14208, + "n": 1536, + "k": 12288, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 14208, + "n": 5120, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 14208, + "n": 12288, + "k": 1024, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 14336, + "n": 24, + "k": 49152, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 14336, + "n": 64, + "k": 12288, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 14336, + "n": 120, + "k": 12288, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 14336, + "n": 256, + "k": 12288, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 14336, + "n": 512, + "k": 12288, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 14336, + "n": 512, + "k": 12288, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 14336, + "n": 1024, + "k": 12288, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 14336, + "n": 1536, + "k": 12288, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 14472, + "n": 24, + "k": 28672, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/hc_attn", + "deepseek-ai/DeepSeek-V4-Pro-0813/hc_ffn", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.hc_attn", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.hc_ffn", + "deepseek-ai/DeepSeek-V4-Pro/hc_attn", + "deepseek-ai/DeepSeek-V4-Pro/hc_ffn", + "deepseek-ai/DeepSeek-V4-Pro/mtp.hc_attn", + "deepseek-ai/DeepSeek-V4-Pro/mtp.hc_ffn", + "sgl-project/DeepSeek-V4-Pro-FP8/hc_attn", + "sgl-project/DeepSeek-V4-Pro-FP8/hc_ffn", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.hc_attn", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.hc_ffn" + ] + }, + { + "type": "gemm", + "args": { + "m": 14472, + "n": 64, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.weights_proj", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.weights_proj", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.weights_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 14472, + "n": 256, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.compressor.wkv", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.compressor.wkv", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.compressor.wgate", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.compressor.wkv" + ] + }, + { + "type": "gemm", + "args": { + "m": 14472, + "n": 512, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wkv", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wkv", + "deepseek-ai/DeepSeek-V4-Pro/attn.wkv", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wkv", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wkv", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wkv" + ] + }, + { + "type": "gemm", + "args": { + "m": 14472, + "n": 512, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.compressor.wkv+wgate", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.compressor.wkv+wgate", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.compressor.wkv+wgate" + ] + }, + { + "type": "gemm", + "args": { + "m": 14472, + "n": 1024, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.compressor.wkv", + "deepseek-ai/DeepSeek-V4-Pro/attn.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro/attn.compressor.wkv", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.compressor.wgate", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.compressor.wkv" + ] + }, + { + "type": "gemm", + "args": { + "m": 14472, + "n": 1536, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w3" + ] + }, + { + "type": "gemm", + "args": { + "m": 14472, + "n": 4096, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 14472, + "n": 7168, + "k": 1024, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 14498, + "n": 7168, + "k": 16384, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 14498, + "n": 65536, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 14528, + "n": 6144, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3" + ] + }, + { + "type": "gemm", + "args": { + "m": 14528, + "n": 7168, + "k": 3072, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w2" + ] + }, + { + "type": "gemm", + "args": { + "m": 14528, + "n": 7168, + "k": 16384, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 14528, + "n": 10240, + "k": 16384, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 14528, + "n": 65536, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 14586, + "n": 10240, + "k": 16384, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 14592, + "n": 5120, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 14592, + "n": 6144, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3" + ] + }, + { + "type": "gemm", + "args": { + "m": 14592, + "n": 7168, + "k": 3072, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w2" + ] + }, + { + "type": "gemm", + "args": { + "m": 14592, + "n": 7168, + "k": 16384, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 14592, + "n": 10240, + "k": 16384, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 14592, + "n": 12288, + "k": 1024, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 14592, + "n": 65536, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 14720, + "n": 6144, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3" + ] + }, + { + "type": "gemm", + "args": { + "m": 14720, + "n": 7168, + "k": 3072, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w2" + ] + }, + { + "type": "gemm", + "args": { + "m": 14720, + "n": 7168, + "k": 16384, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 14720, + "n": 10240, + "k": 16384, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 14720, + "n": 65536, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 14721, + "n": 6144, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3" + ] + }, + { + "type": "gemm", + "args": { + "m": 14721, + "n": 7168, + "k": 3072, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w2" + ] + }, + { + "type": "gemm", + "args": { + "m": 14784, + "n": 7168, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.e_proj", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.h_proj", + "deepseek-ai/DeepSeek-V4-Pro/mtp.e_proj", + "deepseek-ai/DeepSeek-V4-Pro/mtp.h_proj", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.e_proj", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.h_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 14784, + "n": 12288, + "k": 8192, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 14784, + "n": 12288, + "k": 16384, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 14784, + "n": 14336, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 14784, + "n": 16384, + "k": 12288, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 14784, + "n": 81920, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 14848, + "n": 7168, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.e_proj", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.h_proj", + "deepseek-ai/DeepSeek-V4-Pro/mtp.e_proj", + "deepseek-ai/DeepSeek-V4-Pro/mtp.h_proj", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.e_proj", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.h_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 14848, + "n": 14336, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 14876, + "n": 7168, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.e_proj", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.h_proj", + "deepseek-ai/DeepSeek-V4-Pro/mtp.e_proj", + "deepseek-ai/DeepSeek-V4-Pro/mtp.h_proj", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.e_proj", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.h_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 14876, + "n": 12288, + "k": 16384, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 14876, + "n": 14336, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 14876, + "n": 81920, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 14922, + "n": 8192, + "k": 10240, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 14922, + "n": 10240, + "k": 4096, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 14976, + "n": 7168, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.e_proj", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.h_proj", + "deepseek-ai/DeepSeek-V4-Pro/mtp.e_proj", + "deepseek-ai/DeepSeek-V4-Pro/mtp.h_proj", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.e_proj", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.h_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 14976, + "n": 10240, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 14976, + "n": 12288, + "k": 2048, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 14976, + "n": 12288, + "k": 16384, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 14976, + "n": 14336, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 14976, + "n": 81920, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 15000, + "n": 7168, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.e_proj", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.h_proj", + "deepseek-ai/DeepSeek-V4-Pro/mtp.e_proj", + "deepseek-ai/DeepSeek-V4-Pro/mtp.h_proj", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.e_proj", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.h_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 15000, + "n": 7168, + "k": 16384, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 15000, + "n": 8192, + "k": 10240, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 15000, + "n": 10240, + "k": 4096, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 15000, + "n": 10240, + "k": 16384, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 15000, + "n": 14336, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 15000, + "n": 65536, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 15024, + "n": 10240, + "k": 16384, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 15024, + "n": 65536, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 15042, + "n": 1792, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 15042, + "n": 4096, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 15042, + "n": 7168, + "k": 896, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 15042, + "n": 7168, + "k": 1024, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 15042, + "n": 10240, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 15042, + "n": 12288, + "k": 2048, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 15104, + "n": 12288, + "k": 8192, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 15104, + "n": 12288, + "k": 16384, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 15104, + "n": 16384, + "k": 12288, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 15104, + "n": 81920, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 15168, + "n": 24, + "k": 40960, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 15168, + "n": 64, + "k": 10240, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 15168, + "n": 192, + "k": 10240, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 15168, + "n": 256, + "k": 10240, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 15168, + "n": 320, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 15168, + "n": 512, + "k": 10240, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 15168, + "n": 512, + "k": 10240, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 15168, + "n": 1024, + "k": 10240, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 15168, + "n": 1536, + "k": 10240, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 15232, + "n": 24, + "k": 40960, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 15232, + "n": 64, + "k": 10240, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 15232, + "n": 192, + "k": 10240, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 15232, + "n": 256, + "k": 10240, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 15232, + "n": 320, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 15232, + "n": 512, + "k": 10240, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 15232, + "n": 512, + "k": 10240, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 15232, + "n": 1024, + "k": 10240, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 15232, + "n": 1536, + "k": 10240, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 15232, + "n": 4096, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 15232, + "n": 7168, + "k": 1024, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 15232, + "n": 12288, + "k": 8192, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 15232, + "n": 16384, + "k": 12288, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 15312, + "n": 320, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 15312, + "n": 7168, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.e_proj", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.h_proj", + "deepseek-ai/DeepSeek-V4-Pro/mtp.e_proj", + "deepseek-ai/DeepSeek-V4-Pro/mtp.h_proj", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.e_proj", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.h_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 15312, + "n": 7168, + "k": 16384, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 15312, + "n": 14336, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 15360, + "n": 24, + "k": 40960, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 15360, + "n": 64, + "k": 10240, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 15360, + "n": 192, + "k": 10240, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 15360, + "n": 256, + "k": 10240, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 15360, + "n": 320, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 15360, + "n": 512, + "k": 10240, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 15360, + "n": 512, + "k": 10240, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 15360, + "n": 1024, + "k": 10240, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 15360, + "n": 1536, + "k": 10240, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 15360, + "n": 4096, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 15360, + "n": 7168, + "k": 1024, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 15360, + "n": 10240, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 15360, + "n": 12288, + "k": 2048, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 15424, + "n": 4096, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 15424, + "n": 7168, + "k": 1024, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 15424, + "n": 10240, + "k": 2048, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 15425, + "n": 320, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 15560, + "n": 24, + "k": 49152, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 15560, + "n": 64, + "k": 12288, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 15560, + "n": 120, + "k": 12288, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 15560, + "n": 256, + "k": 12288, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 15560, + "n": 512, + "k": 12288, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 15560, + "n": 512, + "k": 12288, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 15560, + "n": 1024, + "k": 12288, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 15560, + "n": 1536, + "k": 12288, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 15560, + "n": 5120, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 15560, + "n": 12288, + "k": 1024, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 15616, + "n": 10240, + "k": 2048, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 15680, + "n": 10240, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 15680, + "n": 12288, + "k": 2048, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 15744, + "n": 10240, + "k": 2048, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 15810, + "n": 24, + "k": 40960, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 15810, + "n": 64, + "k": 10240, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 15810, + "n": 192, + "k": 10240, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 15810, + "n": 256, + "k": 10240, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 15810, + "n": 512, + "k": 10240, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 15810, + "n": 512, + "k": 10240, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 15810, + "n": 1024, + "k": 10240, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 15810, + "n": 1536, + "k": 10240, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 15872, + "n": 2048, + "k": 12288, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 15872, + "n": 12288, + "k": 1024, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 16192, + "n": 7168, + "k": 2048, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 16257, + "n": 7168, + "k": 2048, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 16320, + "n": 8192, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 16347, + "n": 24, + "k": 28672, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/hc_attn", + "deepseek-ai/DeepSeek-V4-Pro-0813/hc_ffn", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.hc_attn", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.hc_ffn", + "deepseek-ai/DeepSeek-V4-Pro/hc_attn", + "deepseek-ai/DeepSeek-V4-Pro/hc_ffn", + "deepseek-ai/DeepSeek-V4-Pro/mtp.hc_attn", + "deepseek-ai/DeepSeek-V4-Pro/mtp.hc_ffn", + "sgl-project/DeepSeek-V4-Pro-FP8/hc_attn", + "sgl-project/DeepSeek-V4-Pro-FP8/hc_ffn", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.hc_attn", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.hc_ffn" + ] + }, + { + "type": "gemm", + "args": { + "m": 16347, + "n": 64, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.weights_proj", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.weights_proj", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.weights_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 16347, + "n": 256, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.compressor.wkv", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.compressor.wkv", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.compressor.wgate", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.compressor.wkv" + ] + }, + { + "type": "gemm", + "args": { + "m": 16347, + "n": 384, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.gate", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.gate", + "deepseek-ai/DeepSeek-V4-Pro/ffn.gate", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.gate", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.gate", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.gate" + ] + }, + { + "type": "gemm", + "args": { + "m": 16347, + "n": 512, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wkv", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wkv", + "deepseek-ai/DeepSeek-V4-Pro/attn.wkv", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wkv", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wkv", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wkv" + ] + }, + { + "type": "gemm", + "args": { + "m": 16347, + "n": 512, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.compressor.wkv+wgate", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.compressor.wkv+wgate", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.compressor.wkv+wgate" + ] + }, + { + "type": "gemm", + "args": { + "m": 16347, + "n": 1024, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.compressor.wkv", + "deepseek-ai/DeepSeek-V4-Pro/attn.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro/attn.compressor.wkv", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.compressor.wgate", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.compressor.wkv" + ] + }, + { + "type": "gemm", + "args": { + "m": 16347, + "n": 1536, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w3" + ] + }, + { + "type": "gemm", + "args": { + "m": 16347, + "n": 8192, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 16384, + "n": 24, + "k": 28672, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/hc_attn", + "deepseek-ai/DeepSeek-V4-Pro-0813/hc_ffn", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.hc_attn", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.hc_ffn", + "deepseek-ai/DeepSeek-V4-Pro/hc_attn", + "deepseek-ai/DeepSeek-V4-Pro/hc_ffn", + "deepseek-ai/DeepSeek-V4-Pro/mtp.hc_attn", + "deepseek-ai/DeepSeek-V4-Pro/mtp.hc_ffn", + "sgl-project/DeepSeek-V4-Pro-FP8/hc_attn", + "sgl-project/DeepSeek-V4-Pro-FP8/hc_ffn", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.hc_attn", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.hc_ffn" + ] + }, + { + "type": "gemm", + "args": { + "m": 16384, + "n": 64, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.weights_proj", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.weights_proj", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.weights_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 16384, + "n": 256, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.compressor.wkv", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.compressor.wkv", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.compressor.wgate", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.compressor.wkv" + ] + }, + { + "type": "gemm", + "args": { + "m": 16384, + "n": 384, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.gate", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.gate", + "deepseek-ai/DeepSeek-V4-Pro/ffn.gate", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.gate", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.gate", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.gate" + ] + }, + { + "type": "gemm", + "args": { + "m": 16384, + "n": 512, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wkv", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wkv", + "deepseek-ai/DeepSeek-V4-Pro/attn.wkv", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wkv", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wkv", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wkv" + ] + }, + { + "type": "gemm", + "args": { + "m": 16384, + "n": 512, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.compressor.wkv+wgate", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.compressor.wkv+wgate", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.compressor.wkv+wgate" + ] + }, + { + "type": "gemm", + "args": { + "m": 16384, + "n": 768, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w3" + ] + }, + { + "type": "gemm", + "args": { + "m": 16384, + "n": 1024, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.compressor.wkv", + "deepseek-ai/DeepSeek-V4-Pro/attn.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro/attn.compressor.wkv", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.compressor.wgate", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.compressor.wkv" + ] + }, + { + "type": "gemm", + "args": { + "m": 16384, + "n": 1024, + "k": 10240, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 16384, + "n": 1024, + "k": 12288, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 16384, + "n": 1536, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w3" + ] + }, + { + "type": "gemm", + "args": { + "m": 16384, + "n": 2048, + "k": 10240, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 16384, + "n": 7168, + "k": 384, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w2" + ] + }, + { + "type": "gemm", + "args": { + "m": 16384, + "n": 7168, + "k": 2048, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 16384, + "n": 8192, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 16384, + "n": 8192, + "k": 10240, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 16384, + "n": 10240, + "k": 512, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 16384, + "n": 10240, + "k": 1024, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 16384, + "n": 10240, + "k": 4096, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 16384, + "n": 12288, + "k": 512, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 16385, + "n": 2048, + "k": 12288, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 16385, + "n": 10240, + "k": 2048, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 16385, + "n": 12288, + "k": 1024, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 16385, + "n": 12288, + "k": 8192, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 16385, + "n": 16384, + "k": 12288, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 16386, + "n": 1024, + "k": 12288, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 16386, + "n": 12288, + "k": 512, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 16392, + "n": 24, + "k": 28672, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/hc_attn", + "deepseek-ai/DeepSeek-V4-Pro-0813/hc_ffn", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.hc_attn", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.hc_ffn", + "deepseek-ai/DeepSeek-V4-Pro/hc_attn", + "deepseek-ai/DeepSeek-V4-Pro/hc_ffn", + "deepseek-ai/DeepSeek-V4-Pro/mtp.hc_attn", + "deepseek-ai/DeepSeek-V4-Pro/mtp.hc_ffn", + "sgl-project/DeepSeek-V4-Pro-FP8/hc_attn", + "sgl-project/DeepSeek-V4-Pro-FP8/hc_ffn", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.hc_attn", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.hc_ffn" + ] + }, + { + "type": "gemm", + "args": { + "m": 16392, + "n": 64, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.weights_proj", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.weights_proj", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.weights_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 16392, + "n": 256, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.compressor.wkv", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.compressor.wkv", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.compressor.wgate", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.compressor.wkv" + ] + }, + { + "type": "gemm", + "args": { + "m": 16392, + "n": 320, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 16392, + "n": 384, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.gate", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.gate", + "deepseek-ai/DeepSeek-V4-Pro/ffn.gate", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.gate", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.gate", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.gate" + ] + }, + { + "type": "gemm", + "args": { + "m": 16392, + "n": 512, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wkv", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wkv", + "deepseek-ai/DeepSeek-V4-Pro/attn.wkv", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wkv", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wkv", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wkv" + ] + }, + { + "type": "gemm", + "args": { + "m": 16392, + "n": 512, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.compressor.wkv+wgate", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.compressor.wkv+wgate", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.compressor.wkv+wgate" + ] + }, + { + "type": "gemm", + "args": { + "m": 16392, + "n": 1024, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.compressor.wkv", + "deepseek-ai/DeepSeek-V4-Pro/attn.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro/attn.compressor.wkv", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.compressor.wgate", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.compressor.wkv" + ] + }, + { + "type": "gemm", + "args": { + "m": 16392, + "n": 1536, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w3" + ] + }, + { + "type": "gemm", + "args": { + "m": 16392, + "n": 1792, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 16392, + "n": 6144, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3" + ] + }, + { + "type": "gemm", + "args": { + "m": 16392, + "n": 7168, + "k": 768, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w2" + ] + }, + { + "type": "gemm", + "args": { + "m": 16392, + "n": 7168, + "k": 896, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [] + }, + { + "type": "gemm", + "args": { + "m": 16392, + "n": 7168, + "k": 2048, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 16392, + "n": 7168, + "k": 3072, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w2" + ] + }, + { + "type": "gemm", + "args": { + "m": 16392, + "n": 7168, + "k": 4096, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 16392, + "n": 8192, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 16392, + "n": 16384, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 22400, + "n": 24, + "k": 28672, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/hc_attn", + "deepseek-ai/DeepSeek-V4-Pro-0813/hc_ffn", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.hc_attn", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.hc_ffn", + "deepseek-ai/DeepSeek-V4-Pro/hc_attn", + "deepseek-ai/DeepSeek-V4-Pro/hc_ffn", + "deepseek-ai/DeepSeek-V4-Pro/mtp.hc_attn", + "deepseek-ai/DeepSeek-V4-Pro/mtp.hc_ffn", + "sgl-project/DeepSeek-V4-Pro-FP8/hc_attn", + "sgl-project/DeepSeek-V4-Pro-FP8/hc_ffn", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.hc_attn", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.hc_ffn" + ] + }, + { + "type": "gemm", + "args": { + "m": 22400, + "n": 64, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.weights_proj", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.weights_proj", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.weights_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 22400, + "n": 256, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.compressor.wkv", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.compressor.wkv", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.compressor.wgate", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.compressor.wkv" + ] + }, + { + "type": "gemm", + "args": { + "m": 22400, + "n": 384, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.gate", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.gate", + "deepseek-ai/DeepSeek-V4-Pro/ffn.gate", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.gate", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.gate", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.gate" + ] + }, + { + "type": "gemm", + "args": { + "m": 22400, + "n": 512, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wkv", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wkv", + "deepseek-ai/DeepSeek-V4-Pro/attn.wkv", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wkv", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wkv", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wkv" + ] + }, + { + "type": "gemm", + "args": { + "m": 22400, + "n": 512, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.compressor.wkv+wgate", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.compressor.wkv+wgate", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.compressor.wkv+wgate" + ] + }, + { + "type": "gemm", + "args": { + "m": 22400, + "n": 1024, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.compressor.wkv", + "deepseek-ai/DeepSeek-V4-Pro/attn.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro/attn.compressor.wkv", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.compressor.wgate", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.compressor.wkv" + ] + }, + { + "type": "gemm", + "args": { + "m": 22400, + "n": 1536, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w3" + ] + }, + { + "type": "gemm", + "args": { + "m": 22400, + "n": 6144, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3" + ] + }, + { + "type": "gemm", + "args": { + "m": 22400, + "n": 7168, + "k": 3072, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w2" + ] + }, + { + "type": "gemm", + "args": { + "m": 22400, + "n": 7168, + "k": 16384, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 22400, + "n": 8192, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 22400, + "n": 65536, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 29291, + "n": 24, + "k": 28672, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/hc_attn", + "deepseek-ai/DeepSeek-V4-Pro-0813/hc_ffn", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.hc_attn", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.hc_ffn", + "deepseek-ai/DeepSeek-V4-Pro/hc_attn", + "deepseek-ai/DeepSeek-V4-Pro/hc_ffn", + "deepseek-ai/DeepSeek-V4-Pro/mtp.hc_attn", + "deepseek-ai/DeepSeek-V4-Pro/mtp.hc_ffn", + "sgl-project/DeepSeek-V4-Pro-FP8/hc_attn", + "sgl-project/DeepSeek-V4-Pro-FP8/hc_ffn", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.hc_attn", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.hc_ffn" + ] + }, + { + "type": "gemm", + "args": { + "m": 29291, + "n": 64, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.weights_proj", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.weights_proj", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.weights_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 29291, + "n": 256, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.compressor.wkv", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.compressor.wkv", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.compressor.wgate", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.compressor.wkv" + ] + }, + { + "type": "gemm", + "args": { + "m": 29291, + "n": 384, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.gate", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.gate", + "deepseek-ai/DeepSeek-V4-Pro/ffn.gate", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.gate", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.gate", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.gate" + ] + }, + { + "type": "gemm", + "args": { + "m": 29291, + "n": 512, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wkv", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wkv", + "deepseek-ai/DeepSeek-V4-Pro/attn.wkv", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wkv", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wkv", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wkv" + ] + }, + { + "type": "gemm", + "args": { + "m": 29291, + "n": 512, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.compressor.wkv+wgate", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.compressor.wkv+wgate", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.compressor.wkv+wgate" + ] + }, + { + "type": "gemm", + "args": { + "m": 29291, + "n": 1024, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.compressor.wkv", + "deepseek-ai/DeepSeek-V4-Pro/attn.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro/attn.compressor.wkv", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.compressor.wgate", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.compressor.wkv" + ] + }, + { + "type": "gemm", + "args": { + "m": 29291, + "n": 1536, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w3" + ] + }, + { + "type": "gemm", + "args": { + "m": 29291, + "n": 6144, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3" + ] + }, + { + "type": "gemm", + "args": { + "m": 29291, + "n": 7168, + "k": 3072, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w2" + ] + }, + { + "type": "gemm", + "args": { + "m": 29291, + "n": 7168, + "k": 16384, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 29291, + "n": 8192, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 29291, + "n": 65536, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 32028, + "n": 6144, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3" + ] + }, + { + "type": "gemm", + "args": { + "m": 32028, + "n": 7168, + "k": 3072, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w2" + ] + }, + { + "type": "gemm", + "args": { + "m": 32768, + "n": 24, + "k": 28672, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/hc_attn", + "deepseek-ai/DeepSeek-V4-Pro-0813/hc_ffn", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.hc_attn", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.hc_ffn", + "deepseek-ai/DeepSeek-V4-Pro/hc_attn", + "deepseek-ai/DeepSeek-V4-Pro/hc_ffn", + "deepseek-ai/DeepSeek-V4-Pro/mtp.hc_attn", + "deepseek-ai/DeepSeek-V4-Pro/mtp.hc_ffn", + "sgl-project/DeepSeek-V4-Pro-FP8/hc_attn", + "sgl-project/DeepSeek-V4-Pro-FP8/hc_ffn", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.hc_attn", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.hc_ffn" + ] + }, + { + "type": "gemm", + "args": { + "m": 32768, + "n": 64, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.weights_proj", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.weights_proj", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.weights_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 32768, + "n": 256, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.compressor.wkv", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.compressor.wkv", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.compressor.wgate", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.compressor.wkv" + ] + }, + { + "type": "gemm", + "args": { + "m": 32768, + "n": 384, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.gate", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.gate", + "deepseek-ai/DeepSeek-V4-Pro/ffn.gate", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.gate", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.gate", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.gate" + ] + }, + { + "type": "gemm", + "args": { + "m": 32768, + "n": 384, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w3" + ] + }, + { + "type": "gemm", + "args": { + "m": 32768, + "n": 512, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wkv", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wkv", + "deepseek-ai/DeepSeek-V4-Pro/attn.wkv", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wkv", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wkv", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wkv" + ] + }, + { + "type": "gemm", + "args": { + "m": 32768, + "n": 512, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.compressor.wkv+wgate", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.compressor.wkv+wgate", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.compressor.wkv+wgate" + ] + }, + { + "type": "gemm", + "args": { + "m": 32768, + "n": 1024, + "k": 7168, + "a": { + "dtype": "bf16" + }, + "b": { + "dtype": "bf16" + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.compressor.wkv", + "deepseek-ai/DeepSeek-V4-Pro/attn.compressor.wgate", + "deepseek-ai/DeepSeek-V4-Pro/attn.compressor.wkv", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.compressor.wgate", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.compressor.wkv" + ] + }, + { + "type": "gemm", + "args": { + "m": 32768, + "n": 1536, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w3", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_a", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w1", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w3", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_a", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w1", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w3" + ] + }, + { + "type": "gemm", + "args": { + "m": 32768, + "n": 4096, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 32768, + "n": 6144, + "k": 7168, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w1+w3", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w1+w3", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w1+w3" + ] + }, + { + "type": "gemm", + "args": { + "m": 32768, + "n": 7168, + "k": 192, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w2" + ] + }, + { + "type": "gemm", + "args": { + "m": 32768, + "n": 7168, + "k": 1024, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 32768, + "n": 7168, + "k": 3072, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/ffn.shared_experts.w2", + "deepseek-ai/DeepSeek-V4-Pro/mtp.ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/ffn.shared_experts.w2", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.ffn.shared_experts.w2" + ] + }, + { + "type": "gemm", + "args": { + "m": 32768, + "n": 7168, + "k": 16384, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wo_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wo_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wo_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 32768, + "n": 8192, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.indexer.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.indexer.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.indexer.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_b" + ] + }, + { + "type": "gemm", + "args": { + "m": 32768, + "n": 65536, + "k": 1536, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "out": "bf16" + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro-0813/mtp.attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/attn.wq_b", + "deepseek-ai/DeepSeek-V4-Pro/mtp.attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/attn.wq_b", + "sgl-project/DeepSeek-V4-Pro-FP8/mtp.attn.wq_b" + ] + } +] diff --git a/operatorx/testlists/kimi_k3_moe_perf.json b/operatorx/testlists/kimi_k3_moe_perf.json deleted file mode 100644 index a1ed8c060a..0000000000 --- a/operatorx/testlists/kimi_k3_moe_perf.json +++ /dev/null @@ -1,146 +0,0 @@ -[ - { - "type": "moe_gemm", - "name": "Kimi K3 (vLLM benchmark profile)", - "args": { - "num_tokens": 1, - "hidden": 7168, - "intermediate": 3072, - "num_experts": 896, - "top_k": 16, - "dtype_act": "bf16", - "dtype_weight": "bf16", - "expert_parallel_size": 8, - "routed_tensor_parallel_size": 1, - "shared_tensor_parallel_size": 1, - "n_shared_experts": 0, - "expert_distribution": "uniform" - } - }, - { - "type": "moe_gemm", - "name": "Kimi K3 (vLLM benchmark profile)", - "args": { - "num_tokens": 16, - "hidden": 7168, - "intermediate": 3072, - "num_experts": 896, - "top_k": 16, - "dtype_act": "bf16", - "dtype_weight": "bf16", - "expert_parallel_size": 8, - "routed_tensor_parallel_size": 1, - "shared_tensor_parallel_size": 1, - "n_shared_experts": 0, - "expert_distribution": "uniform" - } - }, - { - "type": "moe_gemm", - "name": "Kimi K3 (vLLM benchmark profile)", - "args": { - "num_tokens": 128, - "hidden": 7168, - "intermediate": 3072, - "num_experts": 896, - "top_k": 16, - "dtype_act": "bf16", - "dtype_weight": "bf16", - "expert_parallel_size": 8, - "routed_tensor_parallel_size": 1, - "shared_tensor_parallel_size": 1, - "n_shared_experts": 0, - "expert_distribution": "uniform" - } - }, - { - "type": "moe_gemm", - "name": "Kimi K3 (vLLM benchmark profile)", - "args": { - "num_tokens": 1024, - "hidden": 7168, - "intermediate": 3072, - "num_experts": 896, - "top_k": 16, - "dtype_act": "bf16", - "dtype_weight": "bf16", - "expert_parallel_size": 8, - "routed_tensor_parallel_size": 1, - "shared_tensor_parallel_size": 1, - "n_shared_experts": 0, - "expert_distribution": "uniform" - } - }, - { - "type": "moe_gemm", - "name": "Kimi K3 (vLLM benchmark profile)", - "args": { - "num_tokens": 1, - "hidden": 7168, - "intermediate": 3072, - "num_experts": 896, - "top_k": 16, - "dtype_act": "bf16", - "dtype_weight": "bf16", - "expert_parallel_size": 1, - "routed_tensor_parallel_size": 8, - "shared_tensor_parallel_size": 1, - "n_shared_experts": 0, - "expert_distribution": "uniform" - } - }, - { - "type": "moe_gemm", - "name": "Kimi K3 (vLLM benchmark profile)", - "args": { - "num_tokens": 16, - "hidden": 7168, - "intermediate": 3072, - "num_experts": 896, - "top_k": 16, - "dtype_act": "bf16", - "dtype_weight": "bf16", - "expert_parallel_size": 1, - "routed_tensor_parallel_size": 8, - "shared_tensor_parallel_size": 1, - "n_shared_experts": 0, - "expert_distribution": "uniform" - } - }, - { - "type": "moe_gemm", - "name": "Kimi K3 (vLLM benchmark profile)", - "args": { - "num_tokens": 128, - "hidden": 7168, - "intermediate": 3072, - "num_experts": 896, - "top_k": 16, - "dtype_act": "bf16", - "dtype_weight": "bf16", - "expert_parallel_size": 1, - "routed_tensor_parallel_size": 8, - "shared_tensor_parallel_size": 1, - "n_shared_experts": 0, - "expert_distribution": "uniform" - } - }, - { - "type": "moe_gemm", - "name": "Kimi K3 (vLLM benchmark profile)", - "args": { - "num_tokens": 1024, - "hidden": 7168, - "intermediate": 3072, - "num_experts": 896, - "top_k": 16, - "dtype_act": "bf16", - "dtype_weight": "bf16", - "expert_parallel_size": 1, - "routed_tensor_parallel_size": 8, - "shared_tensor_parallel_size": 1, - "n_shared_experts": 0, - "expert_distribution": "uniform" - } - } -] diff --git a/operatorx/testlists/moe.json b/operatorx/testlists/moe.json deleted file mode 100644 index a1cbbd85a2..0000000000 --- a/operatorx/testlists/moe.json +++ /dev/null @@ -1,30415 +0,0 @@ -[ - { - "type": "moe_forward", - "args": { - "num_tokens": 1243136, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 4, - "tp": 1, - "ep": 4, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 2428, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 4, - "tp": 1, - "ep": 4, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 1243136, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 8, - "tp": 1, - "ep": 8, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 2428, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 8, - "tp": 1, - "ep": 8, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 896000, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 4, - "tp": 1, - "ep": 4, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 1750, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 4, - "tp": 1, - "ep": 4, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 896000, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 8, - "tp": 1, - "ep": 8, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 1750, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 8, - "tp": 1, - "ep": 8, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 6144, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 4, - "tp": 1, - "ep": 4, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 12, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 4, - "tp": 1, - "ep": 4, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 6144, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 8, - "tp": 1, - "ep": 8, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 12, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 8, - "tp": 1, - "ep": 8, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 10240, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 4, - "tp": 1, - "ep": 4, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 20, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 4, - "tp": 1, - "ep": 4, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 15360, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 4, - "tp": 1, - "ep": 4, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 30, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 4, - "tp": 1, - "ep": 4, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 25600, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 4, - "tp": 1, - "ep": 4, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 50, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 4, - "tp": 1, - "ep": 4, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 46080, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 4, - "tp": 1, - "ep": 4, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 90, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 4, - "tp": 1, - "ep": 4, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 92160, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 4, - "tp": 1, - "ep": 4, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 180, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 4, - "tp": 1, - "ep": 4, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 184320, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 4, - "tp": 1, - "ep": 4, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 360, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 4, - "tp": 1, - "ep": 4, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 10240, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 8, - "tp": 1, - "ep": 8, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 20, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 8, - "tp": 1, - "ep": 8, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 15360, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 8, - "tp": 1, - "ep": 8, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 30, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 8, - "tp": 1, - "ep": 8, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 25600, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 8, - "tp": 1, - "ep": 8, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 50, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 8, - "tp": 1, - "ep": 8, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 46080, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 8, - "tp": 1, - "ep": 8, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 90, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 8, - "tp": 1, - "ep": 8, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 92160, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 8, - "tp": 1, - "ep": 8, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 180, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 8, - "tp": 1, - "ep": 8, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 184320, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 8, - "tp": 1, - "ep": 8, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 360, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 8, - "tp": 1, - "ep": 8, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 5087232, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 4, - "tp": 1, - "ep": 4, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 9936, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 4, - "tp": 1, - "ep": 4, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 5087232, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 8, - "tp": 1, - "ep": 8, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 9936, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 8, - "tp": 1, - "ep": 8, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 11120640, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 4, - "tp": 1, - "ep": 4, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 21720, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 4, - "tp": 1, - "ep": 4, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 4194304, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 4, - "tp": 1, - "ep": 4, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 4096, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 4, - "tp": 1, - "ep": 4, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 4194304, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 8, - "tp": 1, - "ep": 8, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 4096, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 8, - "tp": 1, - "ep": 8, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 2244608, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 4, - "tp": 1, - "ep": 4, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 2192, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 4, - "tp": 1, - "ep": 4, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 2244608, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 8, - "tp": 1, - "ep": 8, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 2192, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 8, - "tp": 1, - "ep": 8, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 1397760, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 4, - "tp": 1, - "ep": 4, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 1365, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 4, - "tp": 1, - "ep": 4, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 1397760, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 8, - "tp": 1, - "ep": 8, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 1365, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 8, - "tp": 1, - "ep": 8, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 6, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 4, - "tp": 1, - "ep": 4, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 6, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 8, - "tp": 1, - "ep": 8, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 10, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 4, - "tp": 1, - "ep": 4, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 15, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 4, - "tp": 1, - "ep": 4, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 25, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 4, - "tp": 1, - "ep": 4, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 45, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 4, - "tp": 1, - "ep": 4, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 10, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 8, - "tp": 1, - "ep": 8, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 15, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 8, - "tp": 1, - "ep": 8, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 25, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 8, - "tp": 1, - "ep": 8, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 45, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 8, - "tp": 1, - "ep": 8, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 460800, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 4, - "tp": 1, - "ep": 4, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 450, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 4, - "tp": 1, - "ep": 4, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 460800, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 8, - "tp": 1, - "ep": 8, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 450, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 8, - "tp": 1, - "ep": 8, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 737280, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 4, - "tp": 1, - "ep": 4, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 737280, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 8, - "tp": 1, - "ep": 8, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 540672, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 4, - "tp": 1, - "ep": 4, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 132, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 4, - "tp": 1, - "ep": 4, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 540672, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 8, - "tp": 1, - "ep": 8, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 132, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 8, - "tp": 1, - "ep": 8, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 49152, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 4, - "tp": 1, - "ep": 4, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 49152, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 8, - "tp": 1, - "ep": 8, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 81920, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 4, - "tp": 1, - "ep": 4, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 122880, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 4, - "tp": 1, - "ep": 4, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 245760, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 4, - "tp": 1, - "ep": 4, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 60, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 4, - "tp": 1, - "ep": 4, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 491520, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 4, - "tp": 1, - "ep": 4, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 120, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 4, - "tp": 1, - "ep": 4, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 81920, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 8, - "tp": 1, - "ep": 8, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 122880, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 8, - "tp": 1, - "ep": 8, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 245760, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 8, - "tp": 1, - "ep": 8, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 60, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 8, - "tp": 1, - "ep": 8, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 491520, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 8, - "tp": 1, - "ep": 8, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 120, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 8, - "tp": 1, - "ep": 8, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 4489216, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 4, - "tp": 1, - "ep": 4, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 1096, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 4, - "tp": 1, - "ep": 4, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 4489216, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 8, - "tp": 1, - "ep": 8, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 1096, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 8, - "tp": 1, - "ep": 8, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 8978432, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 4, - "tp": 1, - "ep": 4, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 13852672, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 4, - "tp": 1, - "ep": 4, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 3382, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 4, - "tp": 1, - "ep": 4, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 8978432, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 8, - "tp": 1, - "ep": 8, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 13852672, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 8, - "tp": 1, - "ep": 8, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 3382, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 8, - "tp": 1, - "ep": 8, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 5390336, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 4, - "tp": 1, - "ep": 4, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 1316, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 4, - "tp": 1, - "ep": 4, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 5390336, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 8, - "tp": 1, - "ep": 8, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 1316, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 8, - "tp": 1, - "ep": 8, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 204800, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 4, - "tp": 1, - "ep": 4, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 409600, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 4, - "tp": 1, - "ep": 4, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 819200, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 4, - "tp": 1, - "ep": 4, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 100, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 4, - "tp": 1, - "ep": 4, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 204800, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 8, - "tp": 1, - "ep": 8, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 409600, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 8, - "tp": 1, - "ep": 8, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 819200, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 8, - "tp": 1, - "ep": 8, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 100, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 8, - "tp": 1, - "ep": 8, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 3031040, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 4, - "tp": 1, - "ep": 4, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 370, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 4, - "tp": 1, - "ep": 4, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 3031040, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 8, - "tp": 1, - "ep": 8, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 370, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 8, - "tp": 1, - "ep": 8, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 13156352, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 4, - "tp": 1, - "ep": 4, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 1606, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 4, - "tp": 1, - "ep": 4, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 13156352, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 8, - "tp": 1, - "ep": 8, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 1606, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 8, - "tp": 1, - "ep": 8, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 6856704, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 4, - "tp": 1, - "ep": 4, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 837, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 4, - "tp": 1, - "ep": 4, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 6856704, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 8, - "tp": 1, - "ep": 8, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 837, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 8, - "tp": 1, - "ep": 8, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 18202624, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 4, - "tp": 1, - "ep": 4, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 2222, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 4, - "tp": 1, - "ep": 4, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 18202624, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 8, - "tp": 1, - "ep": 8, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 2222, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 8, - "tp": 1, - "ep": 8, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 8192, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 4, - "tp": 1, - "ep": 4, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 1, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 4, - "tp": 1, - "ep": 4, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 8192, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 8, - "tp": 1, - "ep": 8, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 1, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 8, - "tp": 1, - "ep": 8, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 16384, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 4, - "tp": 1, - "ep": 4, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 2, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 4, - "tp": 1, - "ep": 4, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 16384, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 8, - "tp": 1, - "ep": 8, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 2, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 8, - "tp": 1, - "ep": 8, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 32768, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 4, - "tp": 1, - "ep": 4, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 4, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 4, - "tp": 1, - "ep": 4, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 32768, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 8, - "tp": 1, - "ep": 8, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 4, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 8, - "tp": 1, - "ep": 8, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 65536, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 4, - "tp": 1, - "ep": 4, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 8, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 4, - "tp": 1, - "ep": 4, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 65536, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 8, - "tp": 1, - "ep": 8, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 8, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 8, - "tp": 1, - "ep": 8, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 131072, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 4, - "tp": 1, - "ep": 4, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 16, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 4, - "tp": 1, - "ep": 4, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 131072, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 8, - "tp": 1, - "ep": 8, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 16, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 8, - "tp": 1, - "ep": 8, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 262144, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 4, - "tp": 1, - "ep": 4, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 32, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 4, - "tp": 1, - "ep": 4, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 262144, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 8, - "tp": 1, - "ep": 8, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 32, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 8, - "tp": 1, - "ep": 8, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 8192, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "world_size": 8, - "tp": 1, - "ep": 8, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 16, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "world_size": 8, - "tp": 1, - "ep": 8, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 8192, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "world_size": 8, - "tp": 8, - "ep": 1, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 16, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "world_size": 8, - "tp": 8, - "ep": 1, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 32768, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "world_size": 8, - "tp": 1, - "ep": 8, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 64, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "world_size": 8, - "tp": 1, - "ep": 8, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 32768, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "world_size": 8, - "tp": 8, - "ep": 1, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 64, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "world_size": 8, - "tp": 8, - "ep": 1, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 65536, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "world_size": 8, - "tp": 1, - "ep": 8, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 128, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "world_size": 8, - "tp": 1, - "ep": 8, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 65536, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "world_size": 8, - "tp": 8, - "ep": 1, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 128, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "world_size": 8, - "tp": 8, - "ep": 1, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 262144, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "world_size": 8, - "tp": 1, - "ep": 8, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 512, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "world_size": 8, - "tp": 1, - "ep": 8, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 262144, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "world_size": 8, - "tp": 8, - "ep": 1, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 512, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "world_size": 8, - "tp": 8, - "ep": 1, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 917504, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "world_size": 8, - "tp": 1, - "ep": 8, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 1792, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "world_size": 8, - "tp": 1, - "ep": 8, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 1048576, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "world_size": 8, - "tp": 1, - "ep": 8, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 2048, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "world_size": 8, - "tp": 1, - "ep": 8, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 1212416, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "world_size": 8, - "tp": 1, - "ep": 8, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 2368, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "world_size": 8, - "tp": 1, - "ep": 8, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 1638400, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "world_size": 8, - "tp": 1, - "ep": 8, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 3200, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "world_size": 8, - "tp": 1, - "ep": 8, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 4096, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "world_size": 8, - "tp": 1, - "ep": 8, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 4, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "world_size": 8, - "tp": 1, - "ep": 8, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 4096, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "world_size": 8, - "tp": 8, - "ep": 1, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 4, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "world_size": 8, - "tp": 8, - "ep": 1, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 32, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "world_size": 8, - "tp": 1, - "ep": 8, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 32, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "world_size": 8, - "tp": 8, - "ep": 1, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 131072, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "world_size": 8, - "tp": 1, - "ep": 8, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 131072, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "world_size": 8, - "tp": 8, - "ep": 1, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 1966080, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "world_size": 8, - "tp": 1, - "ep": 8, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 1920, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "world_size": 8, - "tp": 1, - "ep": 8, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 4194304, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "world_size": 8, - "tp": 1, - "ep": 8, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 5275648, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "world_size": 8, - "tp": 1, - "ep": 8, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 5152, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "world_size": 8, - "tp": 1, - "ep": 8, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 393216, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "world_size": 8, - "tp": 8, - "ep": 1, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 96, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "world_size": 8, - "tp": 8, - "ep": 1, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 524288, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "world_size": 8, - "tp": 8, - "ep": 1, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 1835008, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "world_size": 8, - "tp": 8, - "ep": 1, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 448, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "world_size": 8, - "tp": 8, - "ep": 1, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 1835008, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "world_size": 8, - "tp": 1, - "ep": 8, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 448, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "world_size": 8, - "tp": 1, - "ep": 8, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 2359296, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "world_size": 8, - "tp": 8, - "ep": 1, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 576, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "world_size": 8, - "tp": 8, - "ep": 1, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 2359296, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "world_size": 8, - "tp": 1, - "ep": 8, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 576, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "world_size": 8, - "tp": 1, - "ep": 8, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 8912896, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "world_size": 8, - "tp": 8, - "ep": 1, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 2176, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "world_size": 8, - "tp": 8, - "ep": 1, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 8912896, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "world_size": 8, - "tp": 1, - "ep": 8, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 2176, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "world_size": 8, - "tp": 1, - "ep": 8, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 1, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "world_size": 8, - "tp": 8, - "ep": 1, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 1048576, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "world_size": 8, - "tp": 8, - "ep": 1, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 786432, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "world_size": 8, - "tp": 8, - "ep": 1, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 2097152, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "world_size": 8, - "tp": 8, - "ep": 1, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 256, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "world_size": 8, - "tp": 8, - "ep": 1, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 2097152, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "world_size": 8, - "tp": 1, - "ep": 8, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 256, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "world_size": 8, - "tp": 1, - "ep": 8, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 5242880, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "world_size": 8, - "tp": 8, - "ep": 1, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 640, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "world_size": 8, - "tp": 8, - "ep": 1, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 5242880, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "world_size": 8, - "tp": 1, - "ep": 8, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 640, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "world_size": 8, - "tp": 1, - "ep": 8, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 669696, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 2, - "tp": 1, - "ep": 2, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 1308, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 2, - "tp": 1, - "ep": 2, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 669696, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 8, - "tp": 1, - "ep": 8, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 1308, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 8, - "tp": 1, - "ep": 8, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 277504, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 2, - "tp": 1, - "ep": 2, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 542, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 2, - "tp": 1, - "ep": 2, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 277504, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 8, - "tp": 1, - "ep": 8, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 542, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 8, - "tp": 1, - "ep": 8, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 11264, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 2, - "tp": 1, - "ep": 2, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 22, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 2, - "tp": 1, - "ep": 2, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 11264, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 8, - "tp": 1, - "ep": 8, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 22, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 8, - "tp": 1, - "ep": 8, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 10240, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 2, - "tp": 1, - "ep": 2, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 20, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 2, - "tp": 1, - "ep": 2, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 20480, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 2, - "tp": 1, - "ep": 2, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 40, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 2, - "tp": 1, - "ep": 2, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 25600, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 2, - "tp": 1, - "ep": 2, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 50, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 2, - "tp": 1, - "ep": 2, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 61440, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 2, - "tp": 1, - "ep": 2, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 120, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 2, - "tp": 1, - "ep": 2, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 122880, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 2, - "tp": 1, - "ep": 2, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 240, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 2, - "tp": 1, - "ep": 2, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 204800, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 2, - "tp": 1, - "ep": 2, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 400, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 2, - "tp": 1, - "ep": 2, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 20480, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 8, - "tp": 1, - "ep": 8, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 40, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 8, - "tp": 1, - "ep": 8, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 61440, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 8, - "tp": 1, - "ep": 8, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 240, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 8, - "tp": 1, - "ep": 8, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 400, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 8, - "tp": 1, - "ep": 8, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 2398208, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 2, - "tp": 1, - "ep": 2, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 4684, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 2, - "tp": 1, - "ep": 2, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 2398208, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 8, - "tp": 1, - "ep": 8, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 4684, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 8, - "tp": 1, - "ep": 8, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 8815616, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 2, - "tp": 1, - "ep": 2, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 17218, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 2, - "tp": 1, - "ep": 2, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 8815616, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 8, - "tp": 1, - "ep": 8, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 17218, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 8, - "tp": 1, - "ep": 8, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 13236224, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 2, - "tp": 1, - "ep": 2, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 25852, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 2, - "tp": 1, - "ep": 2, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 13236224, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 8, - "tp": 1, - "ep": 8, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 25852, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 8, - "tp": 1, - "ep": 8, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 1204224, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 2, - "tp": 1, - "ep": 2, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 1176, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 2, - "tp": 1, - "ep": 2, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 1204224, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 8, - "tp": 1, - "ep": 8, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 1176, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 8, - "tp": 1, - "ep": 8, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 6144, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 2, - "tp": 1, - "ep": 2, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 6, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 2, - "tp": 1, - "ep": 2, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 5120, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 2, - "tp": 1, - "ep": 2, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 5, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 2, - "tp": 1, - "ep": 2, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 10, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 2, - "tp": 1, - "ep": 2, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 15360, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 2, - "tp": 1, - "ep": 2, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 15, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 2, - "tp": 1, - "ep": 2, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 25, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 2, - "tp": 1, - "ep": 2, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 5120, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 4, - "tp": 1, - "ep": 4, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 5, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 4, - "tp": 1, - "ep": 4, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 60, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 2, - "tp": 1, - "ep": 2, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 112640, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 2, - "tp": 1, - "ep": 2, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 110, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 2, - "tp": 1, - "ep": 2, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 199680, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 2, - "tp": 1, - "ep": 2, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 195, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 2, - "tp": 1, - "ep": 2, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 404480, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 2, - "tp": 1, - "ep": 2, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 395, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 2, - "tp": 1, - "ep": 2, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 112640, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 8, - "tp": 1, - "ep": 8, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 110, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 8, - "tp": 1, - "ep": 8, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 199680, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 8, - "tp": 1, - "ep": 8, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 195, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 8, - "tp": 1, - "ep": 8, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 404480, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 8, - "tp": 1, - "ep": 8, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 395, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 8, - "tp": 1, - "ep": 8, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 4510720, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 2, - "tp": 1, - "ep": 2, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 4405, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 2, - "tp": 1, - "ep": 2, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 4510720, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 8, - "tp": 1, - "ep": 8, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 4405, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 8, - "tp": 1, - "ep": 8, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 8388608, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 2, - "tp": 1, - "ep": 2, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 8192, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 2, - "tp": 1, - "ep": 2, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 8388608, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 8, - "tp": 1, - "ep": 8, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 4721664, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 2, - "tp": 1, - "ep": 2, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 4611, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 2, - "tp": 1, - "ep": 2, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 4721664, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 8, - "tp": 1, - "ep": 8, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 4611, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 8, - "tp": 1, - "ep": 8, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 18006016, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 2, - "tp": 1, - "ep": 2, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 4396, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 2, - "tp": 1, - "ep": 2, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 18006016, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 8, - "tp": 1, - "ep": 8, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 4396, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 8, - "tp": 1, - "ep": 8, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 425984, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 2, - "tp": 1, - "ep": 2, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 104, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 2, - "tp": 1, - "ep": 2, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 425984, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 4, - "tp": 1, - "ep": 4, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 104, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 4, - "tp": 1, - "ep": 4, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 65536, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 2, - "tp": 1, - "ep": 2, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 16, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 2, - "tp": 1, - "ep": 2, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 262144, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 2, - "tp": 1, - "ep": 2, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 64, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 2, - "tp": 1, - "ep": 2, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 64, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 8, - "tp": 1, - "ep": 8, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 1482752, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 2, - "tp": 1, - "ep": 2, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 362, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 2, - "tp": 1, - "ep": 2, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 1482752, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 8, - "tp": 1, - "ep": 8, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 362, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 8, - "tp": 1, - "ep": 8, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 9805824, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 2, - "tp": 1, - "ep": 2, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 2394, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 2, - "tp": 1, - "ep": 2, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 9805824, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 8, - "tp": 1, - "ep": 8, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 2394, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 8, - "tp": 1, - "ep": 8, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 860160, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 2, - "tp": 1, - "ep": 2, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 105, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 2, - "tp": 1, - "ep": 2, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 860160, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 4, - "tp": 1, - "ep": 4, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 105, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 4, - "tp": 1, - "ep": 4, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 516096, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 2, - "tp": 1, - "ep": 2, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 63, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 2, - "tp": 1, - "ep": 2, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 516096, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 8, - "tp": 1, - "ep": 8, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 63, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 8, - "tp": 1, - "ep": 8, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 32768, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 2, - "tp": 1, - "ep": 2, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 4, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 2, - "tp": 1, - "ep": 2, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 98304, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 2, - "tp": 1, - "ep": 2, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 12, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 2, - "tp": 1, - "ep": 2, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 98304, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 4, - "tp": 1, - "ep": 4, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 4825088, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 2, - "tp": 1, - "ep": 2, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 589, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 2, - "tp": 1, - "ep": 2, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 4825088, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 8, - "tp": 1, - "ep": 8, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 589, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 8, - "tp": 1, - "ep": 8, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 8953856, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 2, - "tp": 1, - "ep": 2, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 1093, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 2, - "tp": 1, - "ep": 2, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 8953856, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 8, - "tp": 1, - "ep": 8, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 1093, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 8, - "tp": 1, - "ep": 8, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 16777216, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 2, - "tp": 1, - "ep": 2, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 2048, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 2, - "tp": 1, - "ep": 2, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 16777216, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 8, - "tp": 1, - "ep": 8, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 2048, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 8, - "tp": 1, - "ep": 8, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 10240, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "world_size": 4, - "tp": 4, - "ep": 1, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 20, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "world_size": 4, - "tp": 4, - "ep": 1, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 10240, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "world_size": 8, - "tp": 8, - "ep": 1, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 20, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "world_size": 8, - "tp": 8, - "ep": 1, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 163840, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "world_size": 4, - "tp": 4, - "ep": 1, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 320, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "world_size": 4, - "tp": 4, - "ep": 1, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 163840, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "world_size": 8, - "tp": 8, - "ep": 1, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 320, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "world_size": 8, - "tp": 8, - "ep": 1, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 3145728, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "world_size": 4, - "tp": 4, - "ep": 1, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 6144, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "world_size": 4, - "tp": 4, - "ep": 1, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 3145728, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "world_size": 8, - "tp": 8, - "ep": 1, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 6144, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "world_size": 8, - "tp": 8, - "ep": 1, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 2621440, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "world_size": 4, - "tp": 4, - "ep": 1, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 5120, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "world_size": 4, - "tp": 4, - "ep": 1, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 2621440, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "world_size": 8, - "tp": 1, - "ep": 8, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 5120, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "world_size": 8, - "tp": 1, - "ep": 8, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 737280, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "world_size": 4, - "tp": 4, - "ep": 1, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 1440, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "world_size": 4, - "tp": 4, - "ep": 1, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 737280, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "world_size": 8, - "tp": 1, - "ep": 8, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 1440, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "world_size": 8, - "tp": 1, - "ep": 8, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 11534336, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "world_size": 4, - "tp": 4, - "ep": 1, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 22528, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "world_size": 4, - "tp": 4, - "ep": 1, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 11534336, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "world_size": 8, - "tp": 8, - "ep": 1, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 22528, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "world_size": 8, - "tp": 8, - "ep": 1, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 2162688, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "world_size": 4, - "tp": 4, - "ep": 1, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 2112, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "world_size": 4, - "tp": 4, - "ep": 1, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 2162688, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "world_size": 8, - "tp": 1, - "ep": 8, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 2112, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "world_size": 8, - "tp": 1, - "ep": 8, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 3072, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "world_size": 4, - "tp": 4, - "ep": 1, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 3072, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "world_size": 8, - "tp": 8, - "ep": 1, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 1310720, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "world_size": 4, - "tp": 4, - "ep": 1, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 1280, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "world_size": 4, - "tp": 4, - "ep": 1, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 1310720, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "world_size": 8, - "tp": 8, - "ep": 1, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 1280, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "world_size": 8, - "tp": 8, - "ep": 1, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 12288, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "world_size": 4, - "tp": 4, - "ep": 1, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 12, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "world_size": 4, - "tp": 4, - "ep": 1, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 12288, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "world_size": 8, - "tp": 8, - "ep": 1, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 12, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "world_size": 8, - "tp": 8, - "ep": 1, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 131072, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "world_size": 4, - "tp": 4, - "ep": 1, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 128, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "world_size": 4, - "tp": 4, - "ep": 1, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 393216, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "world_size": 4, - "tp": 4, - "ep": 1, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 384, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "world_size": 4, - "tp": 4, - "ep": 1, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 384, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "world_size": 8, - "tp": 8, - "ep": 1, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 16777216, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "world_size": 4, - "tp": 4, - "ep": 1, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 16384, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "world_size": 4, - "tp": 4, - "ep": 1, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 16777216, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "world_size": 8, - "tp": 8, - "ep": 1, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 16384, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "world_size": 8, - "tp": 8, - "ep": 1, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 327680, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "world_size": 4, - "tp": 4, - "ep": 1, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 80, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "world_size": 4, - "tp": 4, - "ep": 1, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 327680, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "world_size": 8, - "tp": 8, - "ep": 1, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 80, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "world_size": 8, - "tp": 8, - "ep": 1, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 65536, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "world_size": 4, - "tp": 4, - "ep": 1, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 16, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "world_size": 4, - "tp": 4, - "ep": 1, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 40, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "world_size": 4, - "tp": 4, - "ep": 1, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 40, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "world_size": 8, - "tp": 8, - "ep": 1, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 589824, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "world_size": 4, - "tp": 4, - "ep": 1, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 144, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "world_size": 4, - "tp": 4, - "ep": 1, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 589824, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "world_size": 8, - "tp": 8, - "ep": 1, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 144, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "world_size": 8, - "tp": 8, - "ep": 1, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 1179648, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "world_size": 4, - "tp": 4, - "ep": 1, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 288, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "world_size": 4, - "tp": 4, - "ep": 1, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 1179648, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "world_size": 8, - "tp": 8, - "ep": 1, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 288, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "world_size": 8, - "tp": 8, - "ep": 1, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 4194304, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "world_size": 4, - "tp": 4, - "ep": 1, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 1024, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "world_size": 4, - "tp": 4, - "ep": 1, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 4194304, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "world_size": 8, - "tp": 8, - "ep": 1, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 1024, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "world_size": 8, - "tp": 8, - "ep": 1, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 524288, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "world_size": 4, - "tp": 4, - "ep": 1, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 64, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "world_size": 4, - "tp": 4, - "ep": 1, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 2097152, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "world_size": 4, - "tp": 4, - "ep": 1, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 256, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "world_size": 4, - "tp": 4, - "ep": 1, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 512, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "world_size": 4, - "tp": 4, - "ep": 1, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 8806400, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "world_size": 4, - "tp": 4, - "ep": 1, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 1075, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "world_size": 4, - "tp": 4, - "ep": 1, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 8806400, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "world_size": 8, - "tp": 8, - "ep": 1, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 1075, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "world_size": 8, - "tp": 8, - "ep": 1, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 25165824, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "world_size": 4, - "tp": 4, - "ep": 1, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 25165824, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "world_size": 8, - "tp": 1, - "ep": 8, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 3072, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "world_size": 8, - "tp": 1, - "ep": 8, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 64, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 4, - "tp": 1, - "ep": 4, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 128, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 4, - "tp": 1, - "ep": 4, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 128, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 8, - "tp": 1, - "ep": 8, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 524288, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 4, - "tp": 1, - "ep": 4, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 1048576, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 4, - "tp": 1, - "ep": 4, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 196608, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 4, - "tp": 1, - "ep": 4, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 24, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 4, - "tp": 1, - "ep": 4, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 393216, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 4, - "tp": 1, - "ep": 4, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 48, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 4, - "tp": 1, - "ep": 4, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 2097152, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 4, - "tp": 1, - "ep": 4, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 256, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 4, - "tp": 1, - "ep": 4, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 98304, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 8, - "tp": 1, - "ep": 8, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 524288, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 8, - "tp": 1, - "ep": 8, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 1048576, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 8, - "tp": 1, - "ep": 8, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 2097152, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 8, - "tp": 1, - "ep": 8, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 256, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 8, - "tp": 1, - "ep": 8, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 512, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 8, - "tp": 1, - "ep": 8, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 1024, - "hidden": 7168, - "intermediate": 3072, - "num_experts": 384, - "top_k": 6, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 8, - "tp": 8, - "ep": 1, - "n_shared_experts": 1 - }, - "name": "dsv4" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 1, - "hidden": 7168, - "intermediate": 3072, - "num_experts": 384, - "top_k": 6, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 8, - "tp": 8, - "ep": 1, - "n_shared_experts": 1 - }, - "name": "dsv4" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 2048, - "hidden": 7168, - "intermediate": 3072, - "num_experts": 384, - "top_k": 6, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 8, - "tp": 8, - "ep": 1, - "n_shared_experts": 1 - }, - "name": "dsv4" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 2, - "hidden": 7168, - "intermediate": 3072, - "num_experts": 384, - "top_k": 6, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 8, - "tp": 8, - "ep": 1, - "n_shared_experts": 1 - }, - "name": "dsv4" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 4096, - "hidden": 7168, - "intermediate": 3072, - "num_experts": 384, - "top_k": 6, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 8, - "tp": 8, - "ep": 1, - "n_shared_experts": 1 - }, - "name": "dsv4" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 4, - "hidden": 7168, - "intermediate": 3072, - "num_experts": 384, - "top_k": 6, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 8, - "tp": 8, - "ep": 1, - "n_shared_experts": 1 - }, - "name": "dsv4" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 8192, - "hidden": 7168, - "intermediate": 3072, - "num_experts": 384, - "top_k": 6, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 8, - "tp": 8, - "ep": 1, - "n_shared_experts": 1 - }, - "name": "dsv4" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 8, - "hidden": 7168, - "intermediate": 3072, - "num_experts": 384, - "top_k": 6, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 8, - "tp": 8, - "ep": 1, - "n_shared_experts": 1 - }, - "name": "dsv4" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 16384, - "hidden": 7168, - "intermediate": 3072, - "num_experts": 384, - "top_k": 6, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 8, - "tp": 8, - "ep": 1, - "n_shared_experts": 1 - }, - "name": "dsv4" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 16, - "hidden": 7168, - "intermediate": 3072, - "num_experts": 384, - "top_k": 6, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 8, - "tp": 8, - "ep": 1, - "n_shared_experts": 1 - }, - "name": "dsv4" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 32768, - "hidden": 7168, - "intermediate": 3072, - "num_experts": 384, - "top_k": 6, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 8, - "tp": 8, - "ep": 1, - "n_shared_experts": 1 - }, - "name": "dsv4" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 32, - "hidden": 7168, - "intermediate": 3072, - "num_experts": 384, - "top_k": 6, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 8, - "tp": 8, - "ep": 1, - "n_shared_experts": 1 - }, - "name": "dsv4" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 65536, - "hidden": 7168, - "intermediate": 3072, - "num_experts": 384, - "top_k": 6, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 8, - "tp": 1, - "ep": 8, - "n_shared_experts": 1 - }, - "name": "dsv4" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 64, - "hidden": 7168, - "intermediate": 3072, - "num_experts": 384, - "top_k": 6, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 8, - "tp": 1, - "ep": 8, - "n_shared_experts": 1 - }, - "name": "dsv4" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 131072, - "hidden": 7168, - "intermediate": 3072, - "num_experts": 384, - "top_k": 6, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 8, - "tp": 1, - "ep": 8, - "n_shared_experts": 1 - }, - "name": "dsv4" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 128, - "hidden": 7168, - "intermediate": 3072, - "num_experts": 384, - "top_k": 6, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 8, - "tp": 1, - "ep": 8, - "n_shared_experts": 1 - }, - "name": "dsv4" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 262144, - "hidden": 7168, - "intermediate": 3072, - "num_experts": 384, - "top_k": 6, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 8, - "tp": 1, - "ep": 8, - "n_shared_experts": 1 - }, - "name": "dsv4" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 256, - "hidden": 7168, - "intermediate": 3072, - "num_experts": 384, - "top_k": 6, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 8, - "tp": 1, - "ep": 8, - "n_shared_experts": 1 - }, - "name": "dsv4" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 524288, - "hidden": 7168, - "intermediate": 3072, - "num_experts": 384, - "top_k": 6, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 8, - "tp": 1, - "ep": 8, - "n_shared_experts": 1 - }, - "name": "dsv4" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 512, - "hidden": 7168, - "intermediate": 3072, - "num_experts": 384, - "top_k": 6, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 8, - "tp": 1, - "ep": 8, - "n_shared_experts": 1 - }, - "name": "dsv4" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 1048576, - "hidden": 7168, - "intermediate": 3072, - "num_experts": 384, - "top_k": 6, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 8, - "tp": 1, - "ep": 8, - "n_shared_experts": 1 - }, - "name": "dsv4" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 1024, - "hidden": 7168, - "intermediate": 3072, - "num_experts": 384, - "top_k": 6, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 8, - "tp": 1, - "ep": 8, - "n_shared_experts": 1 - }, - "name": "dsv4" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 65536, - "hidden": 7168, - "intermediate": 3072, - "num_experts": 384, - "top_k": 6, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 8, - "tp": 8, - "ep": 1, - "n_shared_experts": 1 - }, - "name": "dsv4" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 131072, - "hidden": 7168, - "intermediate": 3072, - "num_experts": 384, - "top_k": 6, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 8, - "tp": 8, - "ep": 1, - "n_shared_experts": 1 - }, - "name": "dsv4" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 262144, - "hidden": 7168, - "intermediate": 3072, - "num_experts": 384, - "top_k": 6, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 8, - "tp": 8, - "ep": 1, - "n_shared_experts": 1 - }, - "name": "dsv4" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 2097152, - "hidden": 7168, - "intermediate": 3072, - "num_experts": 384, - "top_k": 6, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 8, - "tp": 1, - "ep": 8, - "n_shared_experts": 1 - }, - "name": "dsv4" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 4194304, - "hidden": 7168, - "intermediate": 3072, - "num_experts": 384, - "top_k": 6, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 8, - "tp": 1, - "ep": 8, - "n_shared_experts": 1 - }, - "name": "dsv4" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 64, - "hidden": 7168, - "intermediate": 3072, - "num_experts": 384, - "top_k": 6, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 8, - "tp": 8, - "ep": 1, - "n_shared_experts": 1 - }, - "name": "dsv4" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 2048, - "hidden": 7168, - "intermediate": 3072, - "num_experts": 384, - "top_k": 6, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 8, - "tp": 1, - "ep": 8, - "n_shared_experts": 1 - }, - "name": "dsv4" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 4096, - "hidden": 7168, - "intermediate": 3072, - "num_experts": 384, - "top_k": 6, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 8, - "tp": 1, - "ep": 8, - "n_shared_experts": 1 - }, - "name": "dsv4" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 8388608, - "hidden": 7168, - "intermediate": 3072, - "num_experts": 384, - "top_k": 6, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 8, - "tp": 1, - "ep": 8, - "n_shared_experts": 1 - }, - "name": "dsv4" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 32768, - "hidden": 7168, - "intermediate": 3072, - "num_experts": 384, - "top_k": 6, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 8, - "tp": 1, - "ep": 8, - "n_shared_experts": 1 - }, - "name": "dsv4" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 32, - "hidden": 7168, - "intermediate": 3072, - "num_experts": 384, - "top_k": 6, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 8, - "tp": 1, - "ep": 8, - "n_shared_experts": 1 - }, - "name": "dsv4" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 128, - "hidden": 7168, - "intermediate": 3072, - "num_experts": 384, - "top_k": 6, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 8, - "tp": 8, - "ep": 1, - "n_shared_experts": 1 - }, - "name": "dsv4" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 8192, - "hidden": 7168, - "intermediate": 3072, - "num_experts": 384, - "top_k": 6, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 8, - "tp": 1, - "ep": 8, - "n_shared_experts": 1 - }, - "name": "dsv4" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 4096, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 4, - "tp": 4, - "ep": 1, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 4, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 4, - "tp": 4, - "ep": 1, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 8192, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 4, - "tp": 4, - "ep": 1, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 8, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 4, - "tp": 4, - "ep": 1, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 16384, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 4, - "tp": 4, - "ep": 1, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 16, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 4, - "tp": 4, - "ep": 1, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 4096, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 8, - "tp": 8, - "ep": 1, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 4, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 8, - "tp": 8, - "ep": 1, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 32768, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 4, - "tp": 4, - "ep": 1, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 65536, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 4, - "tp": 4, - "ep": 1, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 131072, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 4, - "tp": 4, - "ep": 1, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 262144, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 4, - "tp": 4, - "ep": 1, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 32, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 4, - "tp": 4, - "ep": 1, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 32768, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 8, - "tp": 8, - "ep": 1, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 512, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 4, - "tp": 1, - "ep": 4, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 8, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 8, - "tp": 8, - "ep": 1, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 131072, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 8, - "tp": 8, - "ep": 1, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 256, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 8, - "tp": 8, - "ep": 1, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 8, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "world_size": 8, - "tp": 8, - "ep": 1, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 32768, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "world_size": 4, - "tp": 4, - "ep": 1, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 4, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "world_size": 4, - "tp": 4, - "ep": 1, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 8, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "world_size": 4, - "tp": 4, - "ep": 1, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 262144, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "world_size": 4, - "tp": 4, - "ep": 1, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 32, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "world_size": 4, - "tp": 4, - "ep": 1, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 32768, - "hidden": 7168, - "intermediate": 3072, - "num_experts": 384, - "top_k": 6, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 4, - "tp": 4, - "ep": 1, - "n_shared_experts": 1 - }, - "name": "dsv4" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 32, - "hidden": 7168, - "intermediate": 3072, - "num_experts": 384, - "top_k": 6, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 4, - "tp": 4, - "ep": 1, - "n_shared_experts": 1 - }, - "name": "dsv4" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 524288, - "hidden": 7168, - "intermediate": 3072, - "num_experts": 384, - "top_k": 6, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 4, - "tp": 1, - "ep": 4, - "n_shared_experts": 1 - }, - "name": "dsv4" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 512, - "hidden": 7168, - "intermediate": 3072, - "num_experts": 384, - "top_k": 6, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 4, - "tp": 1, - "ep": 4, - "n_shared_experts": 1 - }, - "name": "dsv4" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 262144, - "hidden": 7168, - "intermediate": 3072, - "num_experts": 384, - "top_k": 6, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 4, - "tp": 4, - "ep": 1, - "n_shared_experts": 1 - }, - "name": "dsv4" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 4194304, - "hidden": 7168, - "intermediate": 3072, - "num_experts": 384, - "top_k": 6, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 4, - "tp": 1, - "ep": 4, - "n_shared_experts": 1 - }, - "name": "dsv4" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 16777216, - "hidden": 7168, - "intermediate": 3072, - "num_experts": 384, - "top_k": 6, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 8, - "tp": 1, - "ep": 8, - "n_shared_experts": 1 - }, - "name": "dsv4" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 33554432, - "hidden": 7168, - "intermediate": 3072, - "num_experts": 384, - "top_k": 6, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 8, - "tp": 1, - "ep": 8, - "n_shared_experts": 1 - }, - "name": "dsv4" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 4096, - "hidden": 7168, - "intermediate": 3072, - "num_experts": 384, - "top_k": 6, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 4, - "tp": 4, - "ep": 1, - "n_shared_experts": 1 - }, - "name": "dsv4" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 8, - "hidden": 7168, - "intermediate": 3072, - "num_experts": 384, - "top_k": 6, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 4, - "tp": 4, - "ep": 1, - "n_shared_experts": 1 - }, - "name": "dsv4" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 8192, - "hidden": 7168, - "intermediate": 3072, - "num_experts": 384, - "top_k": 6, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 4, - "tp": 4, - "ep": 1, - "n_shared_experts": 1 - }, - "name": "dsv4" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 16, - "hidden": 7168, - "intermediate": 3072, - "num_experts": 384, - "top_k": 6, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 4, - "tp": 4, - "ep": 1, - "n_shared_experts": 1 - }, - "name": "dsv4" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 16384, - "hidden": 7168, - "intermediate": 3072, - "num_experts": 384, - "top_k": 6, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 4, - "tp": 4, - "ep": 1, - "n_shared_experts": 1 - }, - "name": "dsv4" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 64, - "hidden": 7168, - "intermediate": 3072, - "num_experts": 384, - "top_k": 6, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 4, - "tp": 4, - "ep": 1, - "n_shared_experts": 1 - }, - "name": "dsv4" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 65536, - "hidden": 7168, - "intermediate": 3072, - "num_experts": 384, - "top_k": 6, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 4, - "tp": 4, - "ep": 1, - "n_shared_experts": 1 - }, - "name": "dsv4" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 131072, - "hidden": 7168, - "intermediate": 3072, - "num_experts": 384, - "top_k": 6, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 4, - "tp": 4, - "ep": 1, - "n_shared_experts": 1 - }, - "name": "dsv4" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 4096, - "hidden": 4096, - "intermediate": 1024, - "num_experts": 512, - "top_k": 10, - "dtype_act": "bf16", - "dtype_weight": "bf16", - "world_size": 8, - "tp": 8, - "ep": 1, - "n_shared_experts": 1 - }, - "name": "qwen3.5" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 4, - "hidden": 4096, - "intermediate": 1024, - "num_experts": 512, - "top_k": 10, - "dtype_act": "bf16", - "dtype_weight": "bf16", - "world_size": 8, - "tp": 8, - "ep": 1, - "n_shared_experts": 1 - }, - "name": "qwen3.5" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 8192, - "hidden": 4096, - "intermediate": 1024, - "num_experts": 512, - "top_k": 10, - "dtype_act": "bf16", - "dtype_weight": "bf16", - "world_size": 8, - "tp": 8, - "ep": 1, - "n_shared_experts": 1 - }, - "name": "qwen3.5" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 8, - "hidden": 4096, - "intermediate": 1024, - "num_experts": 512, - "top_k": 10, - "dtype_act": "bf16", - "dtype_weight": "bf16", - "world_size": 8, - "tp": 8, - "ep": 1, - "n_shared_experts": 1 - }, - "name": "qwen3.5" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 16384, - "hidden": 4096, - "intermediate": 1024, - "num_experts": 512, - "top_k": 10, - "dtype_act": "bf16", - "dtype_weight": "bf16", - "world_size": 8, - "tp": 8, - "ep": 1, - "n_shared_experts": 1 - }, - "name": "qwen3.5" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 16, - "hidden": 4096, - "intermediate": 1024, - "num_experts": 512, - "top_k": 10, - "dtype_act": "bf16", - "dtype_weight": "bf16", - "world_size": 8, - "tp": 8, - "ep": 1, - "n_shared_experts": 1 - }, - "name": "qwen3.5" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 32768, - "hidden": 4096, - "intermediate": 1024, - "num_experts": 512, - "top_k": 10, - "dtype_act": "bf16", - "dtype_weight": "bf16", - "world_size": 8, - "tp": 8, - "ep": 1, - "n_shared_experts": 1 - }, - "name": "qwen3.5" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 32, - "hidden": 4096, - "intermediate": 1024, - "num_experts": 512, - "top_k": 10, - "dtype_act": "bf16", - "dtype_weight": "bf16", - "world_size": 8, - "tp": 8, - "ep": 1, - "n_shared_experts": 1 - }, - "name": "qwen3.5" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 65536, - "hidden": 4096, - "intermediate": 1024, - "num_experts": 512, - "top_k": 10, - "dtype_act": "bf16", - "dtype_weight": "bf16", - "world_size": 8, - "tp": 8, - "ep": 1, - "n_shared_experts": 1 - }, - "name": "qwen3.5" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 64, - "hidden": 4096, - "intermediate": 1024, - "num_experts": 512, - "top_k": 10, - "dtype_act": "bf16", - "dtype_weight": "bf16", - "world_size": 8, - "tp": 8, - "ep": 1, - "n_shared_experts": 1 - }, - "name": "qwen3.5" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 131072, - "hidden": 4096, - "intermediate": 1024, - "num_experts": 512, - "top_k": 10, - "dtype_act": "bf16", - "dtype_weight": "bf16", - "world_size": 8, - "tp": 8, - "ep": 1, - "n_shared_experts": 1 - }, - "name": "qwen3.5" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 262144, - "hidden": 4096, - "intermediate": 1024, - "num_experts": 512, - "top_k": 10, - "dtype_act": "bf16", - "dtype_weight": "bf16", - "world_size": 8, - "tp": 8, - "ep": 1, - "n_shared_experts": 1 - }, - "name": "qwen3.5" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 524288, - "hidden": 4096, - "intermediate": 1024, - "num_experts": 512, - "top_k": 10, - "dtype_act": "bf16", - "dtype_weight": "bf16", - "world_size": 8, - "tp": 8, - "ep": 1, - "n_shared_experts": 1 - }, - "name": "qwen3.5" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 128, - "hidden": 4096, - "intermediate": 1024, - "num_experts": 512, - "top_k": 10, - "dtype_act": "bf16", - "dtype_weight": "bf16", - "world_size": 8, - "tp": 8, - "ep": 1, - "n_shared_experts": 1 - }, - "name": "qwen3.5" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 4096, - "hidden": 4096, - "intermediate": 1024, - "num_experts": 512, - "top_k": 10, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "world_size": 8, - "tp": 8, - "ep": 1, - "n_shared_experts": 1 - }, - "name": "qwen3.5" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 4, - "hidden": 4096, - "intermediate": 1024, - "num_experts": 512, - "top_k": 10, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "world_size": 8, - "tp": 8, - "ep": 1, - "n_shared_experts": 1 - }, - "name": "qwen3.5" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 4096, - "hidden": 4096, - "intermediate": 1024, - "num_experts": 512, - "top_k": 10, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "world_size": 4, - "tp": 4, - "ep": 1, - "n_shared_experts": 1 - }, - "name": "qwen3.5" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 4, - "hidden": 4096, - "intermediate": 1024, - "num_experts": 512, - "top_k": 10, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "world_size": 4, - "tp": 4, - "ep": 1, - "n_shared_experts": 1 - }, - "name": "qwen3.5" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 8192, - "hidden": 4096, - "intermediate": 1024, - "num_experts": 512, - "top_k": 10, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "world_size": 4, - "tp": 4, - "ep": 1, - "n_shared_experts": 1 - }, - "name": "qwen3.5" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 8, - "hidden": 4096, - "intermediate": 1024, - "num_experts": 512, - "top_k": 10, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "world_size": 4, - "tp": 4, - "ep": 1, - "n_shared_experts": 1 - }, - "name": "qwen3.5" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 16384, - "hidden": 4096, - "intermediate": 1024, - "num_experts": 512, - "top_k": 10, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "world_size": 4, - "tp": 4, - "ep": 1, - "n_shared_experts": 1 - }, - "name": "qwen3.5" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 16, - "hidden": 4096, - "intermediate": 1024, - "num_experts": 512, - "top_k": 10, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "world_size": 4, - "tp": 4, - "ep": 1, - "n_shared_experts": 1 - }, - "name": "qwen3.5" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 32768, - "hidden": 4096, - "intermediate": 1024, - "num_experts": 512, - "top_k": 10, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "world_size": 4, - "tp": 4, - "ep": 1, - "n_shared_experts": 1 - }, - "name": "qwen3.5" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 32, - "hidden": 4096, - "intermediate": 1024, - "num_experts": 512, - "top_k": 10, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "world_size": 4, - "tp": 4, - "ep": 1, - "n_shared_experts": 1 - }, - "name": "qwen3.5" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 65536, - "hidden": 4096, - "intermediate": 1024, - "num_experts": 512, - "top_k": 10, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "world_size": 4, - "tp": 4, - "ep": 1, - "n_shared_experts": 1 - }, - "name": "qwen3.5" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 64, - "hidden": 4096, - "intermediate": 1024, - "num_experts": 512, - "top_k": 10, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "world_size": 4, - "tp": 4, - "ep": 1, - "n_shared_experts": 1 - }, - "name": "qwen3.5" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 131072, - "hidden": 4096, - "intermediate": 1024, - "num_experts": 512, - "top_k": 10, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "world_size": 4, - "tp": 4, - "ep": 1, - "n_shared_experts": 1 - }, - "name": "qwen3.5" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 128, - "hidden": 4096, - "intermediate": 1024, - "num_experts": 512, - "top_k": 10, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "world_size": 4, - "tp": 4, - "ep": 1, - "n_shared_experts": 1 - }, - "name": "qwen3.5" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 262144, - "hidden": 4096, - "intermediate": 1024, - "num_experts": 512, - "top_k": 10, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "world_size": 4, - "tp": 4, - "ep": 1, - "n_shared_experts": 1 - }, - "name": "qwen3.5" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 256, - "hidden": 4096, - "intermediate": 1024, - "num_experts": 512, - "top_k": 10, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "world_size": 4, - "tp": 4, - "ep": 1, - "n_shared_experts": 1 - }, - "name": "qwen3.5" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 32768, - "hidden": 4096, - "intermediate": 1024, - "num_experts": 512, - "top_k": 10, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "world_size": 8, - "tp": 8, - "ep": 1, - "n_shared_experts": 1 - }, - "name": "qwen3.5" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 524288, - "hidden": 4096, - "intermediate": 1024, - "num_experts": 512, - "top_k": 10, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "world_size": 4, - "tp": 4, - "ep": 1, - "n_shared_experts": 1 - }, - "name": "qwen3.5" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 1048576, - "hidden": 4096, - "intermediate": 1024, - "num_experts": 512, - "top_k": 10, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "world_size": 4, - "tp": 4, - "ep": 1, - "n_shared_experts": 1 - }, - "name": "qwen3.5" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 2097152, - "hidden": 4096, - "intermediate": 1024, - "num_experts": 512, - "top_k": 10, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "world_size": 4, - "tp": 4, - "ep": 1, - "n_shared_experts": 1 - }, - "name": "qwen3.5" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 4096, - "hidden": 4096, - "intermediate": 1024, - "num_experts": 512, - "top_k": 10, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 4, - "tp": 4, - "ep": 1, - "n_shared_experts": 1 - }, - "name": "qwen3.5" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 4, - "hidden": 4096, - "intermediate": 1024, - "num_experts": 512, - "top_k": 10, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 4, - "tp": 4, - "ep": 1, - "n_shared_experts": 1 - }, - "name": "qwen3.5" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 4096, - "hidden": 4096, - "intermediate": 1024, - "num_experts": 512, - "top_k": 10, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 2, - "tp": 2, - "ep": 1, - "n_shared_experts": 1 - }, - "name": "qwen3.5" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 4, - "hidden": 4096, - "intermediate": 1024, - "num_experts": 512, - "top_k": 10, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 2, - "tp": 2, - "ep": 1, - "n_shared_experts": 1 - }, - "name": "qwen3.5" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 8192, - "hidden": 4096, - "intermediate": 1024, - "num_experts": 512, - "top_k": 10, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 2, - "tp": 2, - "ep": 1, - "n_shared_experts": 1 - }, - "name": "qwen3.5" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 8, - "hidden": 4096, - "intermediate": 1024, - "num_experts": 512, - "top_k": 10, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 2, - "tp": 2, - "ep": 1, - "n_shared_experts": 1 - }, - "name": "qwen3.5" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 16384, - "hidden": 4096, - "intermediate": 1024, - "num_experts": 512, - "top_k": 10, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 2, - "tp": 2, - "ep": 1, - "n_shared_experts": 1 - }, - "name": "qwen3.5" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 16, - "hidden": 4096, - "intermediate": 1024, - "num_experts": 512, - "top_k": 10, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 2, - "tp": 2, - "ep": 1, - "n_shared_experts": 1 - }, - "name": "qwen3.5" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 32768, - "hidden": 4096, - "intermediate": 1024, - "num_experts": 512, - "top_k": 10, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 2, - "tp": 2, - "ep": 1, - "n_shared_experts": 1 - }, - "name": "qwen3.5" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 32, - "hidden": 4096, - "intermediate": 1024, - "num_experts": 512, - "top_k": 10, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 2, - "tp": 2, - "ep": 1, - "n_shared_experts": 1 - }, - "name": "qwen3.5" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 65536, - "hidden": 4096, - "intermediate": 1024, - "num_experts": 512, - "top_k": 10, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 2, - "tp": 2, - "ep": 1, - "n_shared_experts": 1 - }, - "name": "qwen3.5" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 64, - "hidden": 4096, - "intermediate": 1024, - "num_experts": 512, - "top_k": 10, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 2, - "tp": 2, - "ep": 1, - "n_shared_experts": 1 - }, - "name": "qwen3.5" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 131072, - "hidden": 4096, - "intermediate": 1024, - "num_experts": 512, - "top_k": 10, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 2, - "tp": 2, - "ep": 1, - "n_shared_experts": 1 - }, - "name": "qwen3.5" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 128, - "hidden": 4096, - "intermediate": 1024, - "num_experts": 512, - "top_k": 10, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 2, - "tp": 2, - "ep": 1, - "n_shared_experts": 1 - }, - "name": "qwen3.5" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 32768, - "hidden": 4096, - "intermediate": 1024, - "num_experts": 512, - "top_k": 10, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 4, - "tp": 4, - "ep": 1, - "n_shared_experts": 1 - }, - "name": "qwen3.5" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 262144, - "hidden": 4096, - "intermediate": 1024, - "num_experts": 512, - "top_k": 10, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 2, - "tp": 2, - "ep": 1, - "n_shared_experts": 1 - }, - "name": "qwen3.5" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 524288, - "hidden": 4096, - "intermediate": 1024, - "num_experts": 512, - "top_k": 10, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 2, - "tp": 2, - "ep": 1, - "n_shared_experts": 1 - }, - "name": "qwen3.5" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 1048576, - "hidden": 4096, - "intermediate": 1024, - "num_experts": 512, - "top_k": 10, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 2, - "tp": 2, - "ep": 1, - "n_shared_experts": 1 - }, - "name": "qwen3.5" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 8, - "hidden": 4096, - "intermediate": 1024, - "num_experts": 512, - "top_k": 10, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 4, - "tp": 4, - "ep": 1, - "n_shared_experts": 1 - }, - "name": "qwen3.5" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 4096, - "hidden": 6144, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "world_size": 8, - "tp": 8, - "ep": 1, - "n_shared_experts": 1 - }, - "name": "glm5" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 4, - "hidden": 6144, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "world_size": 8, - "tp": 8, - "ep": 1, - "n_shared_experts": 1 - }, - "name": "glm5" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 8192, - "hidden": 6144, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "world_size": 8, - "tp": 8, - "ep": 1, - "n_shared_experts": 1 - }, - "name": "glm5" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 8, - "hidden": 6144, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "world_size": 8, - "tp": 8, - "ep": 1, - "n_shared_experts": 1 - }, - "name": "glm5" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 16384, - "hidden": 6144, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "world_size": 8, - "tp": 8, - "ep": 1, - "n_shared_experts": 1 - }, - "name": "glm5" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 16, - "hidden": 6144, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "world_size": 8, - "tp": 8, - "ep": 1, - "n_shared_experts": 1 - }, - "name": "glm5" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 32768, - "hidden": 6144, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "world_size": 8, - "tp": 8, - "ep": 1, - "n_shared_experts": 1 - }, - "name": "glm5" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 32, - "hidden": 6144, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "world_size": 8, - "tp": 8, - "ep": 1, - "n_shared_experts": 1 - }, - "name": "glm5" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 65536, - "hidden": 6144, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "world_size": 8, - "tp": 8, - "ep": 1, - "n_shared_experts": 1 - }, - "name": "glm5" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 64, - "hidden": 6144, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "world_size": 8, - "tp": 8, - "ep": 1, - "n_shared_experts": 1 - }, - "name": "glm5" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 131072, - "hidden": 6144, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "world_size": 8, - "tp": 8, - "ep": 1, - "n_shared_experts": 1 - }, - "name": "glm5" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 128, - "hidden": 6144, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "world_size": 8, - "tp": 8, - "ep": 1, - "n_shared_experts": 1 - }, - "name": "glm5" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 262144, - "hidden": 6144, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "world_size": 8, - "tp": 8, - "ep": 1, - "n_shared_experts": 1 - }, - "name": "glm5" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 256, - "hidden": 6144, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "world_size": 8, - "tp": 8, - "ep": 1, - "n_shared_experts": 1 - }, - "name": "glm5" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 524288, - "hidden": 6144, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "world_size": 8, - "tp": 8, - "ep": 1, - "n_shared_experts": 1 - }, - "name": "glm5" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 1048576, - "hidden": 6144, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "world_size": 8, - "tp": 8, - "ep": 1, - "n_shared_experts": 1 - }, - "name": "glm5" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 2097152, - "hidden": 6144, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "world_size": 8, - "tp": 8, - "ep": 1, - "n_shared_experts": 1 - }, - "name": "glm5" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 512, - "hidden": 6144, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "world_size": 8, - "tp": 8, - "ep": 1, - "n_shared_experts": 1 - }, - "name": "glm5" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 4096, - "hidden": 6144, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 8, - "tp": 8, - "ep": 1, - "n_shared_experts": 1 - }, - "name": "glm5" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 4, - "hidden": 6144, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 8, - "tp": 8, - "ep": 1, - "n_shared_experts": 1 - }, - "name": "glm5" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 4096, - "hidden": 6144, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 4, - "tp": 4, - "ep": 1, - "n_shared_experts": 1 - }, - "name": "glm5" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 4, - "hidden": 6144, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 4, - "tp": 4, - "ep": 1, - "n_shared_experts": 1 - }, - "name": "glm5" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 8192, - "hidden": 6144, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 4, - "tp": 4, - "ep": 1, - "n_shared_experts": 1 - }, - "name": "glm5" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 8, - "hidden": 6144, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 4, - "tp": 4, - "ep": 1, - "n_shared_experts": 1 - }, - "name": "glm5" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 16384, - "hidden": 6144, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 4, - "tp": 4, - "ep": 1, - "n_shared_experts": 1 - }, - "name": "glm5" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 16, - "hidden": 6144, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 4, - "tp": 4, - "ep": 1, - "n_shared_experts": 1 - }, - "name": "glm5" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 32768, - "hidden": 6144, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 4, - "tp": 4, - "ep": 1, - "n_shared_experts": 1 - }, - "name": "glm5" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 32, - "hidden": 6144, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 4, - "tp": 4, - "ep": 1, - "n_shared_experts": 1 - }, - "name": "glm5" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 65536, - "hidden": 6144, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 4, - "tp": 4, - "ep": 1, - "n_shared_experts": 1 - }, - "name": "glm5" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 64, - "hidden": 6144, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 4, - "tp": 4, - "ep": 1, - "n_shared_experts": 1 - }, - "name": "glm5" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 131072, - "hidden": 6144, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 4, - "tp": 4, - "ep": 1, - "n_shared_experts": 1 - }, - "name": "glm5" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 128, - "hidden": 6144, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 4, - "tp": 4, - "ep": 1, - "n_shared_experts": 1 - }, - "name": "glm5" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 262144, - "hidden": 6144, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 4, - "tp": 4, - "ep": 1, - "n_shared_experts": 1 - }, - "name": "glm5" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 256, - "hidden": 6144, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 4, - "tp": 4, - "ep": 1, - "n_shared_experts": 1 - }, - "name": "glm5" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 32768, - "hidden": 6144, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 8, - "tp": 8, - "ep": 1, - "n_shared_experts": 1 - }, - "name": "glm5" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 524288, - "hidden": 6144, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 4, - "tp": 4, - "ep": 1, - "n_shared_experts": 1 - }, - "name": "glm5" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 1048576, - "hidden": 6144, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 4, - "tp": 4, - "ep": 1, - "n_shared_experts": 1 - }, - "name": "glm5" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 2097152, - "hidden": 6144, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 4, - "tp": 4, - "ep": 1, - "n_shared_experts": 1 - }, - "name": "glm5" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 8, - "hidden": 6144, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 8, - "tp": 8, - "ep": 1, - "n_shared_experts": 1 - }, - "name": "glm5" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 512, - "hidden": 6144, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 4, - "tp": 4, - "ep": 1, - "n_shared_experts": 1 - }, - "name": "glm5" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 8, - "hidden": 4096, - "intermediate": 1024, - "num_experts": 512, - "top_k": 10, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "world_size": 8, - "tp": 8, - "ep": 1, - "n_shared_experts": 1 - }, - "name": "qwen3.5" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 512, - "hidden": 4096, - "intermediate": 1024, - "num_experts": 512, - "top_k": 10, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "world_size": 4, - "tp": 4, - "ep": 1, - "n_shared_experts": 1 - }, - "name": "qwen3.5" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 8192, - "hidden": 4096, - "intermediate": 1024, - "num_experts": 512, - "top_k": 10, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 4, - "tp": 4, - "ep": 1, - "n_shared_experts": 1 - }, - "name": "qwen3.5" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 16384, - "hidden": 4096, - "intermediate": 1024, - "num_experts": 512, - "top_k": 10, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 4, - "tp": 4, - "ep": 1, - "n_shared_experts": 1 - }, - "name": "qwen3.5" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 16, - "hidden": 4096, - "intermediate": 1024, - "num_experts": 512, - "top_k": 10, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 4, - "tp": 4, - "ep": 1, - "n_shared_experts": 1 - }, - "name": "qwen3.5" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 32, - "hidden": 4096, - "intermediate": 1024, - "num_experts": 512, - "top_k": 10, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 4, - "tp": 4, - "ep": 1, - "n_shared_experts": 1 - }, - "name": "qwen3.5" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 65536, - "hidden": 4096, - "intermediate": 1024, - "num_experts": 512, - "top_k": 10, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 4, - "tp": 4, - "ep": 1, - "n_shared_experts": 1 - }, - "name": "qwen3.5" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 64, - "hidden": 4096, - "intermediate": 1024, - "num_experts": 512, - "top_k": 10, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 4, - "tp": 4, - "ep": 1, - "n_shared_experts": 1 - }, - "name": "qwen3.5" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 131072, - "hidden": 4096, - "intermediate": 1024, - "num_experts": 512, - "top_k": 10, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 4, - "tp": 4, - "ep": 1, - "n_shared_experts": 1 - }, - "name": "qwen3.5" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 128, - "hidden": 4096, - "intermediate": 1024, - "num_experts": 512, - "top_k": 10, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 4, - "tp": 4, - "ep": 1, - "n_shared_experts": 1 - }, - "name": "qwen3.5" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 4096, - "hidden": 4096, - "intermediate": 1024, - "num_experts": 512, - "top_k": 10, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 2, - "tp": 1, - "ep": 2, - "n_shared_experts": 1 - }, - "name": "qwen3.5" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 4, - "hidden": 4096, - "intermediate": 1024, - "num_experts": 512, - "top_k": 10, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 2, - "tp": 1, - "ep": 2, - "n_shared_experts": 1 - }, - "name": "qwen3.5" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 8192, - "hidden": 4096, - "intermediate": 1024, - "num_experts": 512, - "top_k": 10, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 2, - "tp": 1, - "ep": 2, - "n_shared_experts": 1 - }, - "name": "qwen3.5" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 8, - "hidden": 4096, - "intermediate": 1024, - "num_experts": 512, - "top_k": 10, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 2, - "tp": 1, - "ep": 2, - "n_shared_experts": 1 - }, - "name": "qwen3.5" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 16384, - "hidden": 4096, - "intermediate": 1024, - "num_experts": 512, - "top_k": 10, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 2, - "tp": 1, - "ep": 2, - "n_shared_experts": 1 - }, - "name": "qwen3.5" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 16, - "hidden": 4096, - "intermediate": 1024, - "num_experts": 512, - "top_k": 10, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 2, - "tp": 1, - "ep": 2, - "n_shared_experts": 1 - }, - "name": "qwen3.5" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 32768, - "hidden": 4096, - "intermediate": 1024, - "num_experts": 512, - "top_k": 10, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 2, - "tp": 1, - "ep": 2, - "n_shared_experts": 1 - }, - "name": "qwen3.5" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 32, - "hidden": 4096, - "intermediate": 1024, - "num_experts": 512, - "top_k": 10, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 2, - "tp": 1, - "ep": 2, - "n_shared_experts": 1 - }, - "name": "qwen3.5" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 65536, - "hidden": 4096, - "intermediate": 1024, - "num_experts": 512, - "top_k": 10, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 2, - "tp": 1, - "ep": 2, - "n_shared_experts": 1 - }, - "name": "qwen3.5" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 64, - "hidden": 4096, - "intermediate": 1024, - "num_experts": 512, - "top_k": 10, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 2, - "tp": 1, - "ep": 2, - "n_shared_experts": 1 - }, - "name": "qwen3.5" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 131072, - "hidden": 4096, - "intermediate": 1024, - "num_experts": 512, - "top_k": 10, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 2, - "tp": 1, - "ep": 2, - "n_shared_experts": 1 - }, - "name": "qwen3.5" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 128, - "hidden": 4096, - "intermediate": 1024, - "num_experts": 512, - "top_k": 10, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 2, - "tp": 1, - "ep": 2, - "n_shared_experts": 1 - }, - "name": "qwen3.5" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 262144, - "hidden": 4096, - "intermediate": 1024, - "num_experts": 512, - "top_k": 10, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 4, - "tp": 4, - "ep": 1, - "n_shared_experts": 1 - }, - "name": "qwen3.5" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 524288, - "hidden": 4096, - "intermediate": 1024, - "num_experts": 512, - "top_k": 10, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 4, - "tp": 4, - "ep": 1, - "n_shared_experts": 1 - }, - "name": "qwen3.5" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 1048576, - "hidden": 4096, - "intermediate": 1024, - "num_experts": 512, - "top_k": 10, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 4, - "tp": 4, - "ep": 1, - "n_shared_experts": 1 - }, - "name": "qwen3.5" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 262144, - "hidden": 4096, - "intermediate": 1024, - "num_experts": 512, - "top_k": 10, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 2, - "tp": 1, - "ep": 2, - "n_shared_experts": 1 - }, - "name": "qwen3.5" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 524288, - "hidden": 4096, - "intermediate": 1024, - "num_experts": 512, - "top_k": 10, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 2, - "tp": 1, - "ep": 2, - "n_shared_experts": 1 - }, - "name": "qwen3.5" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 1048576, - "hidden": 4096, - "intermediate": 1024, - "num_experts": 512, - "top_k": 10, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 2, - "tp": 1, - "ep": 2, - "n_shared_experts": 1 - }, - "name": "qwen3.5" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 256, - "hidden": 4096, - "intermediate": 1024, - "num_experts": 512, - "top_k": 10, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 4, - "tp": 4, - "ep": 1, - "n_shared_experts": 1 - }, - "name": "qwen3.5" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 256, - "hidden": 4096, - "intermediate": 1024, - "num_experts": 512, - "top_k": 10, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 2, - "tp": 1, - "ep": 2, - "n_shared_experts": 1 - }, - "name": "qwen3.5" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 4096, - "hidden": 4096, - "intermediate": 1024, - "num_experts": 512, - "top_k": 10, - "dtype_act": "bf16", - "dtype_weight": "bf16", - "world_size": 4, - "tp": 4, - "ep": 1, - "n_shared_experts": 1 - }, - "name": "qwen3.5" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 4, - "hidden": 4096, - "intermediate": 1024, - "num_experts": 512, - "top_k": 10, - "dtype_act": "bf16", - "dtype_weight": "bf16", - "world_size": 4, - "tp": 4, - "ep": 1, - "n_shared_experts": 1 - }, - "name": "qwen3.5" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 8192, - "hidden": 4096, - "intermediate": 1024, - "num_experts": 512, - "top_k": 10, - "dtype_act": "bf16", - "dtype_weight": "bf16", - "world_size": 4, - "tp": 4, - "ep": 1, - "n_shared_experts": 1 - }, - "name": "qwen3.5" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 8, - "hidden": 4096, - "intermediate": 1024, - "num_experts": 512, - "top_k": 10, - "dtype_act": "bf16", - "dtype_weight": "bf16", - "world_size": 4, - "tp": 4, - "ep": 1, - "n_shared_experts": 1 - }, - "name": "qwen3.5" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 16384, - "hidden": 4096, - "intermediate": 1024, - "num_experts": 512, - "top_k": 10, - "dtype_act": "bf16", - "dtype_weight": "bf16", - "world_size": 4, - "tp": 4, - "ep": 1, - "n_shared_experts": 1 - }, - "name": "qwen3.5" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 16, - "hidden": 4096, - "intermediate": 1024, - "num_experts": 512, - "top_k": 10, - "dtype_act": "bf16", - "dtype_weight": "bf16", - "world_size": 4, - "tp": 4, - "ep": 1, - "n_shared_experts": 1 - }, - "name": "qwen3.5" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 32768, - "hidden": 4096, - "intermediate": 1024, - "num_experts": 512, - "top_k": 10, - "dtype_act": "bf16", - "dtype_weight": "bf16", - "world_size": 4, - "tp": 4, - "ep": 1, - "n_shared_experts": 1 - }, - "name": "qwen3.5" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 32, - "hidden": 4096, - "intermediate": 1024, - "num_experts": 512, - "top_k": 10, - "dtype_act": "bf16", - "dtype_weight": "bf16", - "world_size": 4, - "tp": 4, - "ep": 1, - "n_shared_experts": 1 - }, - "name": "qwen3.5" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 65536, - "hidden": 4096, - "intermediate": 1024, - "num_experts": 512, - "top_k": 10, - "dtype_act": "bf16", - "dtype_weight": "bf16", - "world_size": 4, - "tp": 4, - "ep": 1, - "n_shared_experts": 1 - }, - "name": "qwen3.5" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 64, - "hidden": 4096, - "intermediate": 1024, - "num_experts": 512, - "top_k": 10, - "dtype_act": "bf16", - "dtype_weight": "bf16", - "world_size": 4, - "tp": 4, - "ep": 1, - "n_shared_experts": 1 - }, - "name": "qwen3.5" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 131072, - "hidden": 4096, - "intermediate": 1024, - "num_experts": 512, - "top_k": 10, - "dtype_act": "bf16", - "dtype_weight": "bf16", - "world_size": 4, - "tp": 4, - "ep": 1, - "n_shared_experts": 1 - }, - "name": "qwen3.5" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 262144, - "hidden": 4096, - "intermediate": 1024, - "num_experts": 512, - "top_k": 10, - "dtype_act": "bf16", - "dtype_weight": "bf16", - "world_size": 4, - "tp": 4, - "ep": 1, - "n_shared_experts": 1 - }, - "name": "qwen3.5" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 524288, - "hidden": 4096, - "intermediate": 1024, - "num_experts": 512, - "top_k": 10, - "dtype_act": "bf16", - "dtype_weight": "bf16", - "world_size": 4, - "tp": 4, - "ep": 1, - "n_shared_experts": 1 - }, - "name": "qwen3.5" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 128, - "hidden": 4096, - "intermediate": 1024, - "num_experts": 512, - "top_k": 10, - "dtype_act": "bf16", - "dtype_weight": "bf16", - "world_size": 4, - "tp": 4, - "ep": 1, - "n_shared_experts": 1 - }, - "name": "qwen3.5" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 4096, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 384, - "top_k": 8, - "dtype_act": "int4", - "dtype_weight": "int4", - "world_size": 8, - "tp": 8, - "ep": 1, - "n_shared_experts": 1 - }, - "name": "kimik2.5" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 4, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 384, - "top_k": 8, - "dtype_act": "int4", - "dtype_weight": "int4", - "world_size": 8, - "tp": 8, - "ep": 1, - "n_shared_experts": 1 - }, - "name": "kimik2.5" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 8192, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 384, - "top_k": 8, - "dtype_act": "int4", - "dtype_weight": "int4", - "world_size": 8, - "tp": 8, - "ep": 1, - "n_shared_experts": 1 - }, - "name": "kimik2.5" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 8, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 384, - "top_k": 8, - "dtype_act": "int4", - "dtype_weight": "int4", - "world_size": 8, - "tp": 8, - "ep": 1, - "n_shared_experts": 1 - }, - "name": "kimik2.5" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 16384, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 384, - "top_k": 8, - "dtype_act": "int4", - "dtype_weight": "int4", - "world_size": 8, - "tp": 8, - "ep": 1, - "n_shared_experts": 1 - }, - "name": "kimik2.5" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 16, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 384, - "top_k": 8, - "dtype_act": "int4", - "dtype_weight": "int4", - "world_size": 8, - "tp": 8, - "ep": 1, - "n_shared_experts": 1 - }, - "name": "kimik2.5" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 32768, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 384, - "top_k": 8, - "dtype_act": "int4", - "dtype_weight": "int4", - "world_size": 8, - "tp": 8, - "ep": 1, - "n_shared_experts": 1 - }, - "name": "kimik2.5" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 32, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 384, - "top_k": 8, - "dtype_act": "int4", - "dtype_weight": "int4", - "world_size": 8, - "tp": 8, - "ep": 1, - "n_shared_experts": 1 - }, - "name": "kimik2.5" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 65536, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 384, - "top_k": 8, - "dtype_act": "int4", - "dtype_weight": "int4", - "world_size": 8, - "tp": 8, - "ep": 1, - "n_shared_experts": 1 - }, - "name": "kimik2.5" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 64, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 384, - "top_k": 8, - "dtype_act": "int4", - "dtype_weight": "int4", - "world_size": 8, - "tp": 8, - "ep": 1, - "n_shared_experts": 1 - }, - "name": "kimik2.5" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 131072, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 384, - "top_k": 8, - "dtype_act": "int4", - "dtype_weight": "int4", - "world_size": 8, - "tp": 8, - "ep": 1, - "n_shared_experts": 1 - }, - "name": "kimik2.5" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 262144, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 384, - "top_k": 8, - "dtype_act": "int4", - "dtype_weight": "int4", - "world_size": 8, - "tp": 8, - "ep": 1, - "n_shared_experts": 1 - }, - "name": "kimik2.5" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 524288, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 384, - "top_k": 8, - "dtype_act": "int4", - "dtype_weight": "int4", - "world_size": 8, - "tp": 8, - "ep": 1, - "n_shared_experts": 1 - }, - "name": "kimik2.5" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 4096, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 384, - "top_k": 8, - "dtype_act": "int4", - "dtype_weight": "int4", - "world_size": 4, - "tp": 4, - "ep": 1, - "n_shared_experts": 1 - }, - "name": "kimik2.5" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 4, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 384, - "top_k": 8, - "dtype_act": "int4", - "dtype_weight": "int4", - "world_size": 4, - "tp": 4, - "ep": 1, - "n_shared_experts": 1 - }, - "name": "kimik2.5" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 8192, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 384, - "top_k": 8, - "dtype_act": "int4", - "dtype_weight": "int4", - "world_size": 4, - "tp": 4, - "ep": 1, - "n_shared_experts": 1 - }, - "name": "kimik2.5" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 8, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 384, - "top_k": 8, - "dtype_act": "int4", - "dtype_weight": "int4", - "world_size": 4, - "tp": 4, - "ep": 1, - "n_shared_experts": 1 - }, - "name": "kimik2.5" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 16384, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 384, - "top_k": 8, - "dtype_act": "int4", - "dtype_weight": "int4", - "world_size": 4, - "tp": 4, - "ep": 1, - "n_shared_experts": 1 - }, - "name": "kimik2.5" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 16, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 384, - "top_k": 8, - "dtype_act": "int4", - "dtype_weight": "int4", - "world_size": 4, - "tp": 4, - "ep": 1, - "n_shared_experts": 1 - }, - "name": "kimik2.5" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 32768, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 384, - "top_k": 8, - "dtype_act": "int4", - "dtype_weight": "int4", - "world_size": 4, - "tp": 4, - "ep": 1, - "n_shared_experts": 1 - }, - "name": "kimik2.5" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 32, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 384, - "top_k": 8, - "dtype_act": "int4", - "dtype_weight": "int4", - "world_size": 4, - "tp": 4, - "ep": 1, - "n_shared_experts": 1 - }, - "name": "kimik2.5" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 65536, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 384, - "top_k": 8, - "dtype_act": "int4", - "dtype_weight": "int4", - "world_size": 4, - "tp": 4, - "ep": 1, - "n_shared_experts": 1 - }, - "name": "kimik2.5" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 64, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 384, - "top_k": 8, - "dtype_act": "int4", - "dtype_weight": "int4", - "world_size": 4, - "tp": 4, - "ep": 1, - "n_shared_experts": 1 - }, - "name": "kimik2.5" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 131072, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 384, - "top_k": 8, - "dtype_act": "int4", - "dtype_weight": "int4", - "world_size": 4, - "tp": 4, - "ep": 1, - "n_shared_experts": 1 - }, - "name": "kimik2.5" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 262144, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 384, - "top_k": 8, - "dtype_act": "int4", - "dtype_weight": "int4", - "world_size": 4, - "tp": 4, - "ep": 1, - "n_shared_experts": 1 - }, - "name": "kimik2.5" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 524288, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 384, - "top_k": 8, - "dtype_act": "int4", - "dtype_weight": "int4", - "world_size": 4, - "tp": 4, - "ep": 1, - "n_shared_experts": 1 - }, - "name": "kimik2.5" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 4096, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 384, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 8, - "tp": 8, - "ep": 1, - "n_shared_experts": 1 - }, - "name": "kimik2.5" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 4, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 384, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 8, - "tp": 8, - "ep": 1, - "n_shared_experts": 1 - }, - "name": "kimik2.5" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 4096, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 384, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 4, - "tp": 4, - "ep": 1, - "n_shared_experts": 1 - }, - "name": "kimik2.5" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 4, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 384, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 4, - "tp": 4, - "ep": 1, - "n_shared_experts": 1 - }, - "name": "kimik2.5" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 8192, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 384, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 4, - "tp": 4, - "ep": 1, - "n_shared_experts": 1 - }, - "name": "kimik2.5" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 8, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 384, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 4, - "tp": 4, - "ep": 1, - "n_shared_experts": 1 - }, - "name": "kimik2.5" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 16384, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 384, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 4, - "tp": 4, - "ep": 1, - "n_shared_experts": 1 - }, - "name": "kimik2.5" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 16, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 384, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 4, - "tp": 4, - "ep": 1, - "n_shared_experts": 1 - }, - "name": "kimik2.5" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 32768, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 384, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 4, - "tp": 4, - "ep": 1, - "n_shared_experts": 1 - }, - "name": "kimik2.5" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 32, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 384, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 4, - "tp": 4, - "ep": 1, - "n_shared_experts": 1 - }, - "name": "kimik2.5" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 65536, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 384, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 4, - "tp": 4, - "ep": 1, - "n_shared_experts": 1 - }, - "name": "kimik2.5" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 64, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 384, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 4, - "tp": 4, - "ep": 1, - "n_shared_experts": 1 - }, - "name": "kimik2.5" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 32768, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 384, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 8, - "tp": 8, - "ep": 1, - "n_shared_experts": 1 - }, - "name": "kimik2.5" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 131072, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 384, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 4, - "tp": 4, - "ep": 1, - "n_shared_experts": 1 - }, - "name": "kimik2.5" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 262144, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 384, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 4, - "tp": 4, - "ep": 1, - "n_shared_experts": 1 - }, - "name": "kimik2.5" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 524288, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 384, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 4, - "tp": 4, - "ep": 1, - "n_shared_experts": 1 - }, - "name": "kimik2.5" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 8192, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "world_size": 4, - "tp": 4, - "ep": 1, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 1024, - "hidden": 7168, - "intermediate": 3072, - "num_experts": 384, - "top_k": 6, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "world_size": 8, - "tp": 8, - "ep": 1, - "n_shared_experts": 1 - }, - "name": "dsv4" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 1, - "hidden": 7168, - "intermediate": 3072, - "num_experts": 384, - "top_k": 6, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "world_size": 8, - "tp": 8, - "ep": 1, - "n_shared_experts": 1 - }, - "name": "dsv4" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 2048, - "hidden": 7168, - "intermediate": 3072, - "num_experts": 384, - "top_k": 6, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "world_size": 8, - "tp": 8, - "ep": 1, - "n_shared_experts": 1 - }, - "name": "dsv4" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 2, - "hidden": 7168, - "intermediate": 3072, - "num_experts": 384, - "top_k": 6, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "world_size": 8, - "tp": 8, - "ep": 1, - "n_shared_experts": 1 - }, - "name": "dsv4" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 4096, - "hidden": 7168, - "intermediate": 3072, - "num_experts": 384, - "top_k": 6, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "world_size": 8, - "tp": 8, - "ep": 1, - "n_shared_experts": 1 - }, - "name": "dsv4" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 4, - "hidden": 7168, - "intermediate": 3072, - "num_experts": 384, - "top_k": 6, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "world_size": 8, - "tp": 8, - "ep": 1, - "n_shared_experts": 1 - }, - "name": "dsv4" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 8192, - "hidden": 7168, - "intermediate": 3072, - "num_experts": 384, - "top_k": 6, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "world_size": 8, - "tp": 8, - "ep": 1, - "n_shared_experts": 1 - }, - "name": "dsv4" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 8, - "hidden": 7168, - "intermediate": 3072, - "num_experts": 384, - "top_k": 6, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "world_size": 8, - "tp": 8, - "ep": 1, - "n_shared_experts": 1 - }, - "name": "dsv4" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 16384, - "hidden": 7168, - "intermediate": 3072, - "num_experts": 384, - "top_k": 6, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "world_size": 8, - "tp": 8, - "ep": 1, - "n_shared_experts": 1 - }, - "name": "dsv4" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 16, - "hidden": 7168, - "intermediate": 3072, - "num_experts": 384, - "top_k": 6, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "world_size": 8, - "tp": 8, - "ep": 1, - "n_shared_experts": 1 - }, - "name": "dsv4" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 32768, - "hidden": 7168, - "intermediate": 3072, - "num_experts": 384, - "top_k": 6, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "world_size": 8, - "tp": 8, - "ep": 1, - "n_shared_experts": 1 - }, - "name": "dsv4" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 32, - "hidden": 7168, - "intermediate": 3072, - "num_experts": 384, - "top_k": 6, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "world_size": 8, - "tp": 8, - "ep": 1, - "n_shared_experts": 1 - }, - "name": "dsv4" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 65536, - "hidden": 7168, - "intermediate": 3072, - "num_experts": 384, - "top_k": 6, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "world_size": 8, - "tp": 8, - "ep": 1, - "n_shared_experts": 1 - }, - "name": "dsv4" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 64, - "hidden": 7168, - "intermediate": 3072, - "num_experts": 384, - "top_k": 6, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "world_size": 8, - "tp": 8, - "ep": 1, - "n_shared_experts": 1 - }, - "name": "dsv4" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 131072, - "hidden": 7168, - "intermediate": 3072, - "num_experts": 384, - "top_k": 6, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "world_size": 8, - "tp": 8, - "ep": 1, - "n_shared_experts": 1 - }, - "name": "dsv4" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 128, - "hidden": 7168, - "intermediate": 3072, - "num_experts": 384, - "top_k": 6, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "world_size": 8, - "tp": 8, - "ep": 1, - "n_shared_experts": 1 - }, - "name": "dsv4" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 262144, - "hidden": 7168, - "intermediate": 3072, - "num_experts": 384, - "top_k": 6, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "world_size": 8, - "tp": 8, - "ep": 1, - "n_shared_experts": 1 - }, - "name": "dsv4" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 256, - "hidden": 7168, - "intermediate": 3072, - "num_experts": 384, - "top_k": 6, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "world_size": 8, - "tp": 8, - "ep": 1, - "n_shared_experts": 1 - }, - "name": "dsv4" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 1024, - "hidden": 7168, - "intermediate": 3072, - "num_experts": 384, - "top_k": 6, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "world_size": 8, - "tp": 1, - "ep": 8, - "n_shared_experts": 1 - }, - "name": "dsv4" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 1, - "hidden": 7168, - "intermediate": 3072, - "num_experts": 384, - "top_k": 6, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "world_size": 8, - "tp": 1, - "ep": 8, - "n_shared_experts": 1 - }, - "name": "dsv4" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 2048, - "hidden": 7168, - "intermediate": 3072, - "num_experts": 384, - "top_k": 6, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "world_size": 8, - "tp": 1, - "ep": 8, - "n_shared_experts": 1 - }, - "name": "dsv4" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 2, - "hidden": 7168, - "intermediate": 3072, - "num_experts": 384, - "top_k": 6, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "world_size": 8, - "tp": 1, - "ep": 8, - "n_shared_experts": 1 - }, - "name": "dsv4" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 4096, - "hidden": 7168, - "intermediate": 3072, - "num_experts": 384, - "top_k": 6, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "world_size": 8, - "tp": 1, - "ep": 8, - "n_shared_experts": 1 - }, - "name": "dsv4" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 4, - "hidden": 7168, - "intermediate": 3072, - "num_experts": 384, - "top_k": 6, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "world_size": 8, - "tp": 1, - "ep": 8, - "n_shared_experts": 1 - }, - "name": "dsv4" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 8192, - "hidden": 7168, - "intermediate": 3072, - "num_experts": 384, - "top_k": 6, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "world_size": 8, - "tp": 1, - "ep": 8, - "n_shared_experts": 1 - }, - "name": "dsv4" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 8, - "hidden": 7168, - "intermediate": 3072, - "num_experts": 384, - "top_k": 6, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "world_size": 8, - "tp": 1, - "ep": 8, - "n_shared_experts": 1 - }, - "name": "dsv4" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 16384, - "hidden": 7168, - "intermediate": 3072, - "num_experts": 384, - "top_k": 6, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "world_size": 8, - "tp": 1, - "ep": 8, - "n_shared_experts": 1 - }, - "name": "dsv4" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 16, - "hidden": 7168, - "intermediate": 3072, - "num_experts": 384, - "top_k": 6, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "world_size": 8, - "tp": 1, - "ep": 8, - "n_shared_experts": 1 - }, - "name": "dsv4" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 32768, - "hidden": 7168, - "intermediate": 3072, - "num_experts": 384, - "top_k": 6, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "world_size": 8, - "tp": 1, - "ep": 8, - "n_shared_experts": 1 - }, - "name": "dsv4" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 32, - "hidden": 7168, - "intermediate": 3072, - "num_experts": 384, - "top_k": 6, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "world_size": 8, - "tp": 1, - "ep": 8, - "n_shared_experts": 1 - }, - "name": "dsv4" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 65536, - "hidden": 7168, - "intermediate": 3072, - "num_experts": 384, - "top_k": 6, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "world_size": 8, - "tp": 1, - "ep": 8, - "n_shared_experts": 1 - }, - "name": "dsv4" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 64, - "hidden": 7168, - "intermediate": 3072, - "num_experts": 384, - "top_k": 6, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "world_size": 8, - "tp": 1, - "ep": 8, - "n_shared_experts": 1 - }, - "name": "dsv4" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 131072, - "hidden": 7168, - "intermediate": 3072, - "num_experts": 384, - "top_k": 6, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "world_size": 8, - "tp": 1, - "ep": 8, - "n_shared_experts": 1 - }, - "name": "dsv4" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 128, - "hidden": 7168, - "intermediate": 3072, - "num_experts": 384, - "top_k": 6, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "world_size": 8, - "tp": 1, - "ep": 8, - "n_shared_experts": 1 - }, - "name": "dsv4" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 262144, - "hidden": 7168, - "intermediate": 3072, - "num_experts": 384, - "top_k": 6, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "world_size": 8, - "tp": 1, - "ep": 8, - "n_shared_experts": 1 - }, - "name": "dsv4" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 256, - "hidden": 7168, - "intermediate": 3072, - "num_experts": 384, - "top_k": 6, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "world_size": 8, - "tp": 1, - "ep": 8, - "n_shared_experts": 1 - }, - "name": "dsv4" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 524288, - "hidden": 7168, - "intermediate": 3072, - "num_experts": 384, - "top_k": 6, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "world_size": 8, - "tp": 8, - "ep": 1, - "n_shared_experts": 1 - }, - "name": "dsv4" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 1048576, - "hidden": 7168, - "intermediate": 3072, - "num_experts": 384, - "top_k": 6, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "world_size": 8, - "tp": 8, - "ep": 1, - "n_shared_experts": 1 - }, - "name": "dsv4" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 2097152, - "hidden": 7168, - "intermediate": 3072, - "num_experts": 384, - "top_k": 6, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "world_size": 8, - "tp": 8, - "ep": 1, - "n_shared_experts": 1 - }, - "name": "dsv4" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 524288, - "hidden": 7168, - "intermediate": 3072, - "num_experts": 384, - "top_k": 6, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "world_size": 8, - "tp": 1, - "ep": 8, - "n_shared_experts": 1 - }, - "name": "dsv4" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 1048576, - "hidden": 7168, - "intermediate": 3072, - "num_experts": 384, - "top_k": 6, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "world_size": 8, - "tp": 1, - "ep": 8, - "n_shared_experts": 1 - }, - "name": "dsv4" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 2097152, - "hidden": 7168, - "intermediate": 3072, - "num_experts": 384, - "top_k": 6, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "world_size": 8, - "tp": 1, - "ep": 8, - "n_shared_experts": 1 - }, - "name": "dsv4" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 512, - "hidden": 7168, - "intermediate": 3072, - "num_experts": 384, - "top_k": 6, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "world_size": 8, - "tp": 8, - "ep": 1, - "n_shared_experts": 1 - }, - "name": "dsv4" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 512, - "hidden": 7168, - "intermediate": 3072, - "num_experts": 384, - "top_k": 6, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "world_size": 8, - "tp": 1, - "ep": 8, - "n_shared_experts": 1 - }, - "name": "dsv4" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 1024, - "hidden": 7168, - "intermediate": 3072, - "num_experts": 384, - "top_k": 6, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 4, - "tp": 4, - "ep": 1, - "n_shared_experts": 1 - }, - "name": "dsv4" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 1, - "hidden": 7168, - "intermediate": 3072, - "num_experts": 384, - "top_k": 6, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 4, - "tp": 4, - "ep": 1, - "n_shared_experts": 1 - }, - "name": "dsv4" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 2048, - "hidden": 7168, - "intermediate": 3072, - "num_experts": 384, - "top_k": 6, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 4, - "tp": 4, - "ep": 1, - "n_shared_experts": 1 - }, - "name": "dsv4" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 2, - "hidden": 7168, - "intermediate": 3072, - "num_experts": 384, - "top_k": 6, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 4, - "tp": 4, - "ep": 1, - "n_shared_experts": 1 - }, - "name": "dsv4" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 4, - "hidden": 7168, - "intermediate": 3072, - "num_experts": 384, - "top_k": 6, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 4, - "tp": 4, - "ep": 1, - "n_shared_experts": 1 - }, - "name": "dsv4" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 128, - "hidden": 7168, - "intermediate": 3072, - "num_experts": 384, - "top_k": 6, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 4, - "tp": 4, - "ep": 1, - "n_shared_experts": 1 - }, - "name": "dsv4" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 262144, - "hidden": 7168, - "intermediate": 3072, - "num_experts": 384, - "top_k": 6, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 4, - "tp": 1, - "ep": 4, - "n_shared_experts": 1 - }, - "name": "dsv4" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 256, - "hidden": 7168, - "intermediate": 3072, - "num_experts": 384, - "top_k": 6, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 4, - "tp": 1, - "ep": 4, - "n_shared_experts": 1 - }, - "name": "dsv4" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 2097152, - "hidden": 7168, - "intermediate": 3072, - "num_experts": 384, - "top_k": 6, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 4, - "tp": 1, - "ep": 4, - "n_shared_experts": 1 - }, - "name": "dsv4" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 2048, - "hidden": 7168, - "intermediate": 3072, - "num_experts": 384, - "top_k": 6, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 4, - "tp": 1, - "ep": 4, - "n_shared_experts": 1 - }, - "name": "dsv4" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 524288, - "hidden": 7168, - "intermediate": 3072, - "num_experts": 384, - "top_k": 6, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 4, - "tp": 4, - "ep": 1, - "n_shared_experts": 1 - }, - "name": "dsv4" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 65536, - "hidden": 7168, - "intermediate": 3072, - "num_experts": 384, - "top_k": 6, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 4, - "tp": 1, - "ep": 4, - "n_shared_experts": 1 - }, - "name": "dsv4" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 64, - "hidden": 7168, - "intermediate": 3072, - "num_experts": 384, - "top_k": 6, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 4, - "tp": 1, - "ep": 4, - "n_shared_experts": 1 - }, - "name": "dsv4" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 131072, - "hidden": 7168, - "intermediate": 3072, - "num_experts": 384, - "top_k": 6, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 4, - "tp": 1, - "ep": 4, - "n_shared_experts": 1 - }, - "name": "dsv4" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 128, - "hidden": 7168, - "intermediate": 3072, - "num_experts": 384, - "top_k": 6, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 4, - "tp": 1, - "ep": 4, - "n_shared_experts": 1 - }, - "name": "dsv4" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 256, - "hidden": 7168, - "intermediate": 3072, - "num_experts": 384, - "top_k": 6, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 4, - "tp": 4, - "ep": 1, - "n_shared_experts": 1 - }, - "name": "dsv4" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 512, - "hidden": 7168, - "intermediate": 3072, - "num_experts": 384, - "top_k": 6, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 4, - "tp": 4, - "ep": 1, - "n_shared_experts": 1 - }, - "name": "dsv4" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 524288, - "hidden": 7168, - "intermediate": 3072, - "num_experts": 384, - "top_k": 6, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 8, - "tp": 8, - "ep": 1, - "n_shared_experts": 1 - }, - "name": "dsv4" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 1048576, - "hidden": 7168, - "intermediate": 3072, - "num_experts": 384, - "top_k": 6, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 4, - "tp": 1, - "ep": 4, - "n_shared_experts": 1 - }, - "name": "dsv4" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 4096, - "hidden": 4096, - "intermediate": 1024, - "num_experts": 512, - "top_k": 10, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "world_size": 8, - "tp": 1, - "ep": 8, - "n_shared_experts": 1 - }, - "name": "qwen3.5" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 4, - "hidden": 4096, - "intermediate": 1024, - "num_experts": 512, - "top_k": 10, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "world_size": 8, - "tp": 1, - "ep": 8, - "n_shared_experts": 1 - }, - "name": "qwen3.5" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 8192, - "hidden": 4096, - "intermediate": 1024, - "num_experts": 512, - "top_k": 10, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "world_size": 8, - "tp": 1, - "ep": 8, - "n_shared_experts": 1 - }, - "name": "qwen3.5" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 8, - "hidden": 4096, - "intermediate": 1024, - "num_experts": 512, - "top_k": 10, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "world_size": 8, - "tp": 1, - "ep": 8, - "n_shared_experts": 1 - }, - "name": "qwen3.5" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 16384, - "hidden": 4096, - "intermediate": 1024, - "num_experts": 512, - "top_k": 10, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "world_size": 8, - "tp": 1, - "ep": 8, - "n_shared_experts": 1 - }, - "name": "qwen3.5" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 16, - "hidden": 4096, - "intermediate": 1024, - "num_experts": 512, - "top_k": 10, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "world_size": 8, - "tp": 1, - "ep": 8, - "n_shared_experts": 1 - }, - "name": "qwen3.5" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 32768, - "hidden": 4096, - "intermediate": 1024, - "num_experts": 512, - "top_k": 10, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "world_size": 8, - "tp": 1, - "ep": 8, - "n_shared_experts": 1 - }, - "name": "qwen3.5" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 32, - "hidden": 4096, - "intermediate": 1024, - "num_experts": 512, - "top_k": 10, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "world_size": 8, - "tp": 1, - "ep": 8, - "n_shared_experts": 1 - }, - "name": "qwen3.5" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 65536, - "hidden": 4096, - "intermediate": 1024, - "num_experts": 512, - "top_k": 10, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "world_size": 8, - "tp": 1, - "ep": 8, - "n_shared_experts": 1 - }, - "name": "qwen3.5" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 64, - "hidden": 4096, - "intermediate": 1024, - "num_experts": 512, - "top_k": 10, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "world_size": 8, - "tp": 1, - "ep": 8, - "n_shared_experts": 1 - }, - "name": "qwen3.5" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 131072, - "hidden": 4096, - "intermediate": 1024, - "num_experts": 512, - "top_k": 10, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "world_size": 8, - "tp": 1, - "ep": 8, - "n_shared_experts": 1 - }, - "name": "qwen3.5" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 262144, - "hidden": 4096, - "intermediate": 1024, - "num_experts": 512, - "top_k": 10, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "world_size": 8, - "tp": 1, - "ep": 8, - "n_shared_experts": 1 - }, - "name": "qwen3.5" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 524288, - "hidden": 4096, - "intermediate": 1024, - "num_experts": 512, - "top_k": 10, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "world_size": 8, - "tp": 1, - "ep": 8, - "n_shared_experts": 1 - }, - "name": "qwen3.5" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 128, - "hidden": 4096, - "intermediate": 1024, - "num_experts": 512, - "top_k": 10, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "world_size": 8, - "tp": 1, - "ep": 8, - "n_shared_experts": 1 - }, - "name": "qwen3.5" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 256, - "hidden": 4096, - "intermediate": 1024, - "num_experts": 512, - "top_k": 10, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "world_size": 8, - "tp": 1, - "ep": 8, - "n_shared_experts": 1 - }, - "name": "qwen3.5" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 1048576, - "hidden": 4096, - "intermediate": 1024, - "num_experts": 512, - "top_k": 10, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "world_size": 8, - "tp": 1, - "ep": 8, - "n_shared_experts": 1 - }, - "name": "qwen3.5" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 8, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "world_size": 8, - "tp": 1, - "ep": 8, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 16384, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "world_size": 8, - "tp": 1, - "ep": 8, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 524288, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "world_size": 8, - "tp": 1, - "ep": 8, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 1024, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "world_size": 8, - "tp": 1, - "ep": 8, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 2, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "world_size": 8, - "tp": 1, - "ep": 8, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 1, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "world_size": 8, - "tp": 1, - "ep": 8, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 6144, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "world_size": 16, - "tp": 1, - "ep": 16, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 12, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "world_size": 16, - "tp": 1, - "ep": 16, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 9216, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "world_size": 16, - "tp": 1, - "ep": 16, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 18, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "world_size": 16, - "tp": 1, - "ep": 16, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 30720, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "world_size": 16, - "tp": 1, - "ep": 16, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 60, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "world_size": 16, - "tp": 1, - "ep": 16, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 61440, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "world_size": 16, - "tp": 1, - "ep": 16, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 120, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "world_size": 16, - "tp": 1, - "ep": 16, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 119808, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "world_size": 16, - "tp": 1, - "ep": 16, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 234, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "world_size": 16, - "tp": 1, - "ep": 16, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 236544, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "world_size": 16, - "tp": 1, - "ep": 16, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 462, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "world_size": 16, - "tp": 1, - "ep": 16, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 473088, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "world_size": 16, - "tp": 1, - "ep": 16, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 924, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "world_size": 16, - "tp": 1, - "ep": 16, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 629760, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "world_size": 16, - "tp": 1, - "ep": 16, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 1230, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "world_size": 16, - "tp": 1, - "ep": 16, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 1258496, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "world_size": 16, - "tp": 1, - "ep": 16, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 2458, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "world_size": 16, - "tp": 1, - "ep": 16, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 6, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "world_size": 16, - "tp": 1, - "ep": 16, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 9, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "world_size": 16, - "tp": 1, - "ep": 16, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 30, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "world_size": 16, - "tp": 1, - "ep": 16, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 231, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "world_size": 16, - "tp": 1, - "ep": 16, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 946176, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "world_size": 16, - "tp": 1, - "ep": 16, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 1889280, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "world_size": 16, - "tp": 1, - "ep": 16, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 1845, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "world_size": 16, - "tp": 1, - "ep": 16, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 5033984, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "world_size": 16, - "tp": 1, - "ep": 16, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 4916, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "world_size": 16, - "tp": 1, - "ep": 16, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 49152, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "world_size": 16, - "tp": 1, - "ep": 16, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 73728, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "world_size": 16, - "tp": 1, - "ep": 16, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 245760, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "world_size": 16, - "tp": 1, - "ep": 16, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 630784, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "world_size": 16, - "tp": 1, - "ep": 16, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 154, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "world_size": 16, - "tp": 1, - "ep": 16, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 1261568, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "world_size": 16, - "tp": 1, - "ep": 16, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 308, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "world_size": 16, - "tp": 1, - "ep": 16, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 2523136, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "world_size": 16, - "tp": 1, - "ep": 16, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 262144, - "hidden": 2880, - "intermediate": 2880, - "num_experts": 128, - "top_k": 4, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 1, - "tp": 1, - "ep": 1, - "n_shared_experts": 0 - }, - "name": "gptoss" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 256, - "hidden": 2880, - "intermediate": 2880, - "num_experts": 128, - "top_k": 4, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 1, - "tp": 1, - "ep": 1, - "n_shared_experts": 0 - }, - "name": "gptoss" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 262144, - "hidden": 2880, - "intermediate": 2880, - "num_experts": 128, - "top_k": 4, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 2, - "tp": 1, - "ep": 2, - "n_shared_experts": 0 - }, - "name": "gptoss" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 256, - "hidden": 2880, - "intermediate": 2880, - "num_experts": 128, - "top_k": 4, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 2, - "tp": 1, - "ep": 2, - "n_shared_experts": 0 - }, - "name": "gptoss" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 4096, - "hidden": 2880, - "intermediate": 2880, - "num_experts": 128, - "top_k": 4, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 2, - "tp": 2, - "ep": 1, - "n_shared_experts": 0 - }, - "name": "gptoss" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 4, - "hidden": 2880, - "intermediate": 2880, - "num_experts": 128, - "top_k": 4, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 2, - "tp": 2, - "ep": 1, - "n_shared_experts": 0 - }, - "name": "gptoss" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 8192, - "hidden": 2880, - "intermediate": 2880, - "num_experts": 128, - "top_k": 4, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 2, - "tp": 2, - "ep": 1, - "n_shared_experts": 0 - }, - "name": "gptoss" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 8, - "hidden": 2880, - "intermediate": 2880, - "num_experts": 128, - "top_k": 4, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 2, - "tp": 2, - "ep": 1, - "n_shared_experts": 0 - }, - "name": "gptoss" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 16384, - "hidden": 2880, - "intermediate": 2880, - "num_experts": 128, - "top_k": 4, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 2, - "tp": 2, - "ep": 1, - "n_shared_experts": 0 - }, - "name": "gptoss" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 16, - "hidden": 2880, - "intermediate": 2880, - "num_experts": 128, - "top_k": 4, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 2, - "tp": 2, - "ep": 1, - "n_shared_experts": 0 - }, - "name": "gptoss" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 32768, - "hidden": 2880, - "intermediate": 2880, - "num_experts": 128, - "top_k": 4, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 2, - "tp": 2, - "ep": 1, - "n_shared_experts": 0 - }, - "name": "gptoss" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 32, - "hidden": 2880, - "intermediate": 2880, - "num_experts": 128, - "top_k": 4, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 2, - "tp": 2, - "ep": 1, - "n_shared_experts": 0 - }, - "name": "gptoss" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 65536, - "hidden": 2880, - "intermediate": 2880, - "num_experts": 128, - "top_k": 4, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 2, - "tp": 2, - "ep": 1, - "n_shared_experts": 0 - }, - "name": "gptoss" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 64, - "hidden": 2880, - "intermediate": 2880, - "num_experts": 128, - "top_k": 4, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 2, - "tp": 2, - "ep": 1, - "n_shared_experts": 0 - }, - "name": "gptoss" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 131072, - "hidden": 2880, - "intermediate": 2880, - "num_experts": 128, - "top_k": 4, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 2, - "tp": 2, - "ep": 1, - "n_shared_experts": 0 - }, - "name": "gptoss" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 128, - "hidden": 2880, - "intermediate": 2880, - "num_experts": 128, - "top_k": 4, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 2, - "tp": 2, - "ep": 1, - "n_shared_experts": 0 - }, - "name": "gptoss" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 262144, - "hidden": 2880, - "intermediate": 2880, - "num_experts": 128, - "top_k": 4, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 2, - "tp": 2, - "ep": 1, - "n_shared_experts": 0 - }, - "name": "gptoss" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 256, - "hidden": 2880, - "intermediate": 2880, - "num_experts": 128, - "top_k": 4, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 2, - "tp": 2, - "ep": 1, - "n_shared_experts": 0 - }, - "name": "gptoss" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 65536, - "hidden": 2880, - "intermediate": 2880, - "num_experts": 128, - "top_k": 4, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 4, - "tp": 1, - "ep": 4, - "n_shared_experts": 0 - }, - "name": "gptoss" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 64, - "hidden": 2880, - "intermediate": 2880, - "num_experts": 128, - "top_k": 4, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 4, - "tp": 1, - "ep": 4, - "n_shared_experts": 0 - }, - "name": "gptoss" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 4096, - "hidden": 2880, - "intermediate": 2880, - "num_experts": 128, - "top_k": 4, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 4, - "tp": 4, - "ep": 1, - "n_shared_experts": 0 - }, - "name": "gptoss" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 4, - "hidden": 2880, - "intermediate": 2880, - "num_experts": 128, - "top_k": 4, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 4, - "tp": 4, - "ep": 1, - "n_shared_experts": 0 - }, - "name": "gptoss" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 4096, - "hidden": 2880, - "intermediate": 2880, - "num_experts": 128, - "top_k": 4, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 8, - "tp": 8, - "ep": 1, - "n_shared_experts": 0 - }, - "name": "gptoss" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 4, - "hidden": 2880, - "intermediate": 2880, - "num_experts": 128, - "top_k": 4, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 8, - "tp": 8, - "ep": 1, - "n_shared_experts": 0 - }, - "name": "gptoss" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 32768, - "hidden": 2880, - "intermediate": 2880, - "num_experts": 128, - "top_k": 4, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 1, - "tp": 1, - "ep": 1, - "n_shared_experts": 0 - }, - "name": "gptoss" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 4, - "hidden": 2880, - "intermediate": 2880, - "num_experts": 128, - "top_k": 4, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 1, - "tp": 1, - "ep": 1, - "n_shared_experts": 0 - }, - "name": "gptoss" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 65536, - "hidden": 2880, - "intermediate": 2880, - "num_experts": 128, - "top_k": 4, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 1, - "tp": 1, - "ep": 1, - "n_shared_experts": 0 - }, - "name": "gptoss" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 8, - "hidden": 2880, - "intermediate": 2880, - "num_experts": 128, - "top_k": 4, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 1, - "tp": 1, - "ep": 1, - "n_shared_experts": 0 - }, - "name": "gptoss" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 131072, - "hidden": 2880, - "intermediate": 2880, - "num_experts": 128, - "top_k": 4, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 1, - "tp": 1, - "ep": 1, - "n_shared_experts": 0 - }, - "name": "gptoss" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 16, - "hidden": 2880, - "intermediate": 2880, - "num_experts": 128, - "top_k": 4, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 1, - "tp": 1, - "ep": 1, - "n_shared_experts": 0 - }, - "name": "gptoss" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 32, - "hidden": 2880, - "intermediate": 2880, - "num_experts": 128, - "top_k": 4, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 1, - "tp": 1, - "ep": 1, - "n_shared_experts": 0 - }, - "name": "gptoss" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 524288, - "hidden": 2880, - "intermediate": 2880, - "num_experts": 128, - "top_k": 4, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 1, - "tp": 1, - "ep": 1, - "n_shared_experts": 0 - }, - "name": "gptoss" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 64, - "hidden": 2880, - "intermediate": 2880, - "num_experts": 128, - "top_k": 4, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 1, - "tp": 1, - "ep": 1, - "n_shared_experts": 0 - }, - "name": "gptoss" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 1048576, - "hidden": 2880, - "intermediate": 2880, - "num_experts": 128, - "top_k": 4, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 1, - "tp": 1, - "ep": 1, - "n_shared_experts": 0 - }, - "name": "gptoss" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 128, - "hidden": 2880, - "intermediate": 2880, - "num_experts": 128, - "top_k": 4, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 1, - "tp": 1, - "ep": 1, - "n_shared_experts": 0 - }, - "name": "gptoss" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 2097152, - "hidden": 2880, - "intermediate": 2880, - "num_experts": 128, - "top_k": 4, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 1, - "tp": 1, - "ep": 1, - "n_shared_experts": 0 - }, - "name": "gptoss" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 524288, - "hidden": 2880, - "intermediate": 2880, - "num_experts": 128, - "top_k": 4, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 2, - "tp": 2, - "ep": 1, - "n_shared_experts": 0 - }, - "name": "gptoss" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 1048576, - "hidden": 2880, - "intermediate": 2880, - "num_experts": 128, - "top_k": 4, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 2, - "tp": 2, - "ep": 1, - "n_shared_experts": 0 - }, - "name": "gptoss" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 2097152, - "hidden": 2880, - "intermediate": 2880, - "num_experts": 128, - "top_k": 4, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 2, - "tp": 2, - "ep": 1, - "n_shared_experts": 0 - }, - "name": "gptoss" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 32768, - "hidden": 2880, - "intermediate": 2880, - "num_experts": 128, - "top_k": 4, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 4, - "tp": 4, - "ep": 1, - "n_shared_experts": 0 - }, - "name": "gptoss" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 65536, - "hidden": 2880, - "intermediate": 2880, - "num_experts": 128, - "top_k": 4, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 4, - "tp": 4, - "ep": 1, - "n_shared_experts": 0 - }, - "name": "gptoss" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 8, - "hidden": 2880, - "intermediate": 2880, - "num_experts": 128, - "top_k": 4, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 4, - "tp": 4, - "ep": 1, - "n_shared_experts": 0 - }, - "name": "gptoss" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 131072, - "hidden": 2880, - "intermediate": 2880, - "num_experts": 128, - "top_k": 4, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 4, - "tp": 4, - "ep": 1, - "n_shared_experts": 0 - }, - "name": "gptoss" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 16, - "hidden": 2880, - "intermediate": 2880, - "num_experts": 128, - "top_k": 4, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 4, - "tp": 4, - "ep": 1, - "n_shared_experts": 0 - }, - "name": "gptoss" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 262144, - "hidden": 2880, - "intermediate": 2880, - "num_experts": 128, - "top_k": 4, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 4, - "tp": 4, - "ep": 1, - "n_shared_experts": 0 - }, - "name": "gptoss" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 32, - "hidden": 2880, - "intermediate": 2880, - "num_experts": 128, - "top_k": 4, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 4, - "tp": 4, - "ep": 1, - "n_shared_experts": 0 - }, - "name": "gptoss" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 32768, - "hidden": 2880, - "intermediate": 2880, - "num_experts": 128, - "top_k": 4, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 8, - "tp": 8, - "ep": 1, - "n_shared_experts": 0 - }, - "name": "gptoss" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 4096, - "hidden": 2880, - "intermediate": 2880, - "num_experts": 128, - "top_k": 4, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 1, - "tp": 1, - "ep": 1, - "n_shared_experts": 0 - }, - "name": "gptoss" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 8192, - "hidden": 2880, - "intermediate": 2880, - "num_experts": 128, - "top_k": 4, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 1, - "tp": 1, - "ep": 1, - "n_shared_experts": 0 - }, - "name": "gptoss" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 16384, - "hidden": 2880, - "intermediate": 2880, - "num_experts": 128, - "top_k": 4, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 1, - "tp": 1, - "ep": 1, - "n_shared_experts": 0 - }, - "name": "gptoss" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 8192, - "hidden": 2880, - "intermediate": 2880, - "num_experts": 128, - "top_k": 4, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 4, - "tp": 4, - "ep": 1, - "n_shared_experts": 0 - }, - "name": "gptoss" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 16384, - "hidden": 2880, - "intermediate": 2880, - "num_experts": 128, - "top_k": 4, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 4, - "tp": 4, - "ep": 1, - "n_shared_experts": 0 - }, - "name": "gptoss" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 64, - "hidden": 2880, - "intermediate": 2880, - "num_experts": 128, - "top_k": 4, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 4, - "tp": 4, - "ep": 1, - "n_shared_experts": 0 - }, - "name": "gptoss" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 8192, - "hidden": 2880, - "intermediate": 2880, - "num_experts": 128, - "top_k": 4, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 8, - "tp": 8, - "ep": 1, - "n_shared_experts": 0 - }, - "name": "gptoss" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 8, - "hidden": 2880, - "intermediate": 2880, - "num_experts": 128, - "top_k": 4, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 8, - "tp": 8, - "ep": 1, - "n_shared_experts": 0 - }, - "name": "gptoss" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 524288, - "hidden": 2880, - "intermediate": 2880, - "num_experts": 128, - "top_k": 4, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 4, - "tp": 4, - "ep": 1, - "n_shared_experts": 0 - }, - "name": "gptoss" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 524288, - "hidden": 3072, - "intermediate": 1536, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "world_size": 2, - "tp": 1, - "ep": 2, - "n_shared_experts": 0 - }, - "name": "minimaxm2.5" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 512, - "hidden": 3072, - "intermediate": 1536, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "world_size": 2, - "tp": 1, - "ep": 2, - "n_shared_experts": 0 - }, - "name": "minimaxm2.5" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 4096, - "hidden": 3072, - "intermediate": 1536, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "world_size": 4, - "tp": 4, - "ep": 1, - "n_shared_experts": 0 - }, - "name": "minimaxm2.5" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 4, - "hidden": 3072, - "intermediate": 1536, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "world_size": 4, - "tp": 4, - "ep": 1, - "n_shared_experts": 0 - }, - "name": "minimaxm2.5" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 8192, - "hidden": 3072, - "intermediate": 1536, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "world_size": 4, - "tp": 4, - "ep": 1, - "n_shared_experts": 0 - }, - "name": "minimaxm2.5" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 8, - "hidden": 3072, - "intermediate": 1536, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "world_size": 4, - "tp": 4, - "ep": 1, - "n_shared_experts": 0 - }, - "name": "minimaxm2.5" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 16384, - "hidden": 3072, - "intermediate": 1536, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "world_size": 4, - "tp": 4, - "ep": 1, - "n_shared_experts": 0 - }, - "name": "minimaxm2.5" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 16, - "hidden": 3072, - "intermediate": 1536, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "world_size": 4, - "tp": 4, - "ep": 1, - "n_shared_experts": 0 - }, - "name": "minimaxm2.5" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 32768, - "hidden": 3072, - "intermediate": 1536, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "world_size": 4, - "tp": 4, - "ep": 1, - "n_shared_experts": 0 - }, - "name": "minimaxm2.5" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 32, - "hidden": 3072, - "intermediate": 1536, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "world_size": 4, - "tp": 4, - "ep": 1, - "n_shared_experts": 0 - }, - "name": "minimaxm2.5" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 65536, - "hidden": 3072, - "intermediate": 1536, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "world_size": 4, - "tp": 4, - "ep": 1, - "n_shared_experts": 0 - }, - "name": "minimaxm2.5" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 64, - "hidden": 3072, - "intermediate": 1536, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "world_size": 4, - "tp": 4, - "ep": 1, - "n_shared_experts": 0 - }, - "name": "minimaxm2.5" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 131072, - "hidden": 3072, - "intermediate": 1536, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "world_size": 4, - "tp": 4, - "ep": 1, - "n_shared_experts": 0 - }, - "name": "minimaxm2.5" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 128, - "hidden": 3072, - "intermediate": 1536, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "world_size": 4, - "tp": 4, - "ep": 1, - "n_shared_experts": 0 - }, - "name": "minimaxm2.5" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 262144, - "hidden": 3072, - "intermediate": 1536, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "world_size": 4, - "tp": 1, - "ep": 4, - "n_shared_experts": 0 - }, - "name": "minimaxm2.5" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 256, - "hidden": 3072, - "intermediate": 1536, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "world_size": 4, - "tp": 1, - "ep": 4, - "n_shared_experts": 0 - }, - "name": "minimaxm2.5" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 524288, - "hidden": 3072, - "intermediate": 1536, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "world_size": 4, - "tp": 1, - "ep": 4, - "n_shared_experts": 0 - }, - "name": "minimaxm2.5" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 512, - "hidden": 3072, - "intermediate": 1536, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "world_size": 4, - "tp": 1, - "ep": 4, - "n_shared_experts": 0 - }, - "name": "minimaxm2.5" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 32768, - "hidden": 3072, - "intermediate": 1536, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "world_size": 2, - "tp": 2, - "ep": 1, - "n_shared_experts": 0 - }, - "name": "minimaxm2.5" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 4, - "hidden": 3072, - "intermediate": 1536, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "world_size": 2, - "tp": 2, - "ep": 1, - "n_shared_experts": 0 - }, - "name": "minimaxm2.5" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 65536, - "hidden": 3072, - "intermediate": 1536, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "world_size": 2, - "tp": 2, - "ep": 1, - "n_shared_experts": 0 - }, - "name": "minimaxm2.5" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 8, - "hidden": 3072, - "intermediate": 1536, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "world_size": 2, - "tp": 2, - "ep": 1, - "n_shared_experts": 0 - }, - "name": "minimaxm2.5" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 131072, - "hidden": 3072, - "intermediate": 1536, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "world_size": 2, - "tp": 2, - "ep": 1, - "n_shared_experts": 0 - }, - "name": "minimaxm2.5" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 16, - "hidden": 3072, - "intermediate": 1536, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "world_size": 2, - "tp": 2, - "ep": 1, - "n_shared_experts": 0 - }, - "name": "minimaxm2.5" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 262144, - "hidden": 3072, - "intermediate": 1536, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "world_size": 2, - "tp": 2, - "ep": 1, - "n_shared_experts": 0 - }, - "name": "minimaxm2.5" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 32, - "hidden": 3072, - "intermediate": 1536, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "world_size": 2, - "tp": 2, - "ep": 1, - "n_shared_experts": 0 - }, - "name": "minimaxm2.5" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 524288, - "hidden": 3072, - "intermediate": 1536, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "world_size": 2, - "tp": 2, - "ep": 1, - "n_shared_experts": 0 - }, - "name": "minimaxm2.5" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 64, - "hidden": 3072, - "intermediate": 1536, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "world_size": 2, - "tp": 2, - "ep": 1, - "n_shared_experts": 0 - }, - "name": "minimaxm2.5" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 1048576, - "hidden": 3072, - "intermediate": 1536, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "world_size": 2, - "tp": 2, - "ep": 1, - "n_shared_experts": 0 - }, - "name": "minimaxm2.5" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 128, - "hidden": 3072, - "intermediate": 1536, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "world_size": 2, - "tp": 2, - "ep": 1, - "n_shared_experts": 0 - }, - "name": "minimaxm2.5" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 2097152, - "hidden": 3072, - "intermediate": 1536, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "world_size": 2, - "tp": 2, - "ep": 1, - "n_shared_experts": 0 - }, - "name": "minimaxm2.5" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 256, - "hidden": 3072, - "intermediate": 1536, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "world_size": 2, - "tp": 2, - "ep": 1, - "n_shared_experts": 0 - }, - "name": "minimaxm2.5" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 4194304, - "hidden": 3072, - "intermediate": 1536, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "world_size": 2, - "tp": 2, - "ep": 1, - "n_shared_experts": 0 - }, - "name": "minimaxm2.5" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 512, - "hidden": 3072, - "intermediate": 1536, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "world_size": 2, - "tp": 2, - "ep": 1, - "n_shared_experts": 0 - }, - "name": "minimaxm2.5" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 262144, - "hidden": 3072, - "intermediate": 1536, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "world_size": 4, - "tp": 4, - "ep": 1, - "n_shared_experts": 0 - }, - "name": "minimaxm2.5" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 524288, - "hidden": 3072, - "intermediate": 1536, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "world_size": 4, - "tp": 4, - "ep": 1, - "n_shared_experts": 0 - }, - "name": "minimaxm2.5" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 1048576, - "hidden": 3072, - "intermediate": 1536, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "world_size": 4, - "tp": 4, - "ep": 1, - "n_shared_experts": 0 - }, - "name": "minimaxm2.5" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 2097152, - "hidden": 3072, - "intermediate": 1536, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "world_size": 4, - "tp": 4, - "ep": 1, - "n_shared_experts": 0 - }, - "name": "minimaxm2.5" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 256, - "hidden": 3072, - "intermediate": 1536, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "world_size": 4, - "tp": 4, - "ep": 1, - "n_shared_experts": 0 - }, - "name": "minimaxm2.5" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 4194304, - "hidden": 3072, - "intermediate": 1536, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "world_size": 4, - "tp": 4, - "ep": 1, - "n_shared_experts": 0 - }, - "name": "minimaxm2.5" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 512, - "hidden": 3072, - "intermediate": 1536, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "world_size": 4, - "tp": 4, - "ep": 1, - "n_shared_experts": 0 - }, - "name": "minimaxm2.5" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 1048576, - "hidden": 3072, - "intermediate": 1536, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "world_size": 2, - "tp": 1, - "ep": 2, - "n_shared_experts": 0 - }, - "name": "minimaxm2.5" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 1024, - "hidden": 3072, - "intermediate": 1536, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "world_size": 2, - "tp": 1, - "ep": 2, - "n_shared_experts": 0 - }, - "name": "minimaxm2.5" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 32768, - "hidden": 3072, - "intermediate": 1536, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "world_size": 1, - "tp": 1, - "ep": 1, - "n_shared_experts": 0 - }, - "name": "minimaxm2.5" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 4, - "hidden": 3072, - "intermediate": 1536, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "world_size": 1, - "tp": 1, - "ep": 1, - "n_shared_experts": 0 - }, - "name": "minimaxm2.5" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 65536, - "hidden": 3072, - "intermediate": 1536, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "world_size": 1, - "tp": 1, - "ep": 1, - "n_shared_experts": 0 - }, - "name": "minimaxm2.5" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 8, - "hidden": 3072, - "intermediate": 1536, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "world_size": 1, - "tp": 1, - "ep": 1, - "n_shared_experts": 0 - }, - "name": "minimaxm2.5" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 131072, - "hidden": 3072, - "intermediate": 1536, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "world_size": 1, - "tp": 1, - "ep": 1, - "n_shared_experts": 0 - }, - "name": "minimaxm2.5" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 16, - "hidden": 3072, - "intermediate": 1536, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "world_size": 1, - "tp": 1, - "ep": 1, - "n_shared_experts": 0 - }, - "name": "minimaxm2.5" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 4096, - "hidden": 3072, - "intermediate": 1536, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 1, - "tp": 1, - "ep": 1, - "n_shared_experts": 0 - }, - "name": "minimaxm2.5" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 4, - "hidden": 3072, - "intermediate": 1536, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 1, - "tp": 1, - "ep": 1, - "n_shared_experts": 0 - }, - "name": "minimaxm2.5" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 8192, - "hidden": 3072, - "intermediate": 1536, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 1, - "tp": 1, - "ep": 1, - "n_shared_experts": 0 - }, - "name": "minimaxm2.5" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 8, - "hidden": 3072, - "intermediate": 1536, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 1, - "tp": 1, - "ep": 1, - "n_shared_experts": 0 - }, - "name": "minimaxm2.5" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 16384, - "hidden": 3072, - "intermediate": 1536, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 1, - "tp": 1, - "ep": 1, - "n_shared_experts": 0 - }, - "name": "minimaxm2.5" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 16, - "hidden": 3072, - "intermediate": 1536, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 1, - "tp": 1, - "ep": 1, - "n_shared_experts": 0 - }, - "name": "minimaxm2.5" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 16384, - "hidden": 3072, - "intermediate": 1536, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 2, - "tp": 2, - "ep": 1, - "n_shared_experts": 0 - }, - "name": "minimaxm2.5" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 16, - "hidden": 3072, - "intermediate": 1536, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 2, - "tp": 2, - "ep": 1, - "n_shared_experts": 0 - }, - "name": "minimaxm2.5" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 131072, - "hidden": 3072, - "intermediate": 1536, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 2, - "tp": 1, - "ep": 2, - "n_shared_experts": 0 - }, - "name": "minimaxm2.5" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 128, - "hidden": 3072, - "intermediate": 1536, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 2, - "tp": 1, - "ep": 2, - "n_shared_experts": 0 - }, - "name": "minimaxm2.5" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 262144, - "hidden": 3072, - "intermediate": 1536, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 2, - "tp": 1, - "ep": 2, - "n_shared_experts": 0 - }, - "name": "minimaxm2.5" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 256, - "hidden": 3072, - "intermediate": 1536, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 2, - "tp": 1, - "ep": 2, - "n_shared_experts": 0 - }, - "name": "minimaxm2.5" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 524288, - "hidden": 3072, - "intermediate": 1536, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 2, - "tp": 1, - "ep": 2, - "n_shared_experts": 0 - }, - "name": "minimaxm2.5" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 512, - "hidden": 3072, - "intermediate": 1536, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 2, - "tp": 1, - "ep": 2, - "n_shared_experts": 0 - }, - "name": "minimaxm2.5" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 1048576, - "hidden": 3072, - "intermediate": 1536, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 2, - "tp": 1, - "ep": 2, - "n_shared_experts": 0 - }, - "name": "minimaxm2.5" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 1024, - "hidden": 3072, - "intermediate": 1536, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 2, - "tp": 1, - "ep": 2, - "n_shared_experts": 0 - }, - "name": "minimaxm2.5" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 4096, - "hidden": 3072, - "intermediate": 1536, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 4, - "tp": 4, - "ep": 1, - "n_shared_experts": 0 - }, - "name": "minimaxm2.5" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 4, - "hidden": 3072, - "intermediate": 1536, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 4, - "tp": 4, - "ep": 1, - "n_shared_experts": 0 - }, - "name": "minimaxm2.5" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 8192, - "hidden": 3072, - "intermediate": 1536, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 4, - "tp": 4, - "ep": 1, - "n_shared_experts": 0 - }, - "name": "minimaxm2.5" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 8, - "hidden": 3072, - "intermediate": 1536, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 4, - "tp": 4, - "ep": 1, - "n_shared_experts": 0 - }, - "name": "minimaxm2.5" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 16384, - "hidden": 3072, - "intermediate": 1536, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 4, - "tp": 4, - "ep": 1, - "n_shared_experts": 0 - }, - "name": "minimaxm2.5" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 16, - "hidden": 3072, - "intermediate": 1536, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 4, - "tp": 4, - "ep": 1, - "n_shared_experts": 0 - }, - "name": "minimaxm2.5" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 65536, - "hidden": 3072, - "intermediate": 1536, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 4, - "tp": 1, - "ep": 4, - "n_shared_experts": 0 - }, - "name": "minimaxm2.5" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 64, - "hidden": 3072, - "intermediate": 1536, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 4, - "tp": 1, - "ep": 4, - "n_shared_experts": 0 - }, - "name": "minimaxm2.5" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 131072, - "hidden": 3072, - "intermediate": 1536, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 4, - "tp": 1, - "ep": 4, - "n_shared_experts": 0 - }, - "name": "minimaxm2.5" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 128, - "hidden": 3072, - "intermediate": 1536, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 4, - "tp": 1, - "ep": 4, - "n_shared_experts": 0 - }, - "name": "minimaxm2.5" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 4096, - "hidden": 3072, - "intermediate": 1536, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 8, - "tp": 8, - "ep": 1, - "n_shared_experts": 0 - }, - "name": "minimaxm2.5" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 4, - "hidden": 3072, - "intermediate": 1536, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 8, - "tp": 8, - "ep": 1, - "n_shared_experts": 0 - }, - "name": "minimaxm2.5" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 8192, - "hidden": 3072, - "intermediate": 1536, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 8, - "tp": 8, - "ep": 1, - "n_shared_experts": 0 - }, - "name": "minimaxm2.5" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 8, - "hidden": 3072, - "intermediate": 1536, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 8, - "tp": 8, - "ep": 1, - "n_shared_experts": 0 - }, - "name": "minimaxm2.5" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 32768, - "hidden": 3072, - "intermediate": 1536, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 1, - "tp": 1, - "ep": 1, - "n_shared_experts": 0 - }, - "name": "minimaxm2.5" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 65536, - "hidden": 3072, - "intermediate": 1536, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 1, - "tp": 1, - "ep": 1, - "n_shared_experts": 0 - }, - "name": "minimaxm2.5" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 131072, - "hidden": 3072, - "intermediate": 1536, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 1, - "tp": 1, - "ep": 1, - "n_shared_experts": 0 - }, - "name": "minimaxm2.5" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 262144, - "hidden": 3072, - "intermediate": 1536, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 1, - "tp": 1, - "ep": 1, - "n_shared_experts": 0 - }, - "name": "minimaxm2.5" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 32, - "hidden": 3072, - "intermediate": 1536, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 1, - "tp": 1, - "ep": 1, - "n_shared_experts": 0 - }, - "name": "minimaxm2.5" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 2097152, - "hidden": 3072, - "intermediate": 1536, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 1, - "tp": 1, - "ep": 1, - "n_shared_experts": 0 - }, - "name": "minimaxm2.5" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 256, - "hidden": 3072, - "intermediate": 1536, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 1, - "tp": 1, - "ep": 1, - "n_shared_experts": 0 - }, - "name": "minimaxm2.5" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 2097152, - "hidden": 3072, - "intermediate": 1536, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 2, - "tp": 1, - "ep": 2, - "n_shared_experts": 0 - }, - "name": "minimaxm2.5" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 4194304, - "hidden": 3072, - "intermediate": 1536, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 2, - "tp": 1, - "ep": 2, - "n_shared_experts": 0 - }, - "name": "minimaxm2.5" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 32768, - "hidden": 3072, - "intermediate": 1536, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 4, - "tp": 4, - "ep": 1, - "n_shared_experts": 0 - }, - "name": "minimaxm2.5" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 65536, - "hidden": 3072, - "intermediate": 1536, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 4, - "tp": 4, - "ep": 1, - "n_shared_experts": 0 - }, - "name": "minimaxm2.5" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 32768, - "hidden": 3072, - "intermediate": 1536, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 8, - "tp": 8, - "ep": 1, - "n_shared_experts": 0 - }, - "name": "minimaxm2.5" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 2048, - "hidden": 3072, - "intermediate": 1536, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 2, - "tp": 1, - "ep": 2, - "n_shared_experts": 0 - }, - "name": "minimaxm2.5" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 524288, - "hidden": 3072, - "intermediate": 1536, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 1, - "tp": 1, - "ep": 1, - "n_shared_experts": 0 - }, - "name": "minimaxm2.5" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 64, - "hidden": 3072, - "intermediate": 1536, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 1, - "tp": 1, - "ep": 1, - "n_shared_experts": 0 - }, - "name": "minimaxm2.5" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 1048576, - "hidden": 3072, - "intermediate": 1536, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 1, - "tp": 1, - "ep": 1, - "n_shared_experts": 0 - }, - "name": "minimaxm2.5" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 128, - "hidden": 3072, - "intermediate": 1536, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 1, - "tp": 1, - "ep": 1, - "n_shared_experts": 0 - }, - "name": "minimaxm2.5" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 16384, - "hidden": 2880, - "intermediate": 2880, - "num_experts": 128, - "top_k": 4, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 8, - "tp": 8, - "ep": 1, - "n_shared_experts": 0 - }, - "name": "gptoss" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 16, - "hidden": 2880, - "intermediate": 2880, - "num_experts": 128, - "top_k": 4, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 8, - "tp": 8, - "ep": 1, - "n_shared_experts": 0 - }, - "name": "gptoss" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 32, - "hidden": 2880, - "intermediate": 2880, - "num_experts": 128, - "top_k": 4, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 8, - "tp": 8, - "ep": 1, - "n_shared_experts": 0 - }, - "name": "gptoss" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 65536, - "hidden": 2880, - "intermediate": 2880, - "num_experts": 128, - "top_k": 4, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 8, - "tp": 8, - "ep": 1, - "n_shared_experts": 0 - }, - "name": "gptoss" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 64, - "hidden": 2880, - "intermediate": 2880, - "num_experts": 128, - "top_k": 4, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 8, - "tp": 8, - "ep": 1, - "n_shared_experts": 0 - }, - "name": "gptoss" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 131072, - "hidden": 2880, - "intermediate": 2880, - "num_experts": 128, - "top_k": 4, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 8, - "tp": 8, - "ep": 1, - "n_shared_experts": 0 - }, - "name": "gptoss" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 4096, - "hidden": 3072, - "intermediate": 1536, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "world_size": 4, - "tp": 1, - "ep": 4, - "n_shared_experts": 0 - }, - "name": "minimaxm2.5" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 4, - "hidden": 3072, - "intermediate": 1536, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "world_size": 4, - "tp": 1, - "ep": 4, - "n_shared_experts": 0 - }, - "name": "minimaxm2.5" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 8192, - "hidden": 3072, - "intermediate": 1536, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "world_size": 4, - "tp": 1, - "ep": 4, - "n_shared_experts": 0 - }, - "name": "minimaxm2.5" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 8, - "hidden": 3072, - "intermediate": 1536, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "world_size": 4, - "tp": 1, - "ep": 4, - "n_shared_experts": 0 - }, - "name": "minimaxm2.5" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 16384, - "hidden": 3072, - "intermediate": 1536, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "world_size": 4, - "tp": 1, - "ep": 4, - "n_shared_experts": 0 - }, - "name": "minimaxm2.5" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 16, - "hidden": 3072, - "intermediate": 1536, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "world_size": 4, - "tp": 1, - "ep": 4, - "n_shared_experts": 0 - }, - "name": "minimaxm2.5" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 32768, - "hidden": 3072, - "intermediate": 1536, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "world_size": 4, - "tp": 1, - "ep": 4, - "n_shared_experts": 0 - }, - "name": "minimaxm2.5" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 32, - "hidden": 3072, - "intermediate": 1536, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "world_size": 4, - "tp": 1, - "ep": 4, - "n_shared_experts": 0 - }, - "name": "minimaxm2.5" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 65536, - "hidden": 3072, - "intermediate": 1536, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "world_size": 4, - "tp": 1, - "ep": 4, - "n_shared_experts": 0 - }, - "name": "minimaxm2.5" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 64, - "hidden": 3072, - "intermediate": 1536, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "world_size": 4, - "tp": 1, - "ep": 4, - "n_shared_experts": 0 - }, - "name": "minimaxm2.5" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 131072, - "hidden": 3072, - "intermediate": 1536, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "world_size": 4, - "tp": 1, - "ep": 4, - "n_shared_experts": 0 - }, - "name": "minimaxm2.5" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 1024, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "world_size": 16, - "tp": 16, - "ep": 1, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 2, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "world_size": 16, - "tp": 16, - "ep": 1, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 2048, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "world_size": 16, - "tp": 16, - "ep": 1, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 4, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "world_size": 16, - "tp": 16, - "ep": 1, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 4096, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "world_size": 16, - "tp": 16, - "ep": 1, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 8, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "world_size": 16, - "tp": 16, - "ep": 1, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 8192, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "world_size": 16, - "tp": 16, - "ep": 1, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 16, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "world_size": 16, - "tp": 16, - "ep": 1, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 16384, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "world_size": 16, - "tp": 16, - "ep": 1, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 32, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "world_size": 16, - "tp": 16, - "ep": 1, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 32768, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "world_size": 16, - "tp": 16, - "ep": 1, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 64, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "world_size": 16, - "tp": 16, - "ep": 1, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 65536, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "world_size": 16, - "tp": 16, - "ep": 1, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 128, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "world_size": 16, - "tp": 16, - "ep": 1, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 131072, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "world_size": 16, - "tp": 16, - "ep": 1, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 256, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "world_size": 16, - "tp": 16, - "ep": 1, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 1024, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "world_size": 16, - "tp": 1, - "ep": 16, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 2, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "world_size": 16, - "tp": 1, - "ep": 16, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 2048, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "world_size": 16, - "tp": 1, - "ep": 16, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 4, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "world_size": 16, - "tp": 1, - "ep": 16, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 4096, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "world_size": 16, - "tp": 1, - "ep": 16, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 8, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "world_size": 16, - "tp": 1, - "ep": 16, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 8192, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "world_size": 16, - "tp": 1, - "ep": 16, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 16, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "world_size": 16, - "tp": 1, - "ep": 16, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 16384, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "world_size": 16, - "tp": 1, - "ep": 16, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 32, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "world_size": 16, - "tp": 1, - "ep": 16, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 32768, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "world_size": 16, - "tp": 1, - "ep": 16, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 64, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "world_size": 16, - "tp": 1, - "ep": 16, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 65536, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "world_size": 16, - "tp": 1, - "ep": 16, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 128, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "world_size": 16, - "tp": 1, - "ep": 16, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 262144, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "world_size": 16, - "tp": 16, - "ep": 1, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 524288, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "world_size": 16, - "tp": 16, - "ep": 1, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 1048576, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "world_size": 16, - "tp": 16, - "ep": 1, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 131072, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "world_size": 16, - "tp": 1, - "ep": 16, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 262144, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "world_size": 16, - "tp": 1, - "ep": 16, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 524288, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "world_size": 16, - "tp": 1, - "ep": 16, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 4096, - "hidden": 2880, - "intermediate": 2880, - "num_experts": 128, - "top_k": 4, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 2, - "tp": 1, - "ep": 2, - "n_shared_experts": 0 - }, - "name": "gptoss" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 4, - "hidden": 2880, - "intermediate": 2880, - "num_experts": 128, - "top_k": 4, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 2, - "tp": 1, - "ep": 2, - "n_shared_experts": 0 - }, - "name": "gptoss" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 8192, - "hidden": 2880, - "intermediate": 2880, - "num_experts": 128, - "top_k": 4, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 2, - "tp": 1, - "ep": 2, - "n_shared_experts": 0 - }, - "name": "gptoss" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 8, - "hidden": 2880, - "intermediate": 2880, - "num_experts": 128, - "top_k": 4, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 2, - "tp": 1, - "ep": 2, - "n_shared_experts": 0 - }, - "name": "gptoss" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 16384, - "hidden": 2880, - "intermediate": 2880, - "num_experts": 128, - "top_k": 4, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 2, - "tp": 1, - "ep": 2, - "n_shared_experts": 0 - }, - "name": "gptoss" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 16, - "hidden": 2880, - "intermediate": 2880, - "num_experts": 128, - "top_k": 4, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 2, - "tp": 1, - "ep": 2, - "n_shared_experts": 0 - }, - "name": "gptoss" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 32768, - "hidden": 2880, - "intermediate": 2880, - "num_experts": 128, - "top_k": 4, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 2, - "tp": 1, - "ep": 2, - "n_shared_experts": 0 - }, - "name": "gptoss" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 32, - "hidden": 2880, - "intermediate": 2880, - "num_experts": 128, - "top_k": 4, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 2, - "tp": 1, - "ep": 2, - "n_shared_experts": 0 - }, - "name": "gptoss" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 65536, - "hidden": 2880, - "intermediate": 2880, - "num_experts": 128, - "top_k": 4, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 2, - "tp": 1, - "ep": 2, - "n_shared_experts": 0 - }, - "name": "gptoss" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 64, - "hidden": 2880, - "intermediate": 2880, - "num_experts": 128, - "top_k": 4, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 2, - "tp": 1, - "ep": 2, - "n_shared_experts": 0 - }, - "name": "gptoss" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 4096, - "hidden": 2880, - "intermediate": 2880, - "num_experts": 128, - "top_k": 4, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 4, - "tp": 1, - "ep": 4, - "n_shared_experts": 0 - }, - "name": "gptoss" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 4, - "hidden": 2880, - "intermediate": 2880, - "num_experts": 128, - "top_k": 4, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 4, - "tp": 1, - "ep": 4, - "n_shared_experts": 0 - }, - "name": "gptoss" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 8192, - "hidden": 2880, - "intermediate": 2880, - "num_experts": 128, - "top_k": 4, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 4, - "tp": 1, - "ep": 4, - "n_shared_experts": 0 - }, - "name": "gptoss" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 8, - "hidden": 2880, - "intermediate": 2880, - "num_experts": 128, - "top_k": 4, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 4, - "tp": 1, - "ep": 4, - "n_shared_experts": 0 - }, - "name": "gptoss" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 16384, - "hidden": 2880, - "intermediate": 2880, - "num_experts": 128, - "top_k": 4, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 4, - "tp": 1, - "ep": 4, - "n_shared_experts": 0 - }, - "name": "gptoss" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 16, - "hidden": 2880, - "intermediate": 2880, - "num_experts": 128, - "top_k": 4, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 4, - "tp": 1, - "ep": 4, - "n_shared_experts": 0 - }, - "name": "gptoss" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 32768, - "hidden": 2880, - "intermediate": 2880, - "num_experts": 128, - "top_k": 4, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 4, - "tp": 1, - "ep": 4, - "n_shared_experts": 0 - }, - "name": "gptoss" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 32, - "hidden": 2880, - "intermediate": 2880, - "num_experts": 128, - "top_k": 4, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 4, - "tp": 1, - "ep": 4, - "n_shared_experts": 0 - }, - "name": "gptoss" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 4096, - "hidden": 2880, - "intermediate": 2880, - "num_experts": 128, - "top_k": 4, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 8, - "tp": 1, - "ep": 8, - "n_shared_experts": 0 - }, - "name": "gptoss" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 4, - "hidden": 2880, - "intermediate": 2880, - "num_experts": 128, - "top_k": 4, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 8, - "tp": 1, - "ep": 8, - "n_shared_experts": 0 - }, - "name": "gptoss" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 8192, - "hidden": 2880, - "intermediate": 2880, - "num_experts": 128, - "top_k": 4, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 8, - "tp": 1, - "ep": 8, - "n_shared_experts": 0 - }, - "name": "gptoss" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 8, - "hidden": 2880, - "intermediate": 2880, - "num_experts": 128, - "top_k": 4, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 8, - "tp": 1, - "ep": 8, - "n_shared_experts": 0 - }, - "name": "gptoss" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 131072, - "hidden": 2880, - "intermediate": 2880, - "num_experts": 128, - "top_k": 4, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 2, - "tp": 1, - "ep": 2, - "n_shared_experts": 0 - }, - "name": "gptoss" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 524288, - "hidden": 2880, - "intermediate": 2880, - "num_experts": 128, - "top_k": 4, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 2, - "tp": 1, - "ep": 2, - "n_shared_experts": 0 - }, - "name": "gptoss" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 131072, - "hidden": 2880, - "intermediate": 2880, - "num_experts": 128, - "top_k": 4, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 4, - "tp": 1, - "ep": 4, - "n_shared_experts": 0 - }, - "name": "gptoss" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 262144, - "hidden": 2880, - "intermediate": 2880, - "num_experts": 128, - "top_k": 4, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 4, - "tp": 1, - "ep": 4, - "n_shared_experts": 0 - }, - "name": "gptoss" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 524288, - "hidden": 2880, - "intermediate": 2880, - "num_experts": 128, - "top_k": 4, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 4, - "tp": 1, - "ep": 4, - "n_shared_experts": 0 - }, - "name": "gptoss" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 32768, - "hidden": 2880, - "intermediate": 2880, - "num_experts": 128, - "top_k": 4, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 8, - "tp": 1, - "ep": 8, - "n_shared_experts": 0 - }, - "name": "gptoss" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 65536, - "hidden": 2880, - "intermediate": 2880, - "num_experts": 128, - "top_k": 4, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 8, - "tp": 1, - "ep": 8, - "n_shared_experts": 0 - }, - "name": "gptoss" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 262144, - "hidden": 2880, - "intermediate": 2880, - "num_experts": 128, - "top_k": 4, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 8, - "tp": 8, - "ep": 1, - "n_shared_experts": 0 - }, - "name": "gptoss" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 4096, - "hidden": 3072, - "intermediate": 1536, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "world_size": 8, - "tp": 8, - "ep": 1, - "n_shared_experts": 0 - }, - "name": "minimaxm2.5" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 4, - "hidden": 3072, - "intermediate": 1536, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "world_size": 8, - "tp": 8, - "ep": 1, - "n_shared_experts": 0 - }, - "name": "minimaxm2.5" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 8192, - "hidden": 3072, - "intermediate": 1536, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "world_size": 8, - "tp": 8, - "ep": 1, - "n_shared_experts": 0 - }, - "name": "minimaxm2.5" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 8, - "hidden": 3072, - "intermediate": 1536, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "world_size": 8, - "tp": 8, - "ep": 1, - "n_shared_experts": 0 - }, - "name": "minimaxm2.5" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 16384, - "hidden": 3072, - "intermediate": 1536, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "world_size": 8, - "tp": 8, - "ep": 1, - "n_shared_experts": 0 - }, - "name": "minimaxm2.5" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 16, - "hidden": 3072, - "intermediate": 1536, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "world_size": 8, - "tp": 8, - "ep": 1, - "n_shared_experts": 0 - }, - "name": "minimaxm2.5" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 32768, - "hidden": 3072, - "intermediate": 1536, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "world_size": 8, - "tp": 8, - "ep": 1, - "n_shared_experts": 0 - }, - "name": "minimaxm2.5" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 32, - "hidden": 3072, - "intermediate": 1536, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "world_size": 8, - "tp": 8, - "ep": 1, - "n_shared_experts": 0 - }, - "name": "minimaxm2.5" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 65536, - "hidden": 3072, - "intermediate": 1536, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "world_size": 8, - "tp": 8, - "ep": 1, - "n_shared_experts": 0 - }, - "name": "minimaxm2.5" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 64, - "hidden": 3072, - "intermediate": 1536, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "world_size": 8, - "tp": 8, - "ep": 1, - "n_shared_experts": 0 - }, - "name": "minimaxm2.5" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 131072, - "hidden": 3072, - "intermediate": 1536, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "world_size": 8, - "tp": 8, - "ep": 1, - "n_shared_experts": 0 - }, - "name": "minimaxm2.5" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 128, - "hidden": 3072, - "intermediate": 1536, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "world_size": 8, - "tp": 8, - "ep": 1, - "n_shared_experts": 0 - }, - "name": "minimaxm2.5" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 262144, - "hidden": 3072, - "intermediate": 1536, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "world_size": 8, - "tp": 8, - "ep": 1, - "n_shared_experts": 0 - }, - "name": "minimaxm2.5" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 524288, - "hidden": 3072, - "intermediate": 1536, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "world_size": 8, - "tp": 8, - "ep": 1, - "n_shared_experts": 0 - }, - "name": "minimaxm2.5" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 1048576, - "hidden": 3072, - "intermediate": 1536, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "world_size": 8, - "tp": 8, - "ep": 1, - "n_shared_experts": 0 - }, - "name": "minimaxm2.5" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 184320, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 32, - "tp": 1, - "ep": 32, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 360, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 32, - "tp": 1, - "ep": 32, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 12288, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 4, - "tp": 1, - "ep": 4, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 24576, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 4, - "tp": 1, - "ep": 4, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 96, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 4, - "tp": 1, - "ep": 4, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 12288, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 8, - "tp": 1, - "ep": 8, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 24, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 8, - "tp": 1, - "ep": 8, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 24576, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 8, - "tp": 1, - "ep": 8, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 48, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 8, - "tp": 1, - "ep": 8, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 96, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 8, - "tp": 1, - "ep": 8, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 4404224, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 4, - "tp": 1, - "ep": 4, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 8602, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 4, - "tp": 1, - "ep": 4, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 4404224, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 16, - "tp": 1, - "ep": 16, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 8602, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 16, - "tp": 1, - "ep": 16, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 2307072, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 4, - "tp": 1, - "ep": 4, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 4506, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 4, - "tp": 1, - "ep": 4, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 2307072, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 32, - "tp": 1, - "ep": 32, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 4506, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 32, - "tp": 1, - "ep": 32, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 16517120, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 4, - "tp": 1, - "ep": 4, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 32260, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 4, - "tp": 1, - "ep": 4, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 4301, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 4, - "tp": 1, - "ep": 4, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 4404224, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 8, - "tp": 1, - "ep": 8, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 4301, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 8, - "tp": 1, - "ep": 8, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 681984, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 4, - "tp": 1, - "ep": 4, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 666, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 4, - "tp": 1, - "ep": 4, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 681984, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 32, - "tp": 1, - "ep": 32, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 666, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 32, - "tp": 1, - "ep": 32, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 6291456, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 4, - "tp": 1, - "ep": 4, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 192, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 4, - "tp": 1, - "ep": 4, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 196608, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 8, - "tp": 1, - "ep": 8, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 192, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 8, - "tp": 1, - "ep": 8, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 5120, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 8, - "tp": 1, - "ep": 8, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 5, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 8, - "tp": 1, - "ep": 8, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 4301, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 16, - "tp": 1, - "ep": 16, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 2253, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 4, - "tp": 1, - "ep": 4, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 2253, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 32, - "tp": 1, - "ep": 32, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 393216, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 8, - "tp": 1, - "ep": 8, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 1474560, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 4, - "tp": 1, - "ep": 4, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 1474560, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 32, - "tp": 1, - "ep": 32, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 10067968, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 4, - "tp": 1, - "ep": 4, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 2458, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 4, - "tp": 1, - "ep": 4, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 10067968, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 16, - "tp": 1, - "ep": 16, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 2458, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 16, - "tp": 1, - "ep": 16, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 5455872, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 4, - "tp": 1, - "ep": 4, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 1332, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 4, - "tp": 1, - "ep": 4, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 5455872, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 32, - "tp": 1, - "ep": 32, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 1332, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 32, - "tp": 1, - "ep": 32, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 35233792, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 4, - "tp": 1, - "ep": 4, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 35233792, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 16, - "tp": 1, - "ep": 16, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 360448, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 4, - "tp": 1, - "ep": 4, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 44, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 4, - "tp": 1, - "ep": 4, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 622592, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 4, - "tp": 1, - "ep": 4, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 76, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 4, - "tp": 1, - "ep": 4, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 360448, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 8, - "tp": 1, - "ep": 8, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 44, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 8, - "tp": 1, - "ep": 8, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 622592, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 8, - "tp": 1, - "ep": 8, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 76, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 8, - "tp": 1, - "ep": 8, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 40960, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 4, - "tp": 1, - "ep": 4, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 40960, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 8, - "tp": 1, - "ep": 8, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 2727936, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 4, - "tp": 1, - "ep": 4, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 333, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 4, - "tp": 1, - "ep": 4, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 2727936, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 32, - "tp": 1, - "ep": 32, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 333, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 32, - "tp": 1, - "ep": 32, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 1229, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 4, - "tp": 1, - "ep": 4, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 10067968, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 32, - "tp": 1, - "ep": 32, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 1229, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 32, - "tp": 1, - "ep": 32, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 18456576, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 4, - "tp": 1, - "ep": 4, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 18456576, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 16, - "tp": 1, - "ep": 16, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 2253, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 16, - "tp": 1, - "ep": 16, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 33554432, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 4, - "tp": 1, - "ep": 4, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 33554432, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 16, - "tp": 1, - "ep": 16, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 4096, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 16, - "tp": 1, - "ep": 16, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 4404224, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "world_size": 8, - "tp": 1, - "ep": 8, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 8602, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "world_size": 8, - "tp": 1, - "ep": 8, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 2202624, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "world_size": 8, - "tp": 1, - "ep": 8, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 4302, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "world_size": 8, - "tp": 1, - "ep": 8, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 1258496, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "world_size": 8, - "tp": 1, - "ep": 8, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 2458, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "world_size": 8, - "tp": 1, - "ep": 8, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 629760, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "world_size": 8, - "tp": 1, - "ep": 8, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 1230, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "world_size": 8, - "tp": 1, - "ep": 8, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 629760, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "world_size": 32, - "tp": 1, - "ep": 32, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 1230, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "world_size": 32, - "tp": 1, - "ep": 32, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 36864, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "world_size": 8, - "tp": 1, - "ep": 8, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 72, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "world_size": 8, - "tp": 1, - "ep": 8, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 18432, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "world_size": 8, - "tp": 1, - "ep": 8, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 36, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "world_size": 8, - "tp": 1, - "ep": 8, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 9216, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "world_size": 8, - "tp": 1, - "ep": 8, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 18, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "world_size": 8, - "tp": 1, - "ep": 8, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 6291456, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "world_size": 8, - "tp": 1, - "ep": 8, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 6144, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "world_size": 8, - "tp": 1, - "ep": 8, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 4301, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "world_size": 8, - "tp": 1, - "ep": 8, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 2151, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "world_size": 8, - "tp": 1, - "ep": 8, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 2202624, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "world_size": 16, - "tp": 1, - "ep": 16, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 2151, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "world_size": 16, - "tp": 1, - "ep": 16, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 1154048, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "world_size": 8, - "tp": 1, - "ep": 8, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 1127, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "world_size": 8, - "tp": 1, - "ep": 8, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 1154048, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "world_size": 32, - "tp": 1, - "ep": 32, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 1127, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "world_size": 32, - "tp": 1, - "ep": 32, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 262144, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "world_size": 32, - "tp": 1, - "ep": 32, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 256, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "world_size": 32, - "tp": 1, - "ep": 32, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 27648, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "world_size": 8, - "tp": 1, - "ep": 8, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 27, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "world_size": 8, - "tp": 1, - "ep": 8, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 3, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "world_size": 8, - "tp": 1, - "ep": 8, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 5455872, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "world_size": 8, - "tp": 1, - "ep": 8, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 1332, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "world_size": 8, - "tp": 1, - "ep": 8, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 5455872, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "world_size": 16, - "tp": 1, - "ep": 16, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 1332, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "world_size": 16, - "tp": 1, - "ep": 16, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 2727936, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "world_size": 8, - "tp": 1, - "ep": 8, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 666, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "world_size": 8, - "tp": 1, - "ep": 8, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 2727936, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "world_size": 16, - "tp": 1, - "ep": 16, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 666, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "world_size": 16, - "tp": 1, - "ep": 16, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 2727936, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "world_size": 32, - "tp": 1, - "ep": 32, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 666, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "world_size": 32, - "tp": 1, - "ep": 32, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 180, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "world_size": 8, - "tp": 1, - "ep": 8, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 737280, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "world_size": 32, - "tp": 1, - "ep": 32, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 180, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "world_size": 32, - "tp": 1, - "ep": 32, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 122880, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "world_size": 8, - "tp": 1, - "ep": 8, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 30, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "world_size": 8, - "tp": 1, - "ep": 8, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 49152, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "world_size": 8, - "tp": 1, - "ep": 8, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 12, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "world_size": 8, - "tp": 1, - "ep": 8, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 10067968, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "world_size": 8, - "tp": 1, - "ep": 8, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 1229, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "world_size": 8, - "tp": 1, - "ep": 8, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 10067968, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "world_size": 16, - "tp": 1, - "ep": 16, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 1229, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "world_size": 16, - "tp": 1, - "ep": 16, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 5455872, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "world_size": 32, - "tp": 1, - "ep": 32, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 5038080, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "world_size": 8, - "tp": 1, - "ep": 8, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 615, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "world_size": 8, - "tp": 1, - "ep": 8, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 5038080, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "world_size": 16, - "tp": 1, - "ep": 16, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 615, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "world_size": 16, - "tp": 1, - "ep": 16, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 333, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "world_size": 8, - "tp": 1, - "ep": 8, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 333, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "world_size": 32, - "tp": 1, - "ep": 32, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 516096, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "world_size": 8, - "tp": 1, - "ep": 8, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 63, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "world_size": 8, - "tp": 1, - "ep": 8, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 147456, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "world_size": 8, - "tp": 1, - "ep": 8, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 6, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "world_size": 8, - "tp": 1, - "ep": 8, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 4096, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "world_size": 4, - "tp": 4, - "ep": 1, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 1048576, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "world_size": 48, - "tp": 1, - "ep": 48, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 1024, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "world_size": 48, - "tp": 1, - "ep": 48, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 2097152, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "world_size": 48, - "tp": 1, - "ep": 48, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 2048, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "world_size": 48, - "tp": 1, - "ep": 48, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 4194304, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "world_size": 48, - "tp": 1, - "ep": 48, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 4096, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "world_size": 48, - "tp": 1, - "ep": 48, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 1048576, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "world_size": 32, - "tp": 1, - "ep": 32, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 1024, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "world_size": 32, - "tp": 1, - "ep": 32, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 2097152, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "world_size": 32, - "tp": 1, - "ep": 32, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 2048, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "world_size": 32, - "tp": 1, - "ep": 32, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 4194304, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "world_size": 32, - "tp": 1, - "ep": 32, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 4096, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "world_size": 32, - "tp": 1, - "ep": 32, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 6291456, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "world_size": 32, - "tp": 1, - "ep": 32, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 6144, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "world_size": 32, - "tp": 1, - "ep": 32, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 8388608, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "world_size": 8, - "tp": 1, - "ep": 8, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 16777216, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "world_size": 8, - "tp": 1, - "ep": 8, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 50331648, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "world_size": 8, - "tp": 1, - "ep": 8, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 512, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "world_size": 32, - "tp": 1, - "ep": 32, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 8388608, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "world_size": 32, - "tp": 1, - "ep": 32, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 16777216, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "world_size": 32, - "tp": 1, - "ep": 32, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 50331648, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "world_size": 32, - "tp": 1, - "ep": 32, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 33554432, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "world_size": 8, - "tp": 1, - "ep": 8, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 16777216, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "world_size": 24, - "tp": 1, - "ep": 24, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 2048, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "world_size": 24, - "tp": 1, - "ep": 24, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 33554432, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "world_size": 24, - "tp": 1, - "ep": 24, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 4096, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "world_size": 24, - "tp": 1, - "ep": 24, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 50331648, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "world_size": 24, - "tp": 1, - "ep": 24, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 6144, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "world_size": 24, - "tp": 1, - "ep": 24, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 7340032, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "world_size": 8, - "tp": 1, - "ep": 8, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 7168, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "world_size": 8, - "tp": 1, - "ep": 8, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 7864320, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "world_size": 8, - "tp": 1, - "ep": 8, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 7680, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "world_size": 8, - "tp": 1, - "ep": 8, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 128, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "world_size": 32, - "tp": 1, - "ep": 32, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 2048, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 4, - "tp": 1, - "ep": 4, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 8388608, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 4, - "tp": 1, - "ep": 4, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 524288, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 32, - "tp": 1, - "ep": 32, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 512, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 32, - "tp": 1, - "ep": 32, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 2097152, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 32, - "tp": 1, - "ep": 32, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 2048, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 32, - "tp": 1, - "ep": 32, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 4194304, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 32, - "tp": 1, - "ep": 32, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 4096, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 32, - "tp": 1, - "ep": 32, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 8388608, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 32, - "tp": 1, - "ep": 32, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 8192, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 32, - "tp": 1, - "ep": 32, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 2097152, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 48, - "tp": 1, - "ep": 48, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 2048, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 48, - "tp": 1, - "ep": 48, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 4194304, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 48, - "tp": 1, - "ep": 48, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 4096, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 48, - "tp": 1, - "ep": 48, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 4194304, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 4, - "tp": 4, - "ep": 1, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 512, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 4, - "tp": 4, - "ep": 1, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 16777216, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 4, - "tp": 4, - "ep": 1, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 2048, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 4, - "tp": 4, - "ep": 1, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 33554432, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 4, - "tp": 4, - "ep": 1, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 512, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 48, - "tp": 1, - "ep": 48, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 16777216, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 48, - "tp": 1, - "ep": 48, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 33554432, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 48, - "tp": 1, - "ep": 48, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 16777216, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 32, - "tp": 1, - "ep": 32, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 3303424, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 2, - "tp": 1, - "ep": 2, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 6452, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 2, - "tp": 1, - "ep": 2, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 3303424, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 4, - "tp": 1, - "ep": 4, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 6452, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 4, - "tp": 1, - "ep": 4, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 340992, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 2, - "tp": 1, - "ep": 2, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 666, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 2, - "tp": 1, - "ep": 2, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 340992, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 32, - "tp": 1, - "ep": 32, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 12288, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 2, - "tp": 1, - "ep": 2, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 24, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 2, - "tp": 1, - "ep": 2, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 24576, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 2, - "tp": 1, - "ep": 2, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 48, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 2, - "tp": 1, - "ep": 2, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 49152, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 2, - "tp": 1, - "ep": 2, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 96, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 2, - "tp": 1, - "ep": 2, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 2307072, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 2, - "tp": 1, - "ep": 2, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 4506, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 2, - "tp": 1, - "ep": 2, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 2307072, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 16, - "tp": 1, - "ep": 16, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 4506, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 16, - "tp": 1, - "ep": 16, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 1258496, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 2, - "tp": 1, - "ep": 2, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 2458, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 2, - "tp": 1, - "ep": 2, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 1258496, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 32, - "tp": 1, - "ep": 32, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 2458, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 32, - "tp": 1, - "ep": 32, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 196608, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 2, - "tp": 1, - "ep": 2, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 192, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 2, - "tp": 1, - "ep": 2, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 1229, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 2, - "tp": 1, - "ep": 2, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 4404224, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 2, - "tp": 1, - "ep": 2, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 4301, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 2, - "tp": 1, - "ep": 2, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 2253, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 2, - "tp": 1, - "ep": 2, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 270336, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 2, - "tp": 1, - "ep": 2, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 66, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 2, - "tp": 1, - "ep": 2, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 270336, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 8, - "tp": 1, - "ep": 8, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 66, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 8, - "tp": 1, - "ep": 8, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 40960, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 2, - "tp": 1, - "ep": 2, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 1474560, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 2, - "tp": 1, - "ep": 2, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 360, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 2, - "tp": 1, - "ep": 2, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 2523136, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 2, - "tp": 1, - "ep": 2, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 616, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 2, - "tp": 1, - "ep": 2, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 2523136, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 32, - "tp": 1, - "ep": 32, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 616, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 32, - "tp": 1, - "ep": 32, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 18456576, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 2, - "tp": 1, - "ep": 2, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 18456576, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 8, - "tp": 1, - "ep": 8, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 4506, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 8, - "tp": 1, - "ep": 8, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 5455872, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 2, - "tp": 1, - "ep": 2, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 1332, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 2, - "tp": 1, - "ep": 2, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 5455872, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 16, - "tp": 1, - "ep": 16, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 1332, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 16, - "tp": 1, - "ep": 16, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 9232384, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 2, - "tp": 1, - "ep": 2, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 2254, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 2, - "tp": 1, - "ep": 2, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 9232384, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 16, - "tp": 1, - "ep": 16, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 2254, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 16, - "tp": 1, - "ep": 16, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 589824, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 2, - "tp": 1, - "ep": 2, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 72, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 2, - "tp": 1, - "ep": 2, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 589824, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 8, - "tp": 1, - "ep": 8, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 72, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 8, - "tp": 1, - "ep": 8, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 245760, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 2, - "tp": 1, - "ep": 2, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 30, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 2, - "tp": 1, - "ep": 2, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 10067968, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 2, - "tp": 1, - "ep": 2, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 1229, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 16, - "tp": 1, - "ep": 16, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 26443776, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 2, - "tp": 1, - "ep": 2, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 3228, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 2, - "tp": 1, - "ep": 2, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 26443776, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 4, - "tp": 1, - "ep": 4, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 3228, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 4, - "tp": 1, - "ep": 4, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 524288, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 48, - "tp": 1, - "ep": 48, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 8388608, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 48, - "tp": 1, - "ep": 48, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 8192, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 48, - "tp": 1, - "ep": 48, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 524288, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 4, - "tp": 4, - "ep": 1, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 64, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 4, - "tp": 4, - "ep": 1, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 8192, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "world_size": 4, - "tp": 1, - "ep": 4, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 16, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "world_size": 4, - "tp": 1, - "ep": 4, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 24576, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "world_size": 4, - "tp": 1, - "ep": 4, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 48, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "world_size": 4, - "tp": 1, - "ep": 4, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 24576, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "world_size": 8, - "tp": 1, - "ep": 8, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 48, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "world_size": 8, - "tp": 1, - "ep": 8, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 184320, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "world_size": 4, - "tp": 1, - "ep": 4, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 360, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "world_size": 4, - "tp": 1, - "ep": 4, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 184320, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "world_size": 32, - "tp": 1, - "ep": 32, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 360, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "world_size": 32, - "tp": 1, - "ep": 32, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 577536, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "world_size": 4, - "tp": 1, - "ep": 4, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 1128, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "world_size": 4, - "tp": 1, - "ep": 4, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 577536, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "world_size": 32, - "tp": 1, - "ep": 32, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 1128, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "world_size": 32, - "tp": 1, - "ep": 32, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 681984, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "world_size": 4, - "tp": 1, - "ep": 4, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 1332, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "world_size": 4, - "tp": 1, - "ep": 4, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 681984, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "world_size": 16, - "tp": 1, - "ep": 16, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 2307072, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "world_size": 4, - "tp": 1, - "ep": 4, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 4506, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "world_size": 4, - "tp": 1, - "ep": 4, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 2307072, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "world_size": 16, - "tp": 1, - "ep": 16, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 4506, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "world_size": 16, - "tp": 1, - "ep": 16, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 8388608, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "world_size": 4, - "tp": 1, - "ep": 4, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 16384, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "world_size": 4, - "tp": 1, - "ep": 4, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 4096, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "world_size": 4, - "tp": 1, - "ep": 4, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 4, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "world_size": 4, - "tp": 1, - "ep": 4, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 24, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "world_size": 4, - "tp": 1, - "ep": 4, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 24, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "world_size": 8, - "tp": 1, - "ep": 8, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 86016, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "world_size": 4, - "tp": 1, - "ep": 4, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 84, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "world_size": 4, - "tp": 1, - "ep": 4, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 86016, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "world_size": 8, - "tp": 1, - "ep": 8, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 84, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "world_size": 8, - "tp": 1, - "ep": 8, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 1258496, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "world_size": 4, - "tp": 1, - "ep": 4, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 1229, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "world_size": 4, - "tp": 1, - "ep": 4, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 1258496, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "world_size": 32, - "tp": 1, - "ep": 32, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 1229, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "world_size": 32, - "tp": 1, - "ep": 32, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 2253, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "world_size": 4, - "tp": 1, - "ep": 4, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 2253, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "world_size": 16, - "tp": 1, - "ep": 16, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 8808448, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "world_size": 4, - "tp": 1, - "ep": 4, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 8602, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "world_size": 4, - "tp": 1, - "ep": 4, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 8808448, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "world_size": 8, - "tp": 1, - "ep": 8, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 12582912, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "world_size": 4, - "tp": 1, - "ep": 4, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 12288, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "world_size": 4, - "tp": 1, - "ep": 4, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 12582912, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "world_size": 8, - "tp": 1, - "ep": 8, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 12288, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "world_size": 8, - "tp": 1, - "ep": 8, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 65536, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "world_size": 4, - "tp": 1, - "ep": 4, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 196608, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "world_size": 4, - "tp": 1, - "ep": 4, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 196608, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "world_size": 8, - "tp": 1, - "ep": 8, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 2727936, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "world_size": 4, - "tp": 1, - "ep": 4, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 666, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "world_size": 4, - "tp": 1, - "ep": 4, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 5455872, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "world_size": 4, - "tp": 1, - "ep": 4, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 10067968, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "world_size": 4, - "tp": 1, - "ep": 4, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 2458, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "world_size": 4, - "tp": 1, - "ep": 4, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 32768, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "world_size": 4, - "tp": 1, - "ep": 4, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 294912, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "world_size": 4, - "tp": 1, - "ep": 4, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 36, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "world_size": 4, - "tp": 1, - "ep": 4, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 294912, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "world_size": 8, - "tp": 1, - "ep": 8, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 4194304, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "world_size": 4, - "tp": 1, - "ep": 4, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 512, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "world_size": 4, - "tp": 1, - "ep": 4, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 17620992, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "world_size": 4, - "tp": 1, - "ep": 4, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 2151, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "world_size": 4, - "tp": 1, - "ep": 4, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 17620992, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "world_size": 8, - "tp": 1, - "ep": 8, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 1024, - "hidden": 2880, - "intermediate": 2880, - "num_experts": 128, - "top_k": 4, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 1, - "tp": 1, - "ep": 1, - "n_shared_experts": 0 - }, - "name": "gptoss" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 1, - "hidden": 2880, - "intermediate": 2880, - "num_experts": 128, - "top_k": 4, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 1, - "tp": 1, - "ep": 1, - "n_shared_experts": 0 - }, - "name": "gptoss" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 2048, - "hidden": 2880, - "intermediate": 2880, - "num_experts": 128, - "top_k": 4, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 1, - "tp": 1, - "ep": 1, - "n_shared_experts": 0 - }, - "name": "gptoss" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 2, - "hidden": 2880, - "intermediate": 2880, - "num_experts": 128, - "top_k": 4, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 1, - "tp": 1, - "ep": 1, - "n_shared_experts": 0 - }, - "name": "gptoss" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 1024, - "hidden": 2880, - "intermediate": 2880, - "num_experts": 128, - "top_k": 4, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 4, - "tp": 4, - "ep": 1, - "n_shared_experts": 0 - }, - "name": "gptoss" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 1, - "hidden": 2880, - "intermediate": 2880, - "num_experts": 128, - "top_k": 4, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 4, - "tp": 4, - "ep": 1, - "n_shared_experts": 0 - }, - "name": "gptoss" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 2048, - "hidden": 2880, - "intermediate": 2880, - "num_experts": 128, - "top_k": 4, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 4, - "tp": 4, - "ep": 1, - "n_shared_experts": 0 - }, - "name": "gptoss" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 2, - "hidden": 2880, - "intermediate": 2880, - "num_experts": 128, - "top_k": 4, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 4, - "tp": 4, - "ep": 1, - "n_shared_experts": 0 - }, - "name": "gptoss" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 128, - "hidden": 2880, - "intermediate": 2880, - "num_experts": 128, - "top_k": 4, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 4, - "tp": 4, - "ep": 1, - "n_shared_experts": 0 - }, - "name": "gptoss" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 512, - "hidden": 2880, - "intermediate": 2880, - "num_experts": 128, - "top_k": 4, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 1, - "tp": 1, - "ep": 1, - "n_shared_experts": 0 - }, - "name": "gptoss" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 2621440, - "hidden": 2880, - "intermediate": 2880, - "num_experts": 128, - "top_k": 4, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 1, - "tp": 1, - "ep": 1, - "n_shared_experts": 0 - }, - "name": "gptoss" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 2560, - "hidden": 2880, - "intermediate": 2880, - "num_experts": 128, - "top_k": 4, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 1, - "tp": 1, - "ep": 1, - "n_shared_experts": 0 - }, - "name": "gptoss" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 512, - "hidden": 2880, - "intermediate": 2880, - "num_experts": 128, - "top_k": 4, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 2, - "tp": 1, - "ep": 2, - "n_shared_experts": 0 - }, - "name": "gptoss" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 1048576, - "hidden": 2880, - "intermediate": 2880, - "num_experts": 128, - "top_k": 4, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 2, - "tp": 1, - "ep": 2, - "n_shared_experts": 0 - }, - "name": "gptoss" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 1024, - "hidden": 2880, - "intermediate": 2880, - "num_experts": 128, - "top_k": 4, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 2, - "tp": 1, - "ep": 2, - "n_shared_experts": 0 - }, - "name": "gptoss" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 2097152, - "hidden": 2880, - "intermediate": 2880, - "num_experts": 128, - "top_k": 4, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 2, - "tp": 1, - "ep": 2, - "n_shared_experts": 0 - }, - "name": "gptoss" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 2048, - "hidden": 2880, - "intermediate": 2880, - "num_experts": 128, - "top_k": 4, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 2, - "tp": 1, - "ep": 2, - "n_shared_experts": 0 - }, - "name": "gptoss" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 2621440, - "hidden": 2880, - "intermediate": 2880, - "num_experts": 128, - "top_k": 4, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 2, - "tp": 1, - "ep": 2, - "n_shared_experts": 0 - }, - "name": "gptoss" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 2560, - "hidden": 2880, - "intermediate": 2880, - "num_experts": 128, - "top_k": 4, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 2, - "tp": 1, - "ep": 2, - "n_shared_experts": 0 - }, - "name": "gptoss" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 1572864, - "hidden": 2880, - "intermediate": 2880, - "num_experts": 128, - "top_k": 4, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 1, - "tp": 1, - "ep": 1, - "n_shared_experts": 0 - }, - "name": "gptoss" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 1536, - "hidden": 2880, - "intermediate": 2880, - "num_experts": 128, - "top_k": 4, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 1, - "tp": 1, - "ep": 1, - "n_shared_experts": 0 - }, - "name": "gptoss" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 256, - "hidden": 2880, - "intermediate": 2880, - "num_experts": 128, - "top_k": 4, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 4, - "tp": 1, - "ep": 4, - "n_shared_experts": 0 - }, - "name": "gptoss" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 1048576, - "hidden": 2880, - "intermediate": 2880, - "num_experts": 128, - "top_k": 4, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 4, - "tp": 1, - "ep": 4, - "n_shared_experts": 0 - }, - "name": "gptoss" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 1024, - "hidden": 2880, - "intermediate": 2880, - "num_experts": 128, - "top_k": 4, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 4, - "tp": 1, - "ep": 4, - "n_shared_experts": 0 - }, - "name": "gptoss" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 1572864, - "hidden": 2880, - "intermediate": 2880, - "num_experts": 128, - "top_k": 4, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 4, - "tp": 1, - "ep": 4, - "n_shared_experts": 0 - }, - "name": "gptoss" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 1536, - "hidden": 2880, - "intermediate": 2880, - "num_experts": 128, - "top_k": 4, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 4, - "tp": 1, - "ep": 4, - "n_shared_experts": 0 - }, - "name": "gptoss" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 3145728, - "hidden": 2880, - "intermediate": 2880, - "num_experts": 128, - "top_k": 4, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 1, - "tp": 1, - "ep": 1, - "n_shared_experts": 0 - }, - "name": "gptoss" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 3072, - "hidden": 2880, - "intermediate": 2880, - "num_experts": 128, - "top_k": 4, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 1, - "tp": 1, - "ep": 1, - "n_shared_experts": 0 - }, - "name": "gptoss" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 3145728, - "hidden": 2880, - "intermediate": 2880, - "num_experts": 128, - "top_k": 4, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 4, - "tp": 1, - "ep": 4, - "n_shared_experts": 0 - }, - "name": "gptoss" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 3072, - "hidden": 2880, - "intermediate": 2880, - "num_experts": 128, - "top_k": 4, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 4, - "tp": 1, - "ep": 4, - "n_shared_experts": 0 - }, - "name": "gptoss" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 1, - "hidden": 2880, - "intermediate": 2880, - "num_experts": 128, - "top_k": 4, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 8, - "tp": 8, - "ep": 1, - "n_shared_experts": 0 - }, - "name": "gptoss" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 4194304, - "hidden": 2880, - "intermediate": 2880, - "num_experts": 128, - "top_k": 4, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 1, - "tp": 1, - "ep": 1, - "n_shared_experts": 0 - }, - "name": "gptoss" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 1048576, - "hidden": 2880, - "intermediate": 2880, - "num_experts": 128, - "top_k": 4, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 4, - "tp": 4, - "ep": 1, - "n_shared_experts": 0 - }, - "name": "gptoss" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 4194304, - "hidden": 2880, - "intermediate": 2880, - "num_experts": 128, - "top_k": 4, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 4, - "tp": 4, - "ep": 1, - "n_shared_experts": 0 - }, - "name": "gptoss" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 512, - "hidden": 2880, - "intermediate": 2880, - "num_experts": 128, - "top_k": 4, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 4, - "tp": 4, - "ep": 1, - "n_shared_experts": 0 - }, - "name": "gptoss" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 384, - "hidden": 2880, - "intermediate": 2880, - "num_experts": 128, - "top_k": 4, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 1, - "tp": 1, - "ep": 1, - "n_shared_experts": 0 - }, - "name": "gptoss" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 3145728, - "hidden": 2880, - "intermediate": 2880, - "num_experts": 128, - "top_k": 4, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 2, - "tp": 2, - "ep": 1, - "n_shared_experts": 0 - }, - "name": "gptoss" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 384, - "hidden": 2880, - "intermediate": 2880, - "num_experts": 128, - "top_k": 4, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 2, - "tp": 2, - "ep": 1, - "n_shared_experts": 0 - }, - "name": "gptoss" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 128, - "hidden": 2880, - "intermediate": 2880, - "num_experts": 128, - "top_k": 4, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 2, - "tp": 1, - "ep": 2, - "n_shared_experts": 0 - }, - "name": "gptoss" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 4194304, - "hidden": 2880, - "intermediate": 2880, - "num_experts": 128, - "top_k": 4, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 2, - "tp": 1, - "ep": 2, - "n_shared_experts": 0 - }, - "name": "gptoss" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 2048, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "world_size": 8, - "tp": 8, - "ep": 1, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 2, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "world_size": 8, - "tp": 8, - "ep": 1, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 65536, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 8, - "tp": 8, - "ep": 1, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 16, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 8, - "tp": 8, - "ep": 1, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 524288, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 8, - "tp": 8, - "ep": 1, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 64, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 8, - "tp": 8, - "ep": 1, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 1024, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 4, - "tp": 1, - "ep": 4, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 16777216, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 4, - "tp": 1, - "ep": 4, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 1024, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 8, - "tp": 1, - "ep": 8, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 288, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "world_size": 8, - "tp": 1, - "ep": 8, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 160, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "world_size": 8, - "tp": 8, - "ep": 1, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 1310720, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "world_size": 8, - "tp": 1, - "ep": 8, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 160, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "world_size": 8, - "tp": 1, - "ep": 8, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 8388608, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "world_size": 8, - "tp": 8, - "ep": 1, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 320, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "world_size": 8, - "tp": 1, - "ep": 8, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 128, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 4, - "tp": 4, - "ep": 1, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 32, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 8, - "tp": 8, - "ep": 1, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 128, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 8, - "tp": 8, - "ep": 1, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 4096, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 384, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 4, - "tp": 1, - "ep": 4, - "n_shared_experts": 1 - }, - "name": "kimik2.5" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 4, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 384, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 4, - "tp": 1, - "ep": 4, - "n_shared_experts": 1 - }, - "name": "kimik2.5" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 196608, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 384, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 4, - "tp": 1, - "ep": 4, - "n_shared_experts": 1 - }, - "name": "kimik2.5" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 192, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 384, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 4, - "tp": 1, - "ep": 4, - "n_shared_experts": 1 - }, - "name": "kimik2.5" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 368640, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 384, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 4, - "tp": 1, - "ep": 4, - "n_shared_experts": 1 - }, - "name": "kimik2.5" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 360, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 384, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 4, - "tp": 1, - "ep": 4, - "n_shared_experts": 1 - }, - "name": "kimik2.5" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 684032, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 384, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 4, - "tp": 1, - "ep": 4, - "n_shared_experts": 1 - }, - "name": "kimik2.5" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 668, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 384, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 4, - "tp": 1, - "ep": 4, - "n_shared_experts": 1 - }, - "name": "kimik2.5" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 4096, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 384, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 8, - "tp": 1, - "ep": 8, - "n_shared_experts": 1 - }, - "name": "kimik2.5" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 4, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 384, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 8, - "tp": 1, - "ep": 8, - "n_shared_experts": 1 - }, - "name": "kimik2.5" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 196608, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 384, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 8, - "tp": 1, - "ep": 8, - "n_shared_experts": 1 - }, - "name": "kimik2.5" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 192, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 384, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 8, - "tp": 1, - "ep": 8, - "n_shared_experts": 1 - }, - "name": "kimik2.5" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 368640, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 384, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 8, - "tp": 1, - "ep": 8, - "n_shared_experts": 1 - }, - "name": "kimik2.5" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 360, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 384, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 8, - "tp": 1, - "ep": 8, - "n_shared_experts": 1 - }, - "name": "kimik2.5" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 684032, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 384, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 8, - "tp": 1, - "ep": 8, - "n_shared_experts": 1 - }, - "name": "kimik2.5" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 668, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 384, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 8, - "tp": 1, - "ep": 8, - "n_shared_experts": 1 - }, - "name": "kimik2.5" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 5120, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 384, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 4, - "tp": 1, - "ep": 4, - "n_shared_experts": 1 - }, - "name": "kimik2.5" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 5, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 384, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 4, - "tp": 1, - "ep": 4, - "n_shared_experts": 1 - }, - "name": "kimik2.5" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 15360, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 384, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 4, - "tp": 1, - "ep": 4, - "n_shared_experts": 1 - }, - "name": "kimik2.5" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 15, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 384, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 4, - "tp": 1, - "ep": 4, - "n_shared_experts": 1 - }, - "name": "kimik2.5" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 30720, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 384, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 4, - "tp": 1, - "ep": 4, - "n_shared_experts": 1 - }, - "name": "kimik2.5" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 30, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 384, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 4, - "tp": 1, - "ep": 4, - "n_shared_experts": 1 - }, - "name": "kimik2.5" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 56320, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 384, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 4, - "tp": 1, - "ep": 4, - "n_shared_experts": 1 - }, - "name": "kimik2.5" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 55, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 384, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 4, - "tp": 1, - "ep": 4, - "n_shared_experts": 1 - }, - "name": "kimik2.5" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 681984, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 384, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 4, - "tp": 1, - "ep": 4, - "n_shared_experts": 1 - }, - "name": "kimik2.5" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 666, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 384, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 4, - "tp": 1, - "ep": 4, - "n_shared_experts": 1 - }, - "name": "kimik2.5" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 681984, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 384, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 16, - "tp": 1, - "ep": 16, - "n_shared_experts": 1 - }, - "name": "kimik2.5" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 666, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 384, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 16, - "tp": 1, - "ep": 16, - "n_shared_experts": 1 - }, - "name": "kimik2.5" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 2307072, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 384, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 4, - "tp": 1, - "ep": 4, - "n_shared_experts": 1 - }, - "name": "kimik2.5" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 2253, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 384, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 4, - "tp": 1, - "ep": 4, - "n_shared_experts": 1 - }, - "name": "kimik2.5" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 2307072, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 384, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 32, - "tp": 1, - "ep": 32, - "n_shared_experts": 1 - }, - "name": "kimik2.5" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 2253, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 384, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 32, - "tp": 1, - "ep": 32, - "n_shared_experts": 1 - }, - "name": "kimik2.5" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 4404224, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 384, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 4, - "tp": 1, - "ep": 4, - "n_shared_experts": 1 - }, - "name": "kimik2.5" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 4301, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 384, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 4, - "tp": 1, - "ep": 4, - "n_shared_experts": 1 - }, - "name": "kimik2.5" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 6606848, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 384, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 4, - "tp": 1, - "ep": 4, - "n_shared_experts": 1 - }, - "name": "kimik2.5" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 6452, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 384, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 4, - "tp": 1, - "ep": 4, - "n_shared_experts": 1 - }, - "name": "kimik2.5" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 4404224, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 384, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 8, - "tp": 1, - "ep": 8, - "n_shared_experts": 1 - }, - "name": "kimik2.5" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 4301, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 384, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 8, - "tp": 1, - "ep": 8, - "n_shared_experts": 1 - }, - "name": "kimik2.5" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 6606848, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 384, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 8, - "tp": 1, - "ep": 8, - "n_shared_experts": 1 - }, - "name": "kimik2.5" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 6452, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 384, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 8, - "tp": 1, - "ep": 8, - "n_shared_experts": 1 - }, - "name": "kimik2.5" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 4404224, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 384, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 16, - "tp": 1, - "ep": 16, - "n_shared_experts": 1 - }, - "name": "kimik2.5" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 4301, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 384, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 16, - "tp": 1, - "ep": 16, - "n_shared_experts": 1 - }, - "name": "kimik2.5" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 4404224, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 384, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 32, - "tp": 1, - "ep": 32, - "n_shared_experts": 1 - }, - "name": "kimik2.5" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 4301, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 384, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 32, - "tp": 1, - "ep": 32, - "n_shared_experts": 1 - }, - "name": "kimik2.5" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 32768, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 384, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 4, - "tp": 1, - "ep": 4, - "n_shared_experts": 1 - }, - "name": "kimik2.5" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 32768, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 384, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 8, - "tp": 1, - "ep": 8, - "n_shared_experts": 1 - }, - "name": "kimik2.5" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 1277952, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 384, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 4, - "tp": 1, - "ep": 4, - "n_shared_experts": 1 - }, - "name": "kimik2.5" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 156, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 384, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 4, - "tp": 1, - "ep": 4, - "n_shared_experts": 1 - }, - "name": "kimik2.5" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 40960, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 384, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 4, - "tp": 1, - "ep": 4, - "n_shared_experts": 1 - }, - "name": "kimik2.5" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 122880, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 384, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 4, - "tp": 1, - "ep": 4, - "n_shared_experts": 1 - }, - "name": "kimik2.5" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 245760, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 384, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 4, - "tp": 1, - "ep": 4, - "n_shared_experts": 1 - }, - "name": "kimik2.5" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 491520, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 384, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 4, - "tp": 1, - "ep": 4, - "n_shared_experts": 1 - }, - "name": "kimik2.5" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 60, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 384, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 4, - "tp": 1, - "ep": 4, - "n_shared_experts": 1 - }, - "name": "kimik2.5" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 860160, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 384, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 4, - "tp": 1, - "ep": 4, - "n_shared_experts": 1 - }, - "name": "kimik2.5" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 105, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 384, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 4, - "tp": 1, - "ep": 4, - "n_shared_experts": 1 - }, - "name": "kimik2.5" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 2727936, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 384, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 4, - "tp": 1, - "ep": 4, - "n_shared_experts": 1 - }, - "name": "kimik2.5" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 333, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 384, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 4, - "tp": 1, - "ep": 4, - "n_shared_experts": 1 - }, - "name": "kimik2.5" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 2727936, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 384, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 16, - "tp": 1, - "ep": 16, - "n_shared_experts": 1 - }, - "name": "kimik2.5" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 333, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 384, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 16, - "tp": 1, - "ep": 16, - "n_shared_experts": 1 - }, - "name": "kimik2.5" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 5038080, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 384, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 4, - "tp": 1, - "ep": 4, - "n_shared_experts": 1 - }, - "name": "kimik2.5" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 615, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 384, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 4, - "tp": 1, - "ep": 4, - "n_shared_experts": 1 - }, - "name": "kimik2.5" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 5038080, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 384, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 16, - "tp": 1, - "ep": 16, - "n_shared_experts": 1 - }, - "name": "kimik2.5" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 615, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 384, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 16, - "tp": 1, - "ep": 16, - "n_shared_experts": 1 - }, - "name": "kimik2.5" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 17620992, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 384, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 4, - "tp": 1, - "ep": 4, - "n_shared_experts": 1 - }, - "name": "kimik2.5" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 2151, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 384, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 4, - "tp": 1, - "ep": 4, - "n_shared_experts": 1 - }, - "name": "kimik2.5" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 17620992, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 384, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 8, - "tp": 1, - "ep": 8, - "n_shared_experts": 1 - }, - "name": "kimik2.5" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 2151, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 384, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 8, - "tp": 1, - "ep": 8, - "n_shared_experts": 1 - }, - "name": "kimik2.5" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 18456576, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 384, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 4, - "tp": 1, - "ep": 4, - "n_shared_experts": 1 - }, - "name": "kimik2.5" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 18456576, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 384, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 16, - "tp": 1, - "ep": 16, - "n_shared_experts": 1 - }, - "name": "kimik2.5" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 2253, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 384, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 16, - "tp": 1, - "ep": 16, - "n_shared_experts": 1 - }, - "name": "kimik2.5" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 262144, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 384, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 4, - "tp": 1, - "ep": 4, - "n_shared_experts": 1 - }, - "name": "kimik2.5" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 256, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 384, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 4, - "tp": 1, - "ep": 4, - "n_shared_experts": 1 - }, - "name": "kimik2.5" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 524288, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 384, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 4, - "tp": 1, - "ep": 4, - "n_shared_experts": 1 - }, - "name": "kimik2.5" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 512, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 384, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 4, - "tp": 1, - "ep": 4, - "n_shared_experts": 1 - }, - "name": "kimik2.5" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 1048576, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 384, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 4, - "tp": 1, - "ep": 4, - "n_shared_experts": 1 - }, - "name": "kimik2.5" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 1024, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 384, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 4, - "tp": 1, - "ep": 4, - "n_shared_experts": 1 - }, - "name": "kimik2.5" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 2097152, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 384, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 4, - "tp": 1, - "ep": 4, - "n_shared_experts": 1 - }, - "name": "kimik2.5" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 2048, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 384, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 4, - "tp": 1, - "ep": 4, - "n_shared_experts": 1 - }, - "name": "kimik2.5" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 3145728, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 384, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 4, - "tp": 1, - "ep": 4, - "n_shared_experts": 1 - }, - "name": "kimik2.5" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 3072, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 384, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 4, - "tp": 1, - "ep": 4, - "n_shared_experts": 1 - }, - "name": "kimik2.5" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 4194304, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 384, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 4, - "tp": 1, - "ep": 4, - "n_shared_experts": 1 - }, - "name": "kimik2.5" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 262144, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 384, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 16, - "tp": 1, - "ep": 16, - "n_shared_experts": 1 - }, - "name": "kimik2.5" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 256, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 384, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 16, - "tp": 1, - "ep": 16, - "n_shared_experts": 1 - }, - "name": "kimik2.5" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 524288, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 384, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 16, - "tp": 1, - "ep": 16, - "n_shared_experts": 1 - }, - "name": "kimik2.5" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 512, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 384, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 16, - "tp": 1, - "ep": 16, - "n_shared_experts": 1 - }, - "name": "kimik2.5" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 1048576, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 384, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 16, - "tp": 1, - "ep": 16, - "n_shared_experts": 1 - }, - "name": "kimik2.5" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 1024, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 384, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 16, - "tp": 1, - "ep": 16, - "n_shared_experts": 1 - }, - "name": "kimik2.5" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 2097152, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 384, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 16, - "tp": 1, - "ep": 16, - "n_shared_experts": 1 - }, - "name": "kimik2.5" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 2048, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 384, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 16, - "tp": 1, - "ep": 16, - "n_shared_experts": 1 - }, - "name": "kimik2.5" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 3145728, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 384, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 16, - "tp": 1, - "ep": 16, - "n_shared_experts": 1 - }, - "name": "kimik2.5" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 3072, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 384, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 16, - "tp": 1, - "ep": 16, - "n_shared_experts": 1 - }, - "name": "kimik2.5" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 4194304, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 384, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 16, - "tp": 1, - "ep": 16, - "n_shared_experts": 1 - }, - "name": "kimik2.5" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 4096, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 384, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 16, - "tp": 1, - "ep": 16, - "n_shared_experts": 1 - }, - "name": "kimik2.5" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 8192, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 384, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 4, - "tp": 1, - "ep": 4, - "n_shared_experts": 1 - }, - "name": "kimik2.5" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 8, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 384, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 4, - "tp": 1, - "ep": 4, - "n_shared_experts": 1 - }, - "name": "kimik2.5" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 16384, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 384, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 4, - "tp": 1, - "ep": 4, - "n_shared_experts": 1 - }, - "name": "kimik2.5" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 16, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 384, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 4, - "tp": 1, - "ep": 4, - "n_shared_experts": 1 - }, - "name": "kimik2.5" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 32, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 384, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 4, - "tp": 1, - "ep": 4, - "n_shared_experts": 1 - }, - "name": "kimik2.5" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 65536, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 384, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 4, - "tp": 1, - "ep": 4, - "n_shared_experts": 1 - }, - "name": "kimik2.5" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 64, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 384, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 4, - "tp": 1, - "ep": 4, - "n_shared_experts": 1 - }, - "name": "kimik2.5" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 131072, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 384, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 4, - "tp": 1, - "ep": 4, - "n_shared_experts": 1 - }, - "name": "kimik2.5" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 128, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 384, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 4, - "tp": 1, - "ep": 4, - "n_shared_experts": 1 - }, - "name": "kimik2.5" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 8388608, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 384, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 4, - "tp": 1, - "ep": 4, - "n_shared_experts": 1 - }, - "name": "kimik2.5" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 8388608, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 384, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 16, - "tp": 1, - "ep": 16, - "n_shared_experts": 1 - }, - "name": "kimik2.5" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 16777216, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 384, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 4, - "tp": 1, - "ep": 4, - "n_shared_experts": 1 - }, - "name": "kimik2.5" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 16777216, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 384, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 8, - "tp": 1, - "ep": 8, - "n_shared_experts": 1 - }, - "name": "kimik2.5" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 2048, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 384, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 8, - "tp": 1, - "ep": 8, - "n_shared_experts": 1 - }, - "name": "kimik2.5" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 25165824, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 384, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 4, - "tp": 1, - "ep": 4, - "n_shared_experts": 1 - }, - "name": "kimik2.5" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 33554432, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 384, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 4, - "tp": 1, - "ep": 4, - "n_shared_experts": 1 - }, - "name": "kimik2.5" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 25165824, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 384, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 16, - "tp": 1, - "ep": 16, - "n_shared_experts": 1 - }, - "name": "kimik2.5" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 33554432, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 384, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 16, - "tp": 1, - "ep": 16, - "n_shared_experts": 1 - }, - "name": "kimik2.5" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 1, - "hidden": 7168, - "intermediate": 3072, - "num_experts": 384, - "top_k": 6, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 8, - "tp": 1, - "ep": 8, - "n_shared_experts": 1 - }, - "name": "dsv4" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 16, - "hidden": 7168, - "intermediate": 3072, - "num_experts": 384, - "top_k": 6, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 8, - "tp": 1, - "ep": 8, - "n_shared_experts": 1 - }, - "name": "dsv4" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 1048576, - "hidden": 7168, - "intermediate": 3072, - "num_experts": 384, - "top_k": 6, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 8, - "tp": 8, - "ep": 1, - "n_shared_experts": 1 - }, - "name": "dsv4" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 67108864, - "hidden": 7168, - "intermediate": 3072, - "num_experts": 384, - "top_k": 6, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 8, - "tp": 1, - "ep": 8, - "n_shared_experts": 1 - }, - "name": "dsv4" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 101130240, - "hidden": 7168, - "intermediate": 3072, - "num_experts": 384, - "top_k": 6, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 8, - "tp": 1, - "ep": 8, - "n_shared_experts": 1 - }, - "name": "dsv4" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 12345, - "hidden": 7168, - "intermediate": 3072, - "num_experts": 384, - "top_k": 6, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 8, - "tp": 1, - "ep": 8, - "n_shared_experts": 1 - }, - "name": "dsv4" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 2097152, - "hidden": 7168, - "intermediate": 3072, - "num_experts": 384, - "top_k": 6, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 8, - "tp": 8, - "ep": 1, - "n_shared_experts": 1 - }, - "name": "dsv4" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 256, - "hidden": 7168, - "intermediate": 3072, - "num_experts": 384, - "top_k": 6, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 8, - "tp": 8, - "ep": 1, - "n_shared_experts": 1 - }, - "name": "dsv4" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 4194304, - "hidden": 7168, - "intermediate": 3072, - "num_experts": 384, - "top_k": 6, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 8, - "tp": 8, - "ep": 1, - "n_shared_experts": 1 - }, - "name": "dsv4" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 512, - "hidden": 7168, - "intermediate": 3072, - "num_experts": 384, - "top_k": 6, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 8, - "tp": 8, - "ep": 1, - "n_shared_experts": 1 - }, - "name": "dsv4" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 1572864, - "hidden": 7168, - "intermediate": 3072, - "num_experts": 384, - "top_k": 6, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 4, - "tp": 1, - "ep": 4, - "n_shared_experts": 1 - }, - "name": "dsv4" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 192, - "hidden": 7168, - "intermediate": 3072, - "num_experts": 384, - "top_k": 6, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 4, - "tp": 1, - "ep": 4, - "n_shared_experts": 1 - }, - "name": "dsv4" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 1572864, - "hidden": 7168, - "intermediate": 3072, - "num_experts": 384, - "top_k": 6, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 4, - "tp": 4, - "ep": 1, - "n_shared_experts": 1 - }, - "name": "dsv4" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 192, - "hidden": 7168, - "intermediate": 3072, - "num_experts": 384, - "top_k": 6, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 4, - "tp": 4, - "ep": 1, - "n_shared_experts": 1 - }, - "name": "dsv4" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 147456, - "hidden": 7168, - "intermediate": 3072, - "num_experts": 384, - "top_k": 6, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 4, - "tp": 1, - "ep": 4, - "n_shared_experts": 1 - }, - "name": "dsv4" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 18, - "hidden": 7168, - "intermediate": 3072, - "num_experts": 384, - "top_k": 6, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 4, - "tp": 1, - "ep": 4, - "n_shared_experts": 1 - }, - "name": "dsv4" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 147456, - "hidden": 7168, - "intermediate": 3072, - "num_experts": 384, - "top_k": 6, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 4, - "tp": 4, - "ep": 1, - "n_shared_experts": 1 - }, - "name": "dsv4" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 18, - "hidden": 7168, - "intermediate": 3072, - "num_experts": 384, - "top_k": 6, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 4, - "tp": 4, - "ep": 1, - "n_shared_experts": 1 - }, - "name": "dsv4" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 33554432, - "hidden": 7168, - "intermediate": 3072, - "num_experts": 384, - "top_k": 6, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 4, - "tp": 1, - "ep": 4, - "n_shared_experts": 1 - }, - "name": "dsv4" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 4096, - "hidden": 7168, - "intermediate": 3072, - "num_experts": 384, - "top_k": 6, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 4, - "tp": 1, - "ep": 4, - "n_shared_experts": 1 - }, - "name": "dsv4" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 25165824, - "hidden": 7168, - "intermediate": 3072, - "num_experts": 384, - "top_k": 6, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 4, - "tp": 1, - "ep": 4, - "n_shared_experts": 1 - }, - "name": "dsv4" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 3072, - "hidden": 7168, - "intermediate": 3072, - "num_experts": 384, - "top_k": 6, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 4, - "tp": 1, - "ep": 4, - "n_shared_experts": 1 - }, - "name": "dsv4" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 25165824, - "hidden": 7168, - "intermediate": 3072, - "num_experts": 384, - "top_k": 6, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 16, - "tp": 1, - "ep": 16, - "n_shared_experts": 1 - }, - "name": "dsv4" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 3072, - "hidden": 7168, - "intermediate": 3072, - "num_experts": 384, - "top_k": 6, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 16, - "tp": 1, - "ep": 16, - "n_shared_experts": 1 - }, - "name": "dsv4" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 8388608, - "hidden": 7168, - "intermediate": 3072, - "num_experts": 384, - "top_k": 6, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 4, - "tp": 1, - "ep": 4, - "n_shared_experts": 1 - }, - "name": "dsv4" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 1024, - "hidden": 7168, - "intermediate": 3072, - "num_experts": 384, - "top_k": 6, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 4, - "tp": 1, - "ep": 4, - "n_shared_experts": 1 - }, - "name": "dsv4" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 8388608, - "hidden": 7168, - "intermediate": 3072, - "num_experts": 384, - "top_k": 6, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 16, - "tp": 1, - "ep": 16, - "n_shared_experts": 1 - }, - "name": "dsv4" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 1024, - "hidden": 7168, - "intermediate": 3072, - "num_experts": 384, - "top_k": 6, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 16, - "tp": 1, - "ep": 16, - "n_shared_experts": 1 - }, - "name": "dsv4" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 33554432, - "hidden": 7168, - "intermediate": 3072, - "num_experts": 384, - "top_k": 6, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 16, - "tp": 1, - "ep": 16, - "n_shared_experts": 1 - }, - "name": "dsv4" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 4096, - "hidden": 7168, - "intermediate": 3072, - "num_experts": 384, - "top_k": 6, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 16, - "tp": 1, - "ep": 16, - "n_shared_experts": 1 - }, - "name": "dsv4" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 67108864, - "hidden": 7168, - "intermediate": 3072, - "num_experts": 384, - "top_k": 6, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 4, - "tp": 1, - "ep": 4, - "n_shared_experts": 1 - }, - "name": "dsv4" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 8192, - "hidden": 7168, - "intermediate": 3072, - "num_experts": 384, - "top_k": 6, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 4, - "tp": 1, - "ep": 4, - "n_shared_experts": 1 - }, - "name": "dsv4" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 67108864, - "hidden": 7168, - "intermediate": 3072, - "num_experts": 384, - "top_k": 6, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 16, - "tp": 1, - "ep": 16, - "n_shared_experts": 1 - }, - "name": "dsv4" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 8192, - "hidden": 7168, - "intermediate": 3072, - "num_experts": 384, - "top_k": 6, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 16, - "tp": 1, - "ep": 16, - "n_shared_experts": 1 - }, - "name": "dsv4" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 176160768, - "hidden": 7168, - "intermediate": 3072, - "num_experts": 384, - "top_k": 6, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 4, - "tp": 1, - "ep": 4, - "n_shared_experts": 1 - }, - "name": "dsv4" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 21504, - "hidden": 7168, - "intermediate": 3072, - "num_experts": 384, - "top_k": 6, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 4, - "tp": 1, - "ep": 4, - "n_shared_experts": 1 - }, - "name": "dsv4" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 176160768, - "hidden": 7168, - "intermediate": 3072, - "num_experts": 384, - "top_k": 6, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 12, - "tp": 1, - "ep": 12, - "n_shared_experts": 1 - }, - "name": "dsv4" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 21504, - "hidden": 7168, - "intermediate": 3072, - "num_experts": 384, - "top_k": 6, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 12, - "tp": 1, - "ep": 12, - "n_shared_experts": 1 - }, - "name": "dsv4" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 2097152, - "hidden": 7168, - "intermediate": 3072, - "num_experts": 384, - "top_k": 6, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 16, - "tp": 1, - "ep": 16, - "n_shared_experts": 1 - }, - "name": "dsv4" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 512, - "hidden": 7168, - "intermediate": 3072, - "num_experts": 384, - "top_k": 6, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 16, - "tp": 1, - "ep": 16, - "n_shared_experts": 1 - }, - "name": "dsv4" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 8192, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 8, - "tp": 8, - "ep": 1, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 16384, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 8, - "tp": 8, - "ep": 1, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 262144, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 8, - "tp": 8, - "ep": 1, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 1, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 8, - "tp": 8, - "ep": 1, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 2, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 8, - "tp": 8, - "ep": 1, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 98304, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 8, - "tp": 8, - "ep": 1, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 12, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 8, - "tp": 8, - "ep": 1, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 1048576, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 8, - "tp": 8, - "ep": 1, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 2097152, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 8, - "tp": 8, - "ep": 1, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 256, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 4, - "tp": 4, - "ep": 1, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 1048576, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 4, - "tp": 4, - "ep": 1, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 2097152, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 4, - "tp": 4, - "ep": 1, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 512, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 8, - "tp": 8, - "ep": 1, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 256, - "hidden": 4096, - "intermediate": 1024, - "num_experts": 512, - "top_k": 10, - "dtype_act": "bf16", - "dtype_weight": "bf16", - "world_size": 8, - "tp": 8, - "ep": 1, - "n_shared_experts": 1 - }, - "name": "qwen3.5" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 1048576, - "hidden": 4096, - "intermediate": 1024, - "num_experts": 512, - "top_k": 10, - "dtype_act": "bf16", - "dtype_weight": "bf16", - "world_size": 8, - "tp": 8, - "ep": 1, - "n_shared_experts": 1 - }, - "name": "qwen3.5" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 2097152, - "hidden": 4096, - "intermediate": 1024, - "num_experts": 512, - "top_k": 10, - "dtype_act": "bf16", - "dtype_weight": "bf16", - "world_size": 8, - "tp": 8, - "ep": 1, - "n_shared_experts": 1 - }, - "name": "qwen3.5" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 512, - "hidden": 4096, - "intermediate": 1024, - "num_experts": 512, - "top_k": 10, - "dtype_act": "bf16", - "dtype_weight": "bf16", - "world_size": 8, - "tp": 8, - "ep": 1, - "n_shared_experts": 1 - }, - "name": "qwen3.5" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 8192, - "hidden": 4096, - "intermediate": 1024, - "num_experts": 512, - "top_k": 10, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "world_size": 8, - "tp": 8, - "ep": 1, - "n_shared_experts": 1 - }, - "name": "qwen3.5" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 16384, - "hidden": 4096, - "intermediate": 1024, - "num_experts": 512, - "top_k": 10, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "world_size": 8, - "tp": 8, - "ep": 1, - "n_shared_experts": 1 - }, - "name": "qwen3.5" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 16, - "hidden": 4096, - "intermediate": 1024, - "num_experts": 512, - "top_k": 10, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "world_size": 8, - "tp": 8, - "ep": 1, - "n_shared_experts": 1 - }, - "name": "qwen3.5" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 32, - "hidden": 4096, - "intermediate": 1024, - "num_experts": 512, - "top_k": 10, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "world_size": 8, - "tp": 8, - "ep": 1, - "n_shared_experts": 1 - }, - "name": "qwen3.5" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 65536, - "hidden": 4096, - "intermediate": 1024, - "num_experts": 512, - "top_k": 10, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "world_size": 8, - "tp": 8, - "ep": 1, - "n_shared_experts": 1 - }, - "name": "qwen3.5" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 64, - "hidden": 4096, - "intermediate": 1024, - "num_experts": 512, - "top_k": 10, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "world_size": 8, - "tp": 8, - "ep": 1, - "n_shared_experts": 1 - }, - "name": "qwen3.5" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 131072, - "hidden": 4096, - "intermediate": 1024, - "num_experts": 512, - "top_k": 10, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "world_size": 8, - "tp": 8, - "ep": 1, - "n_shared_experts": 1 - }, - "name": "qwen3.5" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 262144, - "hidden": 4096, - "intermediate": 1024, - "num_experts": 512, - "top_k": 10, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "world_size": 8, - "tp": 8, - "ep": 1, - "n_shared_experts": 1 - }, - "name": "qwen3.5" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 524288, - "hidden": 4096, - "intermediate": 1024, - "num_experts": 512, - "top_k": 10, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "world_size": 8, - "tp": 8, - "ep": 1, - "n_shared_experts": 1 - }, - "name": "qwen3.5" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 131072, - "hidden": 4096, - "intermediate": 1024, - "num_experts": 512, - "top_k": 10, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "world_size": 2, - "tp": 1, - "ep": 2, - "n_shared_experts": 1 - }, - "name": "qwen3.5" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 128, - "hidden": 4096, - "intermediate": 1024, - "num_experts": 512, - "top_k": 10, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "world_size": 2, - "tp": 1, - "ep": 2, - "n_shared_experts": 1 - }, - "name": "qwen3.5" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 262144, - "hidden": 4096, - "intermediate": 1024, - "num_experts": 512, - "top_k": 10, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "world_size": 2, - "tp": 1, - "ep": 2, - "n_shared_experts": 1 - }, - "name": "qwen3.5" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 256, - "hidden": 4096, - "intermediate": 1024, - "num_experts": 512, - "top_k": 10, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "world_size": 2, - "tp": 1, - "ep": 2, - "n_shared_experts": 1 - }, - "name": "qwen3.5" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 32768, - "hidden": 4096, - "intermediate": 1024, - "num_experts": 512, - "top_k": 10, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "world_size": 2, - "tp": 1, - "ep": 2, - "n_shared_experts": 1 - }, - "name": "qwen3.5" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 4, - "hidden": 4096, - "intermediate": 1024, - "num_experts": 512, - "top_k": 10, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "world_size": 2, - "tp": 1, - "ep": 2, - "n_shared_experts": 1 - }, - "name": "qwen3.5" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 65536, - "hidden": 4096, - "intermediate": 1024, - "num_experts": 512, - "top_k": 10, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "world_size": 2, - "tp": 1, - "ep": 2, - "n_shared_experts": 1 - }, - "name": "qwen3.5" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 8, - "hidden": 4096, - "intermediate": 1024, - "num_experts": 512, - "top_k": 10, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "world_size": 2, - "tp": 1, - "ep": 2, - "n_shared_experts": 1 - }, - "name": "qwen3.5" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 16, - "hidden": 4096, - "intermediate": 1024, - "num_experts": 512, - "top_k": 10, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "world_size": 2, - "tp": 1, - "ep": 2, - "n_shared_experts": 1 - }, - "name": "qwen3.5" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 32, - "hidden": 4096, - "intermediate": 1024, - "num_experts": 512, - "top_k": 10, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "world_size": 2, - "tp": 1, - "ep": 2, - "n_shared_experts": 1 - }, - "name": "qwen3.5" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 512, - "hidden": 4096, - "intermediate": 1024, - "num_experts": 512, - "top_k": 10, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "world_size": 8, - "tp": 1, - "ep": 8, - "n_shared_experts": 1 - }, - "name": "qwen3.5" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 512, - "hidden": 4096, - "intermediate": 1024, - "num_experts": 512, - "top_k": 10, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "world_size": 2, - "tp": 1, - "ep": 2, - "n_shared_experts": 1 - }, - "name": "qwen3.5" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 64, - "hidden": 4096, - "intermediate": 1024, - "num_experts": 512, - "top_k": 10, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "world_size": 2, - "tp": 1, - "ep": 2, - "n_shared_experts": 1 - }, - "name": "qwen3.5" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 4096, - "hidden": 4096, - "intermediate": 1024, - "num_experts": 512, - "top_k": 10, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "world_size": 2, - "tp": 2, - "ep": 1, - "n_shared_experts": 1 - }, - "name": "qwen3.5" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 4, - "hidden": 4096, - "intermediate": 1024, - "num_experts": 512, - "top_k": 10, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "world_size": 2, - "tp": 2, - "ep": 1, - "n_shared_experts": 1 - }, - "name": "qwen3.5" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 8192, - "hidden": 4096, - "intermediate": 1024, - "num_experts": 512, - "top_k": 10, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "world_size": 2, - "tp": 2, - "ep": 1, - "n_shared_experts": 1 - }, - "name": "qwen3.5" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 8, - "hidden": 4096, - "intermediate": 1024, - "num_experts": 512, - "top_k": 10, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "world_size": 2, - "tp": 2, - "ep": 1, - "n_shared_experts": 1 - }, - "name": "qwen3.5" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 16384, - "hidden": 4096, - "intermediate": 1024, - "num_experts": 512, - "top_k": 10, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "world_size": 2, - "tp": 2, - "ep": 1, - "n_shared_experts": 1 - }, - "name": "qwen3.5" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 16, - "hidden": 4096, - "intermediate": 1024, - "num_experts": 512, - "top_k": 10, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "world_size": 2, - "tp": 2, - "ep": 1, - "n_shared_experts": 1 - }, - "name": "qwen3.5" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 32768, - "hidden": 4096, - "intermediate": 1024, - "num_experts": 512, - "top_k": 10, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "world_size": 2, - "tp": 2, - "ep": 1, - "n_shared_experts": 1 - }, - "name": "qwen3.5" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 32, - "hidden": 4096, - "intermediate": 1024, - "num_experts": 512, - "top_k": 10, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "world_size": 2, - "tp": 2, - "ep": 1, - "n_shared_experts": 1 - }, - "name": "qwen3.5" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 65536, - "hidden": 4096, - "intermediate": 1024, - "num_experts": 512, - "top_k": 10, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "world_size": 2, - "tp": 2, - "ep": 1, - "n_shared_experts": 1 - }, - "name": "qwen3.5" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 64, - "hidden": 4096, - "intermediate": 1024, - "num_experts": 512, - "top_k": 10, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "world_size": 2, - "tp": 2, - "ep": 1, - "n_shared_experts": 1 - }, - "name": "qwen3.5" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 131072, - "hidden": 4096, - "intermediate": 1024, - "num_experts": 512, - "top_k": 10, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "world_size": 2, - "tp": 2, - "ep": 1, - "n_shared_experts": 1 - }, - "name": "qwen3.5" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 128, - "hidden": 4096, - "intermediate": 1024, - "num_experts": 512, - "top_k": 10, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "world_size": 2, - "tp": 2, - "ep": 1, - "n_shared_experts": 1 - }, - "name": "qwen3.5" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 262144, - "hidden": 4096, - "intermediate": 1024, - "num_experts": 512, - "top_k": 10, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "world_size": 2, - "tp": 2, - "ep": 1, - "n_shared_experts": 1 - }, - "name": "qwen3.5" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 256, - "hidden": 4096, - "intermediate": 1024, - "num_experts": 512, - "top_k": 10, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "world_size": 2, - "tp": 2, - "ep": 1, - "n_shared_experts": 1 - }, - "name": "qwen3.5" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 128, - "hidden": 4096, - "intermediate": 1024, - "num_experts": 512, - "top_k": 10, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "world_size": 8, - "tp": 8, - "ep": 1, - "n_shared_experts": 1 - }, - "name": "qwen3.5" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 256, - "hidden": 4096, - "intermediate": 1024, - "num_experts": 512, - "top_k": 10, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "world_size": 8, - "tp": 8, - "ep": 1, - "n_shared_experts": 1 - }, - "name": "qwen3.5" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 524288, - "hidden": 4096, - "intermediate": 1024, - "num_experts": 512, - "top_k": 10, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "world_size": 2, - "tp": 2, - "ep": 1, - "n_shared_experts": 1 - }, - "name": "qwen3.5" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 1048576, - "hidden": 4096, - "intermediate": 1024, - "num_experts": 512, - "top_k": 10, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "world_size": 2, - "tp": 2, - "ep": 1, - "n_shared_experts": 1 - }, - "name": "qwen3.5" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 2097152, - "hidden": 4096, - "intermediate": 1024, - "num_experts": 512, - "top_k": 10, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "world_size": 2, - "tp": 2, - "ep": 1, - "n_shared_experts": 1 - }, - "name": "qwen3.5" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 1048576, - "hidden": 4096, - "intermediate": 1024, - "num_experts": 512, - "top_k": 10, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "world_size": 8, - "tp": 8, - "ep": 1, - "n_shared_experts": 1 - }, - "name": "qwen3.5" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 2097152, - "hidden": 4096, - "intermediate": 1024, - "num_experts": 512, - "top_k": 10, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "world_size": 8, - "tp": 8, - "ep": 1, - "n_shared_experts": 1 - }, - "name": "qwen3.5" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 512, - "hidden": 4096, - "intermediate": 1024, - "num_experts": 512, - "top_k": 10, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "world_size": 8, - "tp": 8, - "ep": 1, - "n_shared_experts": 1 - }, - "name": "qwen3.5" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 256, - "hidden": 4096, - "intermediate": 1024, - "num_experts": 512, - "top_k": 10, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 2, - "tp": 2, - "ep": 1, - "n_shared_experts": 1 - }, - "name": "qwen3.5" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 2097152, - "hidden": 4096, - "intermediate": 1024, - "num_experts": 512, - "top_k": 10, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 2, - "tp": 2, - "ep": 1, - "n_shared_experts": 1 - }, - "name": "qwen3.5" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 4096, - "hidden": 6144, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "world_size": 4, - "tp": 4, - "ep": 1, - "n_shared_experts": 1 - }, - "name": "glm5" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 8, - "hidden": 6144, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "world_size": 4, - "tp": 4, - "ep": 1, - "n_shared_experts": 1 - }, - "name": "glm5" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 8192, - "hidden": 6144, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "world_size": 4, - "tp": 4, - "ep": 1, - "n_shared_experts": 1 - }, - "name": "glm5" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 16, - "hidden": 6144, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "world_size": 4, - "tp": 4, - "ep": 1, - "n_shared_experts": 1 - }, - "name": "glm5" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 16384, - "hidden": 6144, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "world_size": 4, - "tp": 4, - "ep": 1, - "n_shared_experts": 1 - }, - "name": "glm5" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 32, - "hidden": 6144, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "world_size": 4, - "tp": 4, - "ep": 1, - "n_shared_experts": 1 - }, - "name": "glm5" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 32768, - "hidden": 6144, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "world_size": 4, - "tp": 4, - "ep": 1, - "n_shared_experts": 1 - }, - "name": "glm5" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 64, - "hidden": 6144, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "world_size": 4, - "tp": 4, - "ep": 1, - "n_shared_experts": 1 - }, - "name": "glm5" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 65536, - "hidden": 6144, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "world_size": 4, - "tp": 4, - "ep": 1, - "n_shared_experts": 1 - }, - "name": "glm5" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 128, - "hidden": 6144, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "world_size": 4, - "tp": 4, - "ep": 1, - "n_shared_experts": 1 - }, - "name": "glm5" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 131072, - "hidden": 6144, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "world_size": 4, - "tp": 4, - "ep": 1, - "n_shared_experts": 1 - }, - "name": "glm5" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 256, - "hidden": 6144, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "world_size": 4, - "tp": 4, - "ep": 1, - "n_shared_experts": 1 - }, - "name": "glm5" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 262144, - "hidden": 6144, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "world_size": 4, - "tp": 4, - "ep": 1, - "n_shared_experts": 1 - }, - "name": "glm5" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 524288, - "hidden": 6144, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "world_size": 4, - "tp": 4, - "ep": 1, - "n_shared_experts": 1 - }, - "name": "glm5" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 1048576, - "hidden": 6144, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "world_size": 4, - "tp": 4, - "ep": 1, - "n_shared_experts": 1 - }, - "name": "glm5" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 4, - "hidden": 6144, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "world_size": 4, - "tp": 4, - "ep": 1, - "n_shared_experts": 1 - }, - "name": "glm5" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 2097152, - "hidden": 6144, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "world_size": 4, - "tp": 4, - "ep": 1, - "n_shared_experts": 1 - }, - "name": "glm5" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 4096, - "hidden": 6144, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 2, - "tp": 2, - "ep": 1, - "n_shared_experts": 1 - }, - "name": "glm5.1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 4, - "hidden": 6144, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 2, - "tp": 2, - "ep": 1, - "n_shared_experts": 1 - }, - "name": "glm5.1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 8192, - "hidden": 6144, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 2, - "tp": 2, - "ep": 1, - "n_shared_experts": 1 - }, - "name": "glm5.1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 8, - "hidden": 6144, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 2, - "tp": 2, - "ep": 1, - "n_shared_experts": 1 - }, - "name": "glm5.1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 16384, - "hidden": 6144, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 2, - "tp": 2, - "ep": 1, - "n_shared_experts": 1 - }, - "name": "glm5.1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 16, - "hidden": 6144, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 2, - "tp": 2, - "ep": 1, - "n_shared_experts": 1 - }, - "name": "glm5.1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 32768, - "hidden": 6144, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 2, - "tp": 2, - "ep": 1, - "n_shared_experts": 1 - }, - "name": "glm5.1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 32, - "hidden": 6144, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 2, - "tp": 2, - "ep": 1, - "n_shared_experts": 1 - }, - "name": "glm5.1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 65536, - "hidden": 6144, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 2, - "tp": 2, - "ep": 1, - "n_shared_experts": 1 - }, - "name": "glm5.1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 64, - "hidden": 6144, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 2, - "tp": 2, - "ep": 1, - "n_shared_experts": 1 - }, - "name": "glm5.1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 131072, - "hidden": 6144, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 2, - "tp": 2, - "ep": 1, - "n_shared_experts": 1 - }, - "name": "glm5.1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 128, - "hidden": 6144, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 2, - "tp": 2, - "ep": 1, - "n_shared_experts": 1 - }, - "name": "glm5.1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 262144, - "hidden": 6144, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 2, - "tp": 2, - "ep": 1, - "n_shared_experts": 1 - }, - "name": "glm5.1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 256, - "hidden": 6144, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 2, - "tp": 2, - "ep": 1, - "n_shared_experts": 1 - }, - "name": "glm5.1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 524288, - "hidden": 6144, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 2, - "tp": 2, - "ep": 1, - "n_shared_experts": 1 - }, - "name": "glm5.1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 1048576, - "hidden": 6144, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 2, - "tp": 2, - "ep": 1, - "n_shared_experts": 1 - }, - "name": "glm5.1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 2097152, - "hidden": 6144, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 2, - "tp": 2, - "ep": 1, - "n_shared_experts": 1 - }, - "name": "glm5.1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 8192, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 384, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 8, - "tp": 8, - "ep": 1, - "n_shared_experts": 1 - }, - "name": "kimik2.5" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 8, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 384, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 8, - "tp": 8, - "ep": 1, - "n_shared_experts": 1 - }, - "name": "kimik2.5" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 16384, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 384, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 8, - "tp": 8, - "ep": 1, - "n_shared_experts": 1 - }, - "name": "kimik2.5" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 16, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 384, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 8, - "tp": 8, - "ep": 1, - "n_shared_experts": 1 - }, - "name": "kimik2.5" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 32, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 384, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 8, - "tp": 8, - "ep": 1, - "n_shared_experts": 1 - }, - "name": "kimik2.5" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 65536, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 384, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 8, - "tp": 8, - "ep": 1, - "n_shared_experts": 1 - }, - "name": "kimik2.5" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 64, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 384, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 8, - "tp": 8, - "ep": 1, - "n_shared_experts": 1 - }, - "name": "kimik2.5" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 131072, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 384, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 8, - "tp": 8, - "ep": 1, - "n_shared_experts": 1 - }, - "name": "kimik2.5" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 262144, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 384, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 8, - "tp": 8, - "ep": 1, - "n_shared_experts": 1 - }, - "name": "kimik2.5" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 524288, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 384, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 8, - "tp": 8, - "ep": 1, - "n_shared_experts": 1 - }, - "name": "kimik2.5" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 128, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 384, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 8, - "tp": 8, - "ep": 1, - "n_shared_experts": 1 - }, - "name": "kimik2.5" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 128, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 384, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 4, - "tp": 4, - "ep": 1, - "n_shared_experts": 1 - }, - "name": "kimik2.5" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 1048576, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 384, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 8, - "tp": 8, - "ep": 1, - "n_shared_experts": 1 - }, - "name": "kimik2.5" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 1048576, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 384, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 4, - "tp": 4, - "ep": 1, - "n_shared_experts": 1 - }, - "name": "kimik2.5" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 2048, - "hidden": 3072, - "intermediate": 1536, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "world_size": 2, - "tp": 1, - "ep": 2, - "n_shared_experts": 0 - }, - "name": "minimaxm2.5" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 2, - "hidden": 3072, - "intermediate": 1536, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "world_size": 2, - "tp": 1, - "ep": 2, - "n_shared_experts": 0 - }, - "name": "minimaxm2.5" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 4096, - "hidden": 3072, - "intermediate": 1536, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "world_size": 2, - "tp": 1, - "ep": 2, - "n_shared_experts": 0 - }, - "name": "minimaxm2.5" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 4, - "hidden": 3072, - "intermediate": 1536, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "world_size": 2, - "tp": 1, - "ep": 2, - "n_shared_experts": 0 - }, - "name": "minimaxm2.5" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 8192, - "hidden": 3072, - "intermediate": 1536, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "world_size": 2, - "tp": 1, - "ep": 2, - "n_shared_experts": 0 - }, - "name": "minimaxm2.5" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 8, - "hidden": 3072, - "intermediate": 1536, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "world_size": 2, - "tp": 1, - "ep": 2, - "n_shared_experts": 0 - }, - "name": "minimaxm2.5" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 16384, - "hidden": 3072, - "intermediate": 1536, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "world_size": 2, - "tp": 1, - "ep": 2, - "n_shared_experts": 0 - }, - "name": "minimaxm2.5" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 16, - "hidden": 3072, - "intermediate": 1536, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "world_size": 2, - "tp": 1, - "ep": 2, - "n_shared_experts": 0 - }, - "name": "minimaxm2.5" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 32768, - "hidden": 3072, - "intermediate": 1536, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "world_size": 2, - "tp": 1, - "ep": 2, - "n_shared_experts": 0 - }, - "name": "minimaxm2.5" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 32, - "hidden": 3072, - "intermediate": 1536, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "world_size": 2, - "tp": 1, - "ep": 2, - "n_shared_experts": 0 - }, - "name": "minimaxm2.5" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 65536, - "hidden": 3072, - "intermediate": 1536, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "world_size": 2, - "tp": 1, - "ep": 2, - "n_shared_experts": 0 - }, - "name": "minimaxm2.5" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 64, - "hidden": 3072, - "intermediate": 1536, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "world_size": 2, - "tp": 1, - "ep": 2, - "n_shared_experts": 0 - }, - "name": "minimaxm2.5" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 131072, - "hidden": 3072, - "intermediate": 1536, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "world_size": 2, - "tp": 1, - "ep": 2, - "n_shared_experts": 0 - }, - "name": "minimaxm2.5" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 128, - "hidden": 3072, - "intermediate": 1536, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "world_size": 2, - "tp": 1, - "ep": 2, - "n_shared_experts": 0 - }, - "name": "minimaxm2.5" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 262144, - "hidden": 3072, - "intermediate": 1536, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "world_size": 2, - "tp": 1, - "ep": 2, - "n_shared_experts": 0 - }, - "name": "minimaxm2.5" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 256, - "hidden": 3072, - "intermediate": 1536, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "world_size": 2, - "tp": 1, - "ep": 2, - "n_shared_experts": 0 - }, - "name": "minimaxm2.5" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 128, - "hidden": 3072, - "intermediate": 1536, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "world_size": 4, - "tp": 1, - "ep": 4, - "n_shared_experts": 0 - }, - "name": "minimaxm2.5" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 2048, - "hidden": 3072, - "intermediate": 1536, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "world_size": 8, - "tp": 1, - "ep": 8, - "n_shared_experts": 0 - }, - "name": "minimaxm2.5" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 2, - "hidden": 3072, - "intermediate": 1536, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "world_size": 8, - "tp": 1, - "ep": 8, - "n_shared_experts": 0 - }, - "name": "minimaxm2.5" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 2097152, - "hidden": 3072, - "intermediate": 1536, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "world_size": 2, - "tp": 1, - "ep": 2, - "n_shared_experts": 0 - }, - "name": "minimaxm2.5" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 1048576, - "hidden": 3072, - "intermediate": 1536, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "world_size": 4, - "tp": 1, - "ep": 4, - "n_shared_experts": 0 - }, - "name": "minimaxm2.5" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 2097152, - "hidden": 3072, - "intermediate": 1536, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "world_size": 4, - "tp": 1, - "ep": 4, - "n_shared_experts": 0 - }, - "name": "minimaxm2.5" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 4194304, - "hidden": 3072, - "intermediate": 1536, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "world_size": 4, - "tp": 1, - "ep": 4, - "n_shared_experts": 0 - }, - "name": "minimaxm2.5" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 16384, - "hidden": 3072, - "intermediate": 1536, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "world_size": 8, - "tp": 1, - "ep": 8, - "n_shared_experts": 0 - }, - "name": "minimaxm2.5" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 4096, - "hidden": 3072, - "intermediate": 1536, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "world_size": 2, - "tp": 2, - "ep": 1, - "n_shared_experts": 0 - }, - "name": "minimaxm2.5" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 8192, - "hidden": 3072, - "intermediate": 1536, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "world_size": 2, - "tp": 2, - "ep": 1, - "n_shared_experts": 0 - }, - "name": "minimaxm2.5" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 16384, - "hidden": 3072, - "intermediate": 1536, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "world_size": 2, - "tp": 2, - "ep": 1, - "n_shared_experts": 0 - }, - "name": "minimaxm2.5" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 512, - "hidden": 3072, - "intermediate": 1536, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 1, - "tp": 1, - "ep": 1, - "n_shared_experts": 0 - }, - "name": "minimaxm2.5" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 1024, - "hidden": 3072, - "intermediate": 1536, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 1, - "tp": 1, - "ep": 1, - "n_shared_experts": 0 - }, - "name": "minimaxm2.5" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 4096, - "hidden": 3072, - "intermediate": 1536, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 2, - "tp": 2, - "ep": 1, - "n_shared_experts": 0 - }, - "name": "minimaxm2.5" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 4, - "hidden": 3072, - "intermediate": 1536, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 2, - "tp": 2, - "ep": 1, - "n_shared_experts": 0 - }, - "name": "minimaxm2.5" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 8192, - "hidden": 3072, - "intermediate": 1536, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 2, - "tp": 2, - "ep": 1, - "n_shared_experts": 0 - }, - "name": "minimaxm2.5" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 8, - "hidden": 3072, - "intermediate": 1536, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 2, - "tp": 2, - "ep": 1, - "n_shared_experts": 0 - }, - "name": "minimaxm2.5" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 32768, - "hidden": 3072, - "intermediate": 1536, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 2, - "tp": 2, - "ep": 1, - "n_shared_experts": 0 - }, - "name": "minimaxm2.5" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 32, - "hidden": 3072, - "intermediate": 1536, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 2, - "tp": 2, - "ep": 1, - "n_shared_experts": 0 - }, - "name": "minimaxm2.5" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 65536, - "hidden": 3072, - "intermediate": 1536, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 2, - "tp": 2, - "ep": 1, - "n_shared_experts": 0 - }, - "name": "minimaxm2.5" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 64, - "hidden": 3072, - "intermediate": 1536, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 2, - "tp": 2, - "ep": 1, - "n_shared_experts": 0 - }, - "name": "minimaxm2.5" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 131072, - "hidden": 3072, - "intermediate": 1536, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 2, - "tp": 2, - "ep": 1, - "n_shared_experts": 0 - }, - "name": "minimaxm2.5" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 128, - "hidden": 3072, - "intermediate": 1536, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 2, - "tp": 2, - "ep": 1, - "n_shared_experts": 0 - }, - "name": "minimaxm2.5" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 262144, - "hidden": 3072, - "intermediate": 1536, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 2, - "tp": 2, - "ep": 1, - "n_shared_experts": 0 - }, - "name": "minimaxm2.5" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 256, - "hidden": 3072, - "intermediate": 1536, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 2, - "tp": 2, - "ep": 1, - "n_shared_experts": 0 - }, - "name": "minimaxm2.5" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 524288, - "hidden": 3072, - "intermediate": 1536, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 2, - "tp": 2, - "ep": 1, - "n_shared_experts": 0 - }, - "name": "minimaxm2.5" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 512, - "hidden": 3072, - "intermediate": 1536, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 2, - "tp": 2, - "ep": 1, - "n_shared_experts": 0 - }, - "name": "minimaxm2.5" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 1048576, - "hidden": 3072, - "intermediate": 1536, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 2, - "tp": 2, - "ep": 1, - "n_shared_experts": 0 - }, - "name": "minimaxm2.5" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 1024, - "hidden": 3072, - "intermediate": 1536, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 2, - "tp": 2, - "ep": 1, - "n_shared_experts": 0 - }, - "name": "minimaxm2.5" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 32, - "hidden": 3072, - "intermediate": 1536, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 4, - "tp": 4, - "ep": 1, - "n_shared_experts": 0 - }, - "name": "minimaxm2.5" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 64, - "hidden": 3072, - "intermediate": 1536, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 4, - "tp": 4, - "ep": 1, - "n_shared_experts": 0 - }, - "name": "minimaxm2.5" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 131072, - "hidden": 3072, - "intermediate": 1536, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 4, - "tp": 4, - "ep": 1, - "n_shared_experts": 0 - }, - "name": "minimaxm2.5" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 128, - "hidden": 3072, - "intermediate": 1536, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 4, - "tp": 4, - "ep": 1, - "n_shared_experts": 0 - }, - "name": "minimaxm2.5" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 16384, - "hidden": 3072, - "intermediate": 1536, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 8, - "tp": 8, - "ep": 1, - "n_shared_experts": 0 - }, - "name": "minimaxm2.5" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 16, - "hidden": 3072, - "intermediate": 1536, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 8, - "tp": 8, - "ep": 1, - "n_shared_experts": 0 - }, - "name": "minimaxm2.5" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 4194304, - "hidden": 3072, - "intermediate": 1536, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 1, - "tp": 1, - "ep": 1, - "n_shared_experts": 0 - }, - "name": "minimaxm2.5" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 8388608, - "hidden": 3072, - "intermediate": 1536, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 1, - "tp": 1, - "ep": 1, - "n_shared_experts": 0 - }, - "name": "minimaxm2.5" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 2097152, - "hidden": 3072, - "intermediate": 1536, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 2, - "tp": 2, - "ep": 1, - "n_shared_experts": 0 - }, - "name": "minimaxm2.5" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 4194304, - "hidden": 3072, - "intermediate": 1536, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 2, - "tp": 2, - "ep": 1, - "n_shared_experts": 0 - }, - "name": "minimaxm2.5" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 8388608, - "hidden": 3072, - "intermediate": 1536, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 2, - "tp": 2, - "ep": 1, - "n_shared_experts": 0 - }, - "name": "minimaxm2.5" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 262144, - "hidden": 3072, - "intermediate": 1536, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 4, - "tp": 4, - "ep": 1, - "n_shared_experts": 0 - }, - "name": "minimaxm2.5" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 524288, - "hidden": 3072, - "intermediate": 1536, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 4, - "tp": 4, - "ep": 1, - "n_shared_experts": 0 - }, - "name": "minimaxm2.5" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 1048576, - "hidden": 3072, - "intermediate": 1536, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 4, - "tp": 4, - "ep": 1, - "n_shared_experts": 0 - }, - "name": "minimaxm2.5" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 65536, - "hidden": 3072, - "intermediate": 1536, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 8, - "tp": 8, - "ep": 1, - "n_shared_experts": 0 - }, - "name": "minimaxm2.5" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 131072, - "hidden": 3072, - "intermediate": 1536, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 8, - "tp": 8, - "ep": 1, - "n_shared_experts": 0 - }, - "name": "minimaxm2.5" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 4096, - "hidden": 3072, - "intermediate": 1536, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "world_size": 8, - "tp": 1, - "ep": 8, - "n_shared_experts": 0 - }, - "name": "minimaxm2.5" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 4, - "hidden": 3072, - "intermediate": 1536, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "world_size": 8, - "tp": 1, - "ep": 8, - "n_shared_experts": 0 - }, - "name": "minimaxm2.5" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 8192, - "hidden": 3072, - "intermediate": 1536, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "world_size": 8, - "tp": 1, - "ep": 8, - "n_shared_experts": 0 - }, - "name": "minimaxm2.5" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 8, - "hidden": 3072, - "intermediate": 1536, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "world_size": 8, - "tp": 1, - "ep": 8, - "n_shared_experts": 0 - }, - "name": "minimaxm2.5" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 16, - "hidden": 3072, - "intermediate": 1536, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "world_size": 8, - "tp": 1, - "ep": 8, - "n_shared_experts": 0 - }, - "name": "minimaxm2.5" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 32768, - "hidden": 3072, - "intermediate": 1536, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "world_size": 8, - "tp": 1, - "ep": 8, - "n_shared_experts": 0 - }, - "name": "minimaxm2.5" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 32, - "hidden": 3072, - "intermediate": 1536, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "world_size": 8, - "tp": 1, - "ep": 8, - "n_shared_experts": 0 - }, - "name": "minimaxm2.5" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 65536, - "hidden": 3072, - "intermediate": 1536, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "world_size": 8, - "tp": 1, - "ep": 8, - "n_shared_experts": 0 - }, - "name": "minimaxm2.5" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 64, - "hidden": 3072, - "intermediate": 1536, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "world_size": 8, - "tp": 1, - "ep": 8, - "n_shared_experts": 0 - }, - "name": "minimaxm2.5" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 131072, - "hidden": 3072, - "intermediate": 1536, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "world_size": 8, - "tp": 1, - "ep": 8, - "n_shared_experts": 0 - }, - "name": "minimaxm2.5" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 128, - "hidden": 3072, - "intermediate": 1536, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "world_size": 8, - "tp": 1, - "ep": 8, - "n_shared_experts": 0 - }, - "name": "minimaxm2.5" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 262144, - "hidden": 3072, - "intermediate": 1536, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "world_size": 8, - "tp": 1, - "ep": 8, - "n_shared_experts": 0 - }, - "name": "minimaxm2.5" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 256, - "hidden": 3072, - "intermediate": 1536, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "world_size": 8, - "tp": 1, - "ep": 8, - "n_shared_experts": 0 - }, - "name": "minimaxm2.5" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 524288, - "hidden": 3072, - "intermediate": 1536, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "world_size": 8, - "tp": 1, - "ep": 8, - "n_shared_experts": 0 - }, - "name": "minimaxm2.5" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 512, - "hidden": 3072, - "intermediate": 1536, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "world_size": 8, - "tp": 1, - "ep": 8, - "n_shared_experts": 0 - }, - "name": "minimaxm2.5" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 1048576, - "hidden": 3072, - "intermediate": 1536, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "world_size": 8, - "tp": 1, - "ep": 8, - "n_shared_experts": 0 - }, - "name": "minimaxm2.5" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 2097152, - "hidden": 3072, - "intermediate": 1536, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "world_size": 8, - "tp": 1, - "ep": 8, - "n_shared_experts": 0 - }, - "name": "minimaxm2.5" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 1024, - "hidden": 2880, - "intermediate": 2880, - "num_experts": 128, - "top_k": 4, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 8, - "tp": 8, - "ep": 1, - "n_shared_experts": 0 - }, - "name": "gptoss" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 2048, - "hidden": 2880, - "intermediate": 2880, - "num_experts": 128, - "top_k": 4, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 8, - "tp": 8, - "ep": 1, - "n_shared_experts": 0 - }, - "name": "gptoss" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 2, - "hidden": 2880, - "intermediate": 2880, - "num_experts": 128, - "top_k": 4, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 8, - "tp": 8, - "ep": 1, - "n_shared_experts": 0 - }, - "name": "gptoss" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 1572864, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "world_size": 8, - "tp": 8, - "ep": 1, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 1536, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "world_size": 8, - "tp": 8, - "ep": 1, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 1572864, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "world_size": 8, - "tp": 1, - "ep": 8, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 1536, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "world_size": 8, - "tp": 1, - "ep": 8, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 2048, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "world_size": 4, - "tp": 4, - "ep": 1, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 2, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "world_size": 4, - "tp": 4, - "ep": 1, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 1, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "world_size": 4, - "tp": 4, - "ep": 1, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 1024, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 8, - "tp": 8, - "ep": 1, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 2048, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 8, - "tp": 8, - "ep": 1, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 33554432, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 8, - "tp": 1, - "ep": 8, - "n_shared_experts": 1 - }, - "name": "dsr1" - } -] \ No newline at end of file diff --git a/operatorx/testlists/moe_a2a.json b/operatorx/testlists/moe_a2a.json deleted file mode 100644 index ee134ea33d..0000000000 --- a/operatorx/testlists/moe_a2a.json +++ /dev/null @@ -1,1082 +0,0 @@ -[ - { - "type": "dispatch", - "args": { - "num_tokens": 32768, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 2 - }, - "name": "dsr1" - }, - { - "type": "combine", - "args": { - "num_tokens": 32768, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 2 - }, - "name": "dsr1" - }, - { - "type": "dispatch", - "args": { - "num_tokens": 25600, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 2 - }, - "name": "dsr1" - }, - { - "type": "combine", - "args": { - "num_tokens": 25600, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 2 - }, - "name": "dsr1" - }, - { - "type": "dispatch", - "args": { - "num_tokens": 4096, - "num_experts": 512, - "top_k": 10, - "hidden": 4096, - "dtype": "bf16", - "world_size": 2 - }, - "name": "qwen3.5" - }, - { - "type": "combine", - "args": { - "num_tokens": 4096, - "num_experts": 512, - "top_k": 10, - "hidden": 4096, - "dtype": "bf16", - "world_size": 2 - }, - "name": "qwen3.5" - }, - { - "type": "dispatch", - "args": { - "num_tokens": 16384, - "num_experts": 512, - "top_k": 10, - "hidden": 4096, - "dtype": "bf16", - "world_size": 2 - }, - "name": "qwen3.5" - }, - { - "type": "combine", - "args": { - "num_tokens": 16384, - "num_experts": 512, - "top_k": 10, - "hidden": 4096, - "dtype": "bf16", - "world_size": 2 - }, - "name": "qwen3.5" - }, - { - "type": "dispatch", - "args": { - "num_tokens": 64, - "num_experts": 512, - "top_k": 10, - "hidden": 4096, - "dtype": "bf16", - "world_size": 2 - }, - "name": "qwen3.5" - }, - { - "type": "combine", - "args": { - "num_tokens": 64, - "num_experts": 512, - "top_k": 10, - "hidden": 4096, - "dtype": "bf16", - "world_size": 2 - }, - "name": "qwen3.5" - }, - { - "type": "dispatch", - "args": { - "num_tokens": 256, - "num_experts": 128, - "top_k": 4, - "hidden": 2880, - "dtype": "bf16", - "world_size": 2 - }, - "name": "gptoss" - }, - { - "type": "combine", - "args": { - "num_tokens": 256, - "num_experts": 128, - "top_k": 4, - "hidden": 2880, - "dtype": "bf16", - "world_size": 2 - }, - "name": "gptoss" - }, - { - "type": "dispatch", - "args": { - "num_tokens": 8, - "num_experts": 128, - "top_k": 4, - "hidden": 2880, - "dtype": "bf16", - "world_size": 2 - }, - "name": "gptoss" - }, - { - "type": "combine", - "args": { - "num_tokens": 8, - "num_experts": 128, - "top_k": 4, - "hidden": 2880, - "dtype": "bf16", - "world_size": 2 - }, - "name": "gptoss" - }, - { - "type": "dispatch", - "args": { - "num_tokens": 64, - "num_experts": 128, - "top_k": 4, - "hidden": 2880, - "dtype": "bf16", - "world_size": 2 - }, - "name": "gptoss" - }, - { - "type": "combine", - "args": { - "num_tokens": 64, - "num_experts": 128, - "top_k": 4, - "hidden": 2880, - "dtype": "bf16", - "world_size": 2 - }, - "name": "gptoss" - }, - { - "type": "dispatch", - "args": { - "num_tokens": 4, - "num_experts": 256, - "top_k": 8, - "hidden": 3072, - "dtype": "bf16", - "world_size": 2 - }, - "name": "minimaxm2.5" - }, - { - "type": "combine", - "args": { - "num_tokens": 4, - "num_experts": 256, - "top_k": 8, - "hidden": 3072, - "dtype": "bf16", - "world_size": 2 - }, - "name": "minimaxm2.5" - }, - { - "type": "dispatch", - "args": { - "num_tokens": 64, - "num_experts": 256, - "top_k": 8, - "hidden": 3072, - "dtype": "bf16", - "world_size": 2 - }, - "name": "minimaxm2.5" - }, - { - "type": "combine", - "args": { - "num_tokens": 64, - "num_experts": 256, - "top_k": 8, - "hidden": 3072, - "dtype": "bf16", - "world_size": 2 - }, - "name": "minimaxm2.5" - }, - { - "type": "dispatch", - "args": { - "num_tokens": 16384, - "num_experts": 256, - "top_k": 8, - "hidden": 3072, - "dtype": "bf16", - "world_size": 2 - }, - "name": "minimaxm2.5" - }, - { - "type": "combine", - "args": { - "num_tokens": 16384, - "num_experts": 256, - "top_k": 8, - "hidden": 3072, - "dtype": "bf16", - "world_size": 2 - }, - "name": "minimaxm2.5" - }, - { - "type": "dispatch", - "args": { - "num_tokens": 24576, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 2 - }, - "name": "dsr1" - }, - { - "type": "combine", - "args": { - "num_tokens": 24576, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 2 - }, - "name": "dsr1" - }, - { - "type": "dispatch", - "args": { - "num_tokens": 2560, - "num_experts": 128, - "top_k": 4, - "hidden": 2880, - "dtype": "bf16", - "world_size": 2 - }, - "name": "gptoss" - }, - { - "type": "combine", - "args": { - "num_tokens": 2560, - "num_experts": 128, - "top_k": 4, - "hidden": 2880, - "dtype": "bf16", - "world_size": 2 - }, - "name": "gptoss" - }, - { - "type": "dispatch", - "args": { - "num_tokens": 4, - "num_experts": 256, - "top_k": 8, - "hidden": 6144, - "dtype": "bf16", - "world_size": 2 - }, - "name": "glm5.1" - }, - { - "type": "combine", - "args": { - "num_tokens": 4, - "num_experts": 256, - "top_k": 8, - "hidden": 6144, - "dtype": "bf16", - "world_size": 2 - }, - "name": "glm5.1" - }, - { - "type": "dispatch", - "args": { - "num_tokens": 16, - "num_experts": 256, - "top_k": 8, - "hidden": 6144, - "dtype": "bf16", - "world_size": 2 - }, - "name": "glm5.1" - }, - { - "type": "combine", - "args": { - "num_tokens": 16, - "num_experts": 256, - "top_k": 8, - "hidden": 6144, - "dtype": "bf16", - "world_size": 2 - }, - "name": "glm5.1" - }, - { - "type": "dispatch", - "args": { - "num_tokens": 32768, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 4 - }, - "name": "dsr1" - }, - { - "type": "combine", - "args": { - "num_tokens": 32768, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 4 - }, - "name": "dsr1" - }, - { - "type": "dispatch", - "args": { - "num_tokens": 2, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 4 - }, - "name": "dsr1" - }, - { - "type": "combine", - "args": { - "num_tokens": 2, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 4 - }, - "name": "dsr1" - }, - { - "type": "dispatch", - "args": { - "num_tokens": 8, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 4 - }, - "name": "dsr1" - }, - { - "type": "combine", - "args": { - "num_tokens": 8, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 4 - }, - "name": "dsr1" - }, - { - "type": "dispatch", - "args": { - "num_tokens": 256, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 4 - }, - "name": "dsr1" - }, - { - "type": "combine", - "args": { - "num_tokens": 256, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 4 - }, - "name": "dsr1" - }, - { - "type": "dispatch", - "args": { - "num_tokens": 16, - "num_experts": 384, - "top_k": 6, - "hidden": 7168, - "dtype": "bf16", - "world_size": 4 - }, - "name": "dsv4" - }, - { - "type": "combine", - "args": { - "num_tokens": 16, - "num_experts": 384, - "top_k": 6, - "hidden": 7168, - "dtype": "bf16", - "world_size": 4 - }, - "name": "dsv4" - }, - { - "type": "dispatch", - "args": { - "num_tokens": 16, - "num_experts": 512, - "top_k": 10, - "hidden": 4096, - "dtype": "bf16", - "world_size": 4 - }, - "name": "qwen3.5" - }, - { - "type": "combine", - "args": { - "num_tokens": 16, - "num_experts": 512, - "top_k": 10, - "hidden": 4096, - "dtype": "bf16", - "world_size": 4 - }, - "name": "qwen3.5" - }, - { - "type": "dispatch", - "args": { - "num_tokens": 8192, - "num_experts": 256, - "top_k": 8, - "hidden": 6144, - "dtype": "bf16", - "world_size": 4 - }, - "name": "glm5" - }, - { - "type": "combine", - "args": { - "num_tokens": 8192, - "num_experts": 256, - "top_k": 8, - "hidden": 6144, - "dtype": "bf16", - "world_size": 4 - }, - "name": "glm5" - }, - { - "type": "dispatch", - "args": { - "num_tokens": 256, - "num_experts": 256, - "top_k": 8, - "hidden": 6144, - "dtype": "bf16", - "world_size": 4 - }, - "name": "glm5" - }, - { - "type": "combine", - "args": { - "num_tokens": 256, - "num_experts": 256, - "top_k": 8, - "hidden": 6144, - "dtype": "bf16", - "world_size": 4 - }, - "name": "glm5" - }, - { - "type": "dispatch", - "args": { - "num_tokens": 16, - "num_experts": 384, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 4 - }, - "name": "kimik2.5" - }, - { - "type": "combine", - "args": { - "num_tokens": 16, - "num_experts": 384, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 4 - }, - "name": "kimik2.5" - }, - { - "type": "dispatch", - "args": { - "num_tokens": 4, - "num_experts": 384, - "top_k": 6, - "hidden": 7168, - "dtype": "bf16", - "world_size": 4 - }, - "name": "dsv4" - }, - { - "type": "combine", - "args": { - "num_tokens": 4, - "num_experts": 384, - "top_k": 6, - "hidden": 7168, - "dtype": "bf16", - "world_size": 4 - }, - "name": "dsv4" - }, - { - "type": "dispatch", - "args": { - "num_tokens": 16, - "num_experts": 128, - "top_k": 4, - "hidden": 2880, - "dtype": "bf16", - "world_size": 4 - }, - "name": "gptoss" - }, - { - "type": "combine", - "args": { - "num_tokens": 16, - "num_experts": 128, - "top_k": 4, - "hidden": 2880, - "dtype": "bf16", - "world_size": 4 - }, - "name": "gptoss" - }, - { - "type": "dispatch", - "args": { - "num_tokens": 16384, - "num_experts": 256, - "top_k": 8, - "hidden": 3072, - "dtype": "bf16", - "world_size": 4 - }, - "name": "minimaxm2.5" - }, - { - "type": "combine", - "args": { - "num_tokens": 16384, - "num_experts": 256, - "top_k": 8, - "hidden": 3072, - "dtype": "bf16", - "world_size": 4 - }, - "name": "minimaxm2.5" - }, - { - "type": "dispatch", - "args": { - "num_tokens": 24576, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 4 - }, - "name": "dsr1" - }, - { - "type": "combine", - "args": { - "num_tokens": 24576, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 4 - }, - "name": "dsr1" - }, - { - "type": "dispatch", - "args": { - "num_tokens": 256, - "num_experts": 128, - "top_k": 4, - "hidden": 2880, - "dtype": "bf16", - "world_size": 4 - }, - "name": "gptoss" - }, - { - "type": "combine", - "args": { - "num_tokens": 256, - "num_experts": 128, - "top_k": 4, - "hidden": 2880, - "dtype": "bf16", - "world_size": 4 - }, - "name": "gptoss" - }, - { - "type": "dispatch", - "args": { - "num_tokens": 3, - "num_experts": 384, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 4 - }, - "name": "kimik2.5" - }, - { - "type": "combine", - "args": { - "num_tokens": 3, - "num_experts": 384, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 4 - }, - "name": "kimik2.5" - }, - { - "type": "dispatch", - "args": { - "num_tokens": 1214, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 8 - }, - "name": "dsr1" - }, - { - "type": "combine", - "args": { - "num_tokens": 1214, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 8 - }, - "name": "dsr1" - }, - { - "type": "dispatch", - "args": { - "num_tokens": 1096, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 8 - }, - "name": "dsr1" - }, - { - "type": "combine", - "args": { - "num_tokens": 1096, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 8 - }, - "name": "dsr1" - }, - { - "type": "dispatch", - "args": { - "num_tokens": 2192, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 8 - }, - "name": "dsr1" - }, - { - "type": "combine", - "args": { - "num_tokens": 2192, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 8 - }, - "name": "dsr1" - }, - { - "type": "dispatch", - "args": { - "num_tokens": 8192, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 8 - }, - "name": "dsr1" - }, - { - "type": "combine", - "args": { - "num_tokens": 8192, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 8 - }, - "name": "dsr1" - }, - { - "type": "dispatch", - "args": { - "num_tokens": 271, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 8 - }, - "name": "dsr1" - }, - { - "type": "combine", - "args": { - "num_tokens": 271, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 8 - }, - "name": "dsr1" - }, - { - "type": "dispatch", - "args": { - "num_tokens": 2305, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 8 - }, - "name": "dsr1" - }, - { - "type": "combine", - "args": { - "num_tokens": 2305, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 8 - }, - "name": "dsr1" - }, - { - "type": "dispatch", - "args": { - "num_tokens": 288, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 8 - }, - "name": "dsr1" - }, - { - "type": "combine", - "args": { - "num_tokens": 288, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 8 - }, - "name": "dsr1" - }, - { - "type": "dispatch", - "args": { - "num_tokens": 3072, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 8 - }, - "name": "dsr1" - }, - { - "type": "combine", - "args": { - "num_tokens": 3072, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 8 - }, - "name": "dsr1" - }, - { - "type": "dispatch", - "args": { - "num_tokens": 32768, - "num_experts": 384, - "top_k": 6, - "hidden": 7168, - "dtype": "bf16", - "world_size": 8 - }, - "name": "dsv4" - }, - { - "type": "combine", - "args": { - "num_tokens": 32768, - "num_experts": 384, - "top_k": 6, - "hidden": 7168, - "dtype": "bf16", - "world_size": 8 - }, - "name": "dsv4" - }, - { - "type": "dispatch", - "args": { - "num_tokens": 16, - "num_experts": 512, - "top_k": 10, - "hidden": 4096, - "dtype": "bf16", - "world_size": 8 - }, - "name": "qwen3.5" - }, - { - "type": "combine", - "args": { - "num_tokens": 16, - "num_experts": 512, - "top_k": 10, - "hidden": 4096, - "dtype": "bf16", - "world_size": 8 - }, - "name": "qwen3.5" - }, - { - "type": "dispatch", - "args": { - "num_tokens": 32768, - "num_experts": 256, - "top_k": 8, - "hidden": 6144, - "dtype": "bf16", - "world_size": 8 - }, - "name": "glm5" - }, - { - "type": "combine", - "args": { - "num_tokens": 32768, - "num_experts": 256, - "top_k": 8, - "hidden": 6144, - "dtype": "bf16", - "world_size": 8 - }, - "name": "glm5" - }, - { - "type": "dispatch", - "args": { - "num_tokens": 16, - "num_experts": 384, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 8 - }, - "name": "kimik2.5" - }, - { - "type": "combine", - "args": { - "num_tokens": 16, - "num_experts": 384, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 8 - }, - "name": "kimik2.5" - }, - { - "type": "dispatch", - "args": { - "num_tokens": 4096, - "num_experts": 256, - "top_k": 8, - "hidden": 3072, - "dtype": "bf16", - "world_size": 8 - }, - "name": "minimaxm2.5" - }, - { - "type": "combine", - "args": { - "num_tokens": 4096, - "num_experts": 256, - "top_k": 8, - "hidden": 3072, - "dtype": "bf16", - "world_size": 8 - }, - "name": "minimaxm2.5" - }, - { - "type": "dispatch", - "args": { - "num_tokens": 32, - "num_experts": 256, - "top_k": 8, - "hidden": 3072, - "dtype": "bf16", - "world_size": 8 - }, - "name": "minimaxm2.5" - }, - { - "type": "combine", - "args": { - "num_tokens": 32, - "num_experts": 256, - "top_k": 8, - "hidden": 3072, - "dtype": "bf16", - "world_size": 8 - }, - "name": "minimaxm2.5" - }, - { - "type": "dispatch", - "args": { - "num_tokens": 2151, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 8 - }, - "name": "dsr1" - }, - { - "type": "combine", - "args": { - "num_tokens": 2151, - "num_experts": 256, - "top_k": 8, - "hidden": 7168, - "dtype": "bf16", - "world_size": 8 - }, - "name": "dsr1" - } -] \ No newline at end of file diff --git a/operatorx/testlists/moe_deepseek_v4.json b/operatorx/testlists/moe_deepseek_v4.json new file mode 100644 index 0000000000..880a1d7ac7 --- /dev/null +++ b/operatorx/testlists/moe_deepseek_v4.json @@ -0,0 +1,11330 @@ +[ + { + "type": "moe", + "args": { + "tokens": 1, + "activation": { + "kind": "silu", + "limit": 10.0 + }, + "experts": { + "a1": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "a2": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "inter": 3072, + "num": 384, + "top_k": 6, + "w1": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + }, + "w2": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + } + }, + "hidden": 7168, + "router": { + "bias": true, + "gate": { + "dtype": "bf16", + "logits": "fp32" + }, + "renormalize": true, + "scale": 2.5, + "scoring": "sqrtsoftplus", + "select": { + "kind": "topk" + } + }, + "routing": { + "distribution": "natural", + "seed": 0 + }, + "shared": { + "a1": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 128 + ], + "static": false + } + }, + "count": 1, + "inter": 3072, + "w1": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + }, + "w2": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + } + } + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn", + "deepseek-ai/DeepSeek-V4-Pro/ffn" + ] + }, + { + "type": "moe", + "args": { + "tokens": 2, + "activation": { + "kind": "silu", + "limit": 10.0 + }, + "experts": { + "a1": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "a2": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "inter": 3072, + "num": 384, + "top_k": 6, + "w1": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + }, + "w2": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + } + }, + "hidden": 7168, + "router": { + "bias": true, + "gate": { + "dtype": "bf16", + "logits": "fp32" + }, + "renormalize": true, + "scale": 2.5, + "scoring": "sqrtsoftplus", + "select": { + "kind": "topk" + } + }, + "routing": { + "distribution": "natural", + "seed": 0 + }, + "shared": { + "a1": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 128 + ], + "static": false + } + }, + "count": 1, + "inter": 3072, + "w1": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + }, + "w2": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + } + } + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn", + "deepseek-ai/DeepSeek-V4-Pro/ffn" + ] + }, + { + "type": "moe", + "args": { + "tokens": 3, + "activation": { + "kind": "silu", + "limit": 10.0 + }, + "experts": { + "a1": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "a2": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "inter": 3072, + "num": 384, + "top_k": 6, + "w1": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + }, + "w2": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + } + }, + "hidden": 7168, + "router": { + "bias": true, + "gate": { + "dtype": "bf16", + "logits": "fp32" + }, + "renormalize": true, + "scale": 2.5, + "scoring": "sqrtsoftplus", + "select": { + "kind": "topk" + } + }, + "routing": { + "distribution": "natural", + "seed": 0 + }, + "shared": { + "a1": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 128 + ], + "static": false + } + }, + "count": 1, + "inter": 3072, + "w1": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + }, + "w2": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + } + } + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn", + "deepseek-ai/DeepSeek-V4-Pro/ffn" + ] + }, + { + "type": "moe", + "args": { + "tokens": 4, + "activation": { + "kind": "silu", + "limit": 10.0 + }, + "experts": { + "a1": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "a2": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "inter": 3072, + "num": 384, + "top_k": 6, + "w1": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + }, + "w2": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + } + }, + "hidden": 7168, + "router": { + "bias": true, + "gate": { + "dtype": "bf16", + "logits": "fp32" + }, + "renormalize": true, + "scale": 2.5, + "scoring": "sqrtsoftplus", + "select": { + "kind": "topk" + } + }, + "routing": { + "distribution": "natural", + "seed": 0 + }, + "shared": { + "a1": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 128 + ], + "static": false + } + }, + "count": 1, + "inter": 3072, + "w1": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + }, + "w2": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + } + } + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn", + "deepseek-ai/DeepSeek-V4-Pro/ffn" + ] + }, + { + "type": "moe", + "args": { + "tokens": 6, + "activation": { + "kind": "silu", + "limit": 10.0 + }, + "experts": { + "a1": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "a2": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "inter": 3072, + "num": 384, + "top_k": 6, + "w1": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + }, + "w2": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + } + }, + "hidden": 7168, + "router": { + "bias": true, + "gate": { + "dtype": "bf16", + "logits": "fp32" + }, + "renormalize": true, + "scale": 2.5, + "scoring": "sqrtsoftplus", + "select": { + "kind": "topk" + } + }, + "routing": { + "distribution": "natural", + "seed": 0 + }, + "shared": { + "a1": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 128 + ], + "static": false + } + }, + "count": 1, + "inter": 3072, + "w1": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + }, + "w2": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + } + } + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn", + "deepseek-ai/DeepSeek-V4-Pro/ffn" + ] + }, + { + "type": "moe", + "args": { + "tokens": 8, + "activation": { + "kind": "silu", + "limit": 10.0 + }, + "experts": { + "a1": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "a2": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "inter": 3072, + "num": 384, + "top_k": 6, + "w1": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + }, + "w2": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + } + }, + "hidden": 7168, + "router": { + "bias": true, + "gate": { + "dtype": "bf16", + "logits": "fp32" + }, + "renormalize": true, + "scale": 2.5, + "scoring": "sqrtsoftplus", + "select": { + "kind": "topk" + } + }, + "routing": { + "distribution": "natural", + "seed": 0 + }, + "shared": { + "a1": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 128 + ], + "static": false + } + }, + "count": 1, + "inter": 3072, + "w1": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + }, + "w2": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + } + } + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn", + "deepseek-ai/DeepSeek-V4-Pro/ffn" + ] + }, + { + "type": "moe", + "args": { + "tokens": 12, + "activation": { + "kind": "silu", + "limit": 10.0 + }, + "experts": { + "a1": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "a2": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "inter": 3072, + "num": 384, + "top_k": 6, + "w1": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + }, + "w2": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + } + }, + "hidden": 7168, + "router": { + "bias": true, + "gate": { + "dtype": "bf16", + "logits": "fp32" + }, + "renormalize": true, + "scale": 2.5, + "scoring": "sqrtsoftplus", + "select": { + "kind": "topk" + } + }, + "routing": { + "distribution": "natural", + "seed": 0 + }, + "shared": { + "a1": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 128 + ], + "static": false + } + }, + "count": 1, + "inter": 3072, + "w1": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + }, + "w2": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + } + } + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn", + "deepseek-ai/DeepSeek-V4-Pro/ffn" + ] + }, + { + "type": "moe", + "args": { + "tokens": 16, + "activation": { + "kind": "silu", + "limit": 10.0 + }, + "experts": { + "a1": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "a2": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "inter": 3072, + "num": 384, + "top_k": 6, + "w1": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + }, + "w2": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + } + }, + "hidden": 7168, + "router": { + "bias": true, + "gate": { + "dtype": "bf16", + "logits": "fp32" + }, + "renormalize": true, + "scale": 2.5, + "scoring": "sqrtsoftplus", + "select": { + "kind": "topk" + } + }, + "routing": { + "distribution": "natural", + "seed": 0 + }, + "shared": { + "a1": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 128 + ], + "static": false + } + }, + "count": 1, + "inter": 3072, + "w1": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + }, + "w2": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + } + } + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn", + "deepseek-ai/DeepSeek-V4-Pro/ffn" + ] + }, + { + "type": "moe", + "args": { + "tokens": 24, + "activation": { + "kind": "silu", + "limit": 10.0 + }, + "experts": { + "a1": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "a2": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "inter": 3072, + "num": 384, + "top_k": 6, + "w1": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + }, + "w2": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + } + }, + "hidden": 7168, + "router": { + "bias": true, + "gate": { + "dtype": "bf16", + "logits": "fp32" + }, + "renormalize": true, + "scale": 2.5, + "scoring": "sqrtsoftplus", + "select": { + "kind": "topk" + } + }, + "routing": { + "distribution": "natural", + "seed": 0 + }, + "shared": { + "a1": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 128 + ], + "static": false + } + }, + "count": 1, + "inter": 3072, + "w1": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + }, + "w2": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + } + } + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn", + "deepseek-ai/DeepSeek-V4-Pro/ffn" + ] + }, + { + "type": "moe", + "args": { + "tokens": 32, + "activation": { + "kind": "silu", + "limit": 10.0 + }, + "experts": { + "a1": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "a2": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "inter": 3072, + "num": 384, + "top_k": 6, + "w1": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + }, + "w2": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + } + }, + "hidden": 7168, + "router": { + "bias": true, + "gate": { + "dtype": "bf16", + "logits": "fp32" + }, + "renormalize": true, + "scale": 2.5, + "scoring": "sqrtsoftplus", + "select": { + "kind": "topk" + } + }, + "routing": { + "distribution": "natural", + "seed": 0 + }, + "shared": { + "a1": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 128 + ], + "static": false + } + }, + "count": 1, + "inter": 3072, + "w1": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + }, + "w2": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + } + } + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn", + "deepseek-ai/DeepSeek-V4-Pro/ffn" + ] + }, + { + "type": "moe", + "args": { + "tokens": 48, + "activation": { + "kind": "silu", + "limit": 10.0 + }, + "experts": { + "a1": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "a2": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "inter": 3072, + "num": 384, + "top_k": 6, + "w1": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + }, + "w2": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + } + }, + "hidden": 7168, + "router": { + "bias": true, + "gate": { + "dtype": "bf16", + "logits": "fp32" + }, + "renormalize": true, + "scale": 2.5, + "scoring": "sqrtsoftplus", + "select": { + "kind": "topk" + } + }, + "routing": { + "distribution": "natural", + "seed": 0 + }, + "shared": { + "a1": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 128 + ], + "static": false + } + }, + "count": 1, + "inter": 3072, + "w1": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + }, + "w2": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + } + } + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn", + "deepseek-ai/DeepSeek-V4-Pro/ffn" + ] + }, + { + "type": "moe", + "args": { + "tokens": 64, + "activation": { + "kind": "silu", + "limit": 10.0 + }, + "experts": { + "a1": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "a2": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "inter": 3072, + "num": 384, + "top_k": 6, + "w1": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + }, + "w2": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + } + }, + "hidden": 7168, + "router": { + "bias": true, + "gate": { + "dtype": "bf16", + "logits": "fp32" + }, + "renormalize": true, + "scale": 2.5, + "scoring": "sqrtsoftplus", + "select": { + "kind": "topk" + } + }, + "routing": { + "distribution": "natural", + "seed": 0 + }, + "shared": { + "a1": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 128 + ], + "static": false + } + }, + "count": 1, + "inter": 3072, + "w1": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + }, + "w2": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + } + } + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn", + "deepseek-ai/DeepSeek-V4-Pro/ffn" + ] + }, + { + "type": "moe", + "args": { + "tokens": 96, + "activation": { + "kind": "silu", + "limit": 10.0 + }, + "experts": { + "a1": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "a2": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "inter": 3072, + "num": 384, + "top_k": 6, + "w1": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + }, + "w2": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + } + }, + "hidden": 7168, + "router": { + "bias": true, + "gate": { + "dtype": "bf16", + "logits": "fp32" + }, + "renormalize": true, + "scale": 2.5, + "scoring": "sqrtsoftplus", + "select": { + "kind": "topk" + } + }, + "routing": { + "distribution": "natural", + "seed": 0 + }, + "shared": { + "a1": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 128 + ], + "static": false + } + }, + "count": 1, + "inter": 3072, + "w1": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + }, + "w2": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + } + } + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn", + "deepseek-ai/DeepSeek-V4-Pro/ffn" + ] + }, + { + "type": "moe", + "args": { + "tokens": 128, + "activation": { + "kind": "silu", + "limit": 10.0 + }, + "experts": { + "a1": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "a2": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "inter": 3072, + "num": 384, + "top_k": 6, + "w1": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + }, + "w2": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + } + }, + "hidden": 7168, + "router": { + "bias": true, + "gate": { + "dtype": "bf16", + "logits": "fp32" + }, + "renormalize": true, + "scale": 2.5, + "scoring": "sqrtsoftplus", + "select": { + "kind": "topk" + } + }, + "routing": { + "distribution": "natural", + "seed": 0 + }, + "shared": { + "a1": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 128 + ], + "static": false + } + }, + "count": 1, + "inter": 3072, + "w1": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + }, + "w2": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + } + } + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn", + "deepseek-ai/DeepSeek-V4-Pro/ffn" + ] + }, + { + "type": "moe", + "args": { + "tokens": 192, + "activation": { + "kind": "silu", + "limit": 10.0 + }, + "experts": { + "a1": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "a2": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "inter": 3072, + "num": 384, + "top_k": 6, + "w1": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + }, + "w2": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + } + }, + "hidden": 7168, + "router": { + "bias": true, + "gate": { + "dtype": "bf16", + "logits": "fp32" + }, + "renormalize": true, + "scale": 2.5, + "scoring": "sqrtsoftplus", + "select": { + "kind": "topk" + } + }, + "routing": { + "distribution": "natural", + "seed": 0 + }, + "shared": { + "a1": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 128 + ], + "static": false + } + }, + "count": 1, + "inter": 3072, + "w1": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + }, + "w2": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + } + } + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn", + "deepseek-ai/DeepSeek-V4-Pro/ffn" + ] + }, + { + "type": "moe", + "args": { + "tokens": 256, + "activation": { + "kind": "silu", + "limit": 10.0 + }, + "experts": { + "a1": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "a2": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "inter": 3072, + "num": 384, + "top_k": 6, + "w1": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + }, + "w2": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + } + }, + "hidden": 7168, + "router": { + "bias": true, + "gate": { + "dtype": "bf16", + "logits": "fp32" + }, + "renormalize": true, + "scale": 2.5, + "scoring": "sqrtsoftplus", + "select": { + "kind": "topk" + } + }, + "routing": { + "distribution": "natural", + "seed": 0 + }, + "shared": { + "a1": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 128 + ], + "static": false + } + }, + "count": 1, + "inter": 3072, + "w1": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + }, + "w2": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + } + } + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn", + "deepseek-ai/DeepSeek-V4-Pro/ffn" + ] + }, + { + "type": "moe", + "args": { + "tokens": 384, + "activation": { + "kind": "silu", + "limit": 10.0 + }, + "experts": { + "a1": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "a2": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "inter": 3072, + "num": 384, + "top_k": 6, + "w1": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + }, + "w2": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + } + }, + "hidden": 7168, + "router": { + "bias": true, + "gate": { + "dtype": "bf16", + "logits": "fp32" + }, + "renormalize": true, + "scale": 2.5, + "scoring": "sqrtsoftplus", + "select": { + "kind": "topk" + } + }, + "routing": { + "distribution": "natural", + "seed": 0 + }, + "shared": { + "a1": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 128 + ], + "static": false + } + }, + "count": 1, + "inter": 3072, + "w1": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + }, + "w2": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + } + } + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn", + "deepseek-ai/DeepSeek-V4-Pro/ffn" + ] + }, + { + "type": "moe", + "args": { + "tokens": 512, + "activation": { + "kind": "silu", + "limit": 10.0 + }, + "experts": { + "a1": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "a2": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "inter": 3072, + "num": 384, + "top_k": 6, + "w1": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + }, + "w2": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + } + }, + "hidden": 7168, + "router": { + "bias": true, + "gate": { + "dtype": "bf16", + "logits": "fp32" + }, + "renormalize": true, + "scale": 2.5, + "scoring": "sqrtsoftplus", + "select": { + "kind": "topk" + } + }, + "routing": { + "distribution": "natural", + "seed": 0 + }, + "shared": { + "a1": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 128 + ], + "static": false + } + }, + "count": 1, + "inter": 3072, + "w1": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + }, + "w2": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + } + } + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn", + "deepseek-ai/DeepSeek-V4-Pro/ffn" + ] + }, + { + "type": "moe", + "args": { + "tokens": 768, + "activation": { + "kind": "silu", + "limit": 10.0 + }, + "experts": { + "a1": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "a2": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "inter": 3072, + "num": 384, + "top_k": 6, + "w1": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + }, + "w2": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + } + }, + "hidden": 7168, + "router": { + "bias": true, + "gate": { + "dtype": "bf16", + "logits": "fp32" + }, + "renormalize": true, + "scale": 2.5, + "scoring": "sqrtsoftplus", + "select": { + "kind": "topk" + } + }, + "routing": { + "distribution": "natural", + "seed": 0 + }, + "shared": { + "a1": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 128 + ], + "static": false + } + }, + "count": 1, + "inter": 3072, + "w1": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + }, + "w2": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + } + } + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn", + "deepseek-ai/DeepSeek-V4-Pro/ffn" + ] + }, + { + "type": "moe", + "args": { + "tokens": 1024, + "activation": { + "kind": "silu", + "limit": 10.0 + }, + "experts": { + "a1": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "a2": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "inter": 3072, + "num": 384, + "top_k": 6, + "w1": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + }, + "w2": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + } + }, + "hidden": 7168, + "router": { + "bias": true, + "gate": { + "dtype": "bf16", + "logits": "fp32" + }, + "renormalize": true, + "scale": 2.5, + "scoring": "sqrtsoftplus", + "select": { + "kind": "topk" + } + }, + "routing": { + "distribution": "natural", + "seed": 0 + }, + "shared": { + "a1": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 128 + ], + "static": false + } + }, + "count": 1, + "inter": 3072, + "w1": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + }, + "w2": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + } + } + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn", + "deepseek-ai/DeepSeek-V4-Pro/ffn" + ] + }, + { + "type": "moe", + "args": { + "tokens": 1536, + "activation": { + "kind": "silu", + "limit": 10.0 + }, + "experts": { + "a1": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "a2": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "inter": 3072, + "num": 384, + "top_k": 6, + "w1": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + }, + "w2": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + } + }, + "hidden": 7168, + "router": { + "bias": true, + "gate": { + "dtype": "bf16", + "logits": "fp32" + }, + "renormalize": true, + "scale": 2.5, + "scoring": "sqrtsoftplus", + "select": { + "kind": "topk" + } + }, + "routing": { + "distribution": "natural", + "seed": 0 + }, + "shared": { + "a1": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 128 + ], + "static": false + } + }, + "count": 1, + "inter": 3072, + "w1": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + }, + "w2": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + } + } + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn", + "deepseek-ai/DeepSeek-V4-Pro/ffn" + ] + }, + { + "type": "moe", + "args": { + "tokens": 2048, + "activation": { + "kind": "silu", + "limit": 10.0 + }, + "experts": { + "a1": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "a2": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "inter": 3072, + "num": 384, + "top_k": 6, + "w1": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + }, + "w2": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + } + }, + "hidden": 7168, + "router": { + "bias": true, + "gate": { + "dtype": "bf16", + "logits": "fp32" + }, + "renormalize": true, + "scale": 2.5, + "scoring": "sqrtsoftplus", + "select": { + "kind": "topk" + } + }, + "routing": { + "distribution": "natural", + "seed": 0 + }, + "shared": { + "a1": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 128 + ], + "static": false + } + }, + "count": 1, + "inter": 3072, + "w1": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + }, + "w2": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + } + } + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn", + "deepseek-ai/DeepSeek-V4-Pro/ffn" + ] + }, + { + "type": "moe", + "args": { + "tokens": 3072, + "activation": { + "kind": "silu", + "limit": 10.0 + }, + "experts": { + "a1": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "a2": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "inter": 3072, + "num": 384, + "top_k": 6, + "w1": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + }, + "w2": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + } + }, + "hidden": 7168, + "router": { + "bias": true, + "gate": { + "dtype": "bf16", + "logits": "fp32" + }, + "renormalize": true, + "scale": 2.5, + "scoring": "sqrtsoftplus", + "select": { + "kind": "topk" + } + }, + "routing": { + "distribution": "natural", + "seed": 0 + }, + "shared": { + "a1": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 128 + ], + "static": false + } + }, + "count": 1, + "inter": 3072, + "w1": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + }, + "w2": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + } + } + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn", + "deepseek-ai/DeepSeek-V4-Pro/ffn" + ] + }, + { + "type": "moe", + "args": { + "tokens": 4096, + "activation": { + "kind": "silu", + "limit": 10.0 + }, + "experts": { + "a1": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "a2": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "inter": 3072, + "num": 384, + "top_k": 6, + "w1": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + }, + "w2": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + } + }, + "hidden": 7168, + "router": { + "bias": true, + "gate": { + "dtype": "bf16", + "logits": "fp32" + }, + "renormalize": true, + "scale": 2.5, + "scoring": "sqrtsoftplus", + "select": { + "kind": "topk" + } + }, + "routing": { + "distribution": "natural", + "seed": 0 + }, + "shared": { + "a1": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 128 + ], + "static": false + } + }, + "count": 1, + "inter": 3072, + "w1": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + }, + "w2": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + } + } + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn", + "deepseek-ai/DeepSeek-V4-Pro/ffn" + ] + }, + { + "type": "moe", + "args": { + "tokens": 6144, + "activation": { + "kind": "silu", + "limit": 10.0 + }, + "experts": { + "a1": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "a2": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "inter": 3072, + "num": 384, + "top_k": 6, + "w1": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + }, + "w2": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + } + }, + "hidden": 7168, + "router": { + "bias": true, + "gate": { + "dtype": "bf16", + "logits": "fp32" + }, + "renormalize": true, + "scale": 2.5, + "scoring": "sqrtsoftplus", + "select": { + "kind": "topk" + } + }, + "routing": { + "distribution": "natural", + "seed": 0 + }, + "shared": { + "a1": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 128 + ], + "static": false + } + }, + "count": 1, + "inter": 3072, + "w1": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + }, + "w2": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + } + } + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn", + "deepseek-ai/DeepSeek-V4-Pro/ffn" + ] + }, + { + "type": "moe", + "args": { + "tokens": 8192, + "activation": { + "kind": "silu", + "limit": 10.0 + }, + "experts": { + "a1": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "a2": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "inter": 3072, + "num": 384, + "top_k": 6, + "w1": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + }, + "w2": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + } + }, + "hidden": 7168, + "router": { + "bias": true, + "gate": { + "dtype": "bf16", + "logits": "fp32" + }, + "renormalize": true, + "scale": 2.5, + "scoring": "sqrtsoftplus", + "select": { + "kind": "topk" + } + }, + "routing": { + "distribution": "natural", + "seed": 0 + }, + "shared": { + "a1": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 128 + ], + "static": false + } + }, + "count": 1, + "inter": 3072, + "w1": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + }, + "w2": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + } + } + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn", + "deepseek-ai/DeepSeek-V4-Pro/ffn" + ] + }, + { + "type": "moe", + "args": { + "tokens": 12288, + "activation": { + "kind": "silu", + "limit": 10.0 + }, + "experts": { + "a1": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "a2": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "inter": 3072, + "num": 384, + "top_k": 6, + "w1": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + }, + "w2": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + } + }, + "hidden": 7168, + "router": { + "bias": true, + "gate": { + "dtype": "bf16", + "logits": "fp32" + }, + "renormalize": true, + "scale": 2.5, + "scoring": "sqrtsoftplus", + "select": { + "kind": "topk" + } + }, + "routing": { + "distribution": "natural", + "seed": 0 + }, + "shared": { + "a1": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 128 + ], + "static": false + } + }, + "count": 1, + "inter": 3072, + "w1": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + }, + "w2": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + } + } + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn", + "deepseek-ai/DeepSeek-V4-Pro/ffn" + ] + }, + { + "type": "moe", + "args": { + "tokens": 16384, + "activation": { + "kind": "silu", + "limit": 10.0 + }, + "experts": { + "a1": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "a2": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "inter": 3072, + "num": 384, + "top_k": 6, + "w1": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + }, + "w2": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + } + }, + "hidden": 7168, + "router": { + "bias": true, + "gate": { + "dtype": "bf16", + "logits": "fp32" + }, + "renormalize": true, + "scale": 2.5, + "scoring": "sqrtsoftplus", + "select": { + "kind": "topk" + } + }, + "routing": { + "distribution": "natural", + "seed": 0 + }, + "shared": { + "a1": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 128 + ], + "static": false + } + }, + "count": 1, + "inter": 3072, + "w1": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + }, + "w2": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + } + } + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn", + "deepseek-ai/DeepSeek-V4-Pro/ffn" + ] + }, + { + "type": "moe", + "args": { + "tokens": 24576, + "activation": { + "kind": "silu", + "limit": 10.0 + }, + "experts": { + "a1": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "a2": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "inter": 3072, + "num": 384, + "top_k": 6, + "w1": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + }, + "w2": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + } + }, + "hidden": 7168, + "router": { + "bias": true, + "gate": { + "dtype": "bf16", + "logits": "fp32" + }, + "renormalize": true, + "scale": 2.5, + "scoring": "sqrtsoftplus", + "select": { + "kind": "topk" + } + }, + "routing": { + "distribution": "natural", + "seed": 0 + }, + "shared": { + "a1": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 128 + ], + "static": false + } + }, + "count": 1, + "inter": 3072, + "w1": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + }, + "w2": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + } + } + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn", + "deepseek-ai/DeepSeek-V4-Pro/ffn" + ] + }, + { + "type": "moe", + "args": { + "tokens": 32768, + "activation": { + "kind": "silu", + "limit": 10.0 + }, + "experts": { + "a1": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "a2": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "inter": 3072, + "num": 384, + "top_k": 6, + "w1": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + }, + "w2": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + } + }, + "hidden": 7168, + "router": { + "bias": true, + "gate": { + "dtype": "bf16", + "logits": "fp32" + }, + "renormalize": true, + "scale": 2.5, + "scoring": "sqrtsoftplus", + "select": { + "kind": "topk" + } + }, + "routing": { + "distribution": "natural", + "seed": 0 + }, + "shared": { + "a1": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 128 + ], + "static": false + } + }, + "count": 1, + "inter": 3072, + "w1": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + }, + "w2": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + } + } + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn", + "deepseek-ai/DeepSeek-V4-Pro/ffn" + ] + }, + { + "type": "moe", + "args": { + "tokens": 49152, + "activation": { + "kind": "silu", + "limit": 10.0 + }, + "experts": { + "a1": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "a2": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "inter": 3072, + "num": 384, + "top_k": 6, + "w1": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + }, + "w2": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + } + }, + "hidden": 7168, + "router": { + "bias": true, + "gate": { + "dtype": "bf16", + "logits": "fp32" + }, + "renormalize": true, + "scale": 2.5, + "scoring": "sqrtsoftplus", + "select": { + "kind": "topk" + } + }, + "routing": { + "distribution": "natural", + "seed": 0 + }, + "shared": { + "a1": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 128 + ], + "static": false + } + }, + "count": 1, + "inter": 3072, + "w1": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + }, + "w2": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + } + } + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn", + "deepseek-ai/DeepSeek-V4-Pro/ffn" + ] + }, + { + "type": "moe", + "args": { + "tokens": 65536, + "activation": { + "kind": "silu", + "limit": 10.0 + }, + "experts": { + "a1": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "a2": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "inter": 3072, + "num": 384, + "top_k": 6, + "w1": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + }, + "w2": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + } + }, + "hidden": 7168, + "router": { + "bias": true, + "gate": { + "dtype": "bf16", + "logits": "fp32" + }, + "renormalize": true, + "scale": 2.5, + "scoring": "sqrtsoftplus", + "select": { + "kind": "topk" + } + }, + "routing": { + "distribution": "natural", + "seed": 0 + }, + "shared": { + "a1": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 128 + ], + "static": false + } + }, + "count": 1, + "inter": 3072, + "w1": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + }, + "w2": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + } + } + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn", + "deepseek-ai/DeepSeek-V4-Pro/ffn" + ] + }, + { + "type": "moe", + "args": { + "tokens": 1, + "activation": { + "kind": "silu", + "limit": 10.0 + }, + "experts": { + "a1": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "a2": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "inter": 3072, + "num": 384, + "top_k": 6, + "w1": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + }, + "w2": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + } + }, + "hidden": 7168, + "router": { + "bias": false, + "gate": { + "dtype": "bf16", + "logits": "fp32" + }, + "renormalize": true, + "scale": 2.5, + "scoring": "sqrtsoftplus", + "select": { + "kind": "hash", + "vocab": 129280 + } + }, + "routing": { + "distribution": "natural", + "seed": 0 + }, + "shared": { + "a1": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 128 + ], + "static": false + } + }, + "count": 1, + "inter": 3072, + "w1": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + }, + "w2": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + } + } + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn", + "deepseek-ai/DeepSeek-V4-Pro/ffn" + ] + }, + { + "type": "moe", + "args": { + "tokens": 2, + "activation": { + "kind": "silu", + "limit": 10.0 + }, + "experts": { + "a1": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "a2": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "inter": 3072, + "num": 384, + "top_k": 6, + "w1": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + }, + "w2": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + } + }, + "hidden": 7168, + "router": { + "bias": false, + "gate": { + "dtype": "bf16", + "logits": "fp32" + }, + "renormalize": true, + "scale": 2.5, + "scoring": "sqrtsoftplus", + "select": { + "kind": "hash", + "vocab": 129280 + } + }, + "routing": { + "distribution": "natural", + "seed": 0 + }, + "shared": { + "a1": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 128 + ], + "static": false + } + }, + "count": 1, + "inter": 3072, + "w1": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + }, + "w2": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + } + } + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn", + "deepseek-ai/DeepSeek-V4-Pro/ffn" + ] + }, + { + "type": "moe", + "args": { + "tokens": 3, + "activation": { + "kind": "silu", + "limit": 10.0 + }, + "experts": { + "a1": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "a2": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "inter": 3072, + "num": 384, + "top_k": 6, + "w1": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + }, + "w2": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + } + }, + "hidden": 7168, + "router": { + "bias": false, + "gate": { + "dtype": "bf16", + "logits": "fp32" + }, + "renormalize": true, + "scale": 2.5, + "scoring": "sqrtsoftplus", + "select": { + "kind": "hash", + "vocab": 129280 + } + }, + "routing": { + "distribution": "natural", + "seed": 0 + }, + "shared": { + "a1": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 128 + ], + "static": false + } + }, + "count": 1, + "inter": 3072, + "w1": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + }, + "w2": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + } + } + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn", + "deepseek-ai/DeepSeek-V4-Pro/ffn" + ] + }, + { + "type": "moe", + "args": { + "tokens": 4, + "activation": { + "kind": "silu", + "limit": 10.0 + }, + "experts": { + "a1": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "a2": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "inter": 3072, + "num": 384, + "top_k": 6, + "w1": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + }, + "w2": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + } + }, + "hidden": 7168, + "router": { + "bias": false, + "gate": { + "dtype": "bf16", + "logits": "fp32" + }, + "renormalize": true, + "scale": 2.5, + "scoring": "sqrtsoftplus", + "select": { + "kind": "hash", + "vocab": 129280 + } + }, + "routing": { + "distribution": "natural", + "seed": 0 + }, + "shared": { + "a1": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 128 + ], + "static": false + } + }, + "count": 1, + "inter": 3072, + "w1": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + }, + "w2": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + } + } + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn", + "deepseek-ai/DeepSeek-V4-Pro/ffn" + ] + }, + { + "type": "moe", + "args": { + "tokens": 6, + "activation": { + "kind": "silu", + "limit": 10.0 + }, + "experts": { + "a1": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "a2": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "inter": 3072, + "num": 384, + "top_k": 6, + "w1": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + }, + "w2": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + } + }, + "hidden": 7168, + "router": { + "bias": false, + "gate": { + "dtype": "bf16", + "logits": "fp32" + }, + "renormalize": true, + "scale": 2.5, + "scoring": "sqrtsoftplus", + "select": { + "kind": "hash", + "vocab": 129280 + } + }, + "routing": { + "distribution": "natural", + "seed": 0 + }, + "shared": { + "a1": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 128 + ], + "static": false + } + }, + "count": 1, + "inter": 3072, + "w1": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + }, + "w2": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + } + } + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn", + "deepseek-ai/DeepSeek-V4-Pro/ffn" + ] + }, + { + "type": "moe", + "args": { + "tokens": 8, + "activation": { + "kind": "silu", + "limit": 10.0 + }, + "experts": { + "a1": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "a2": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "inter": 3072, + "num": 384, + "top_k": 6, + "w1": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + }, + "w2": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + } + }, + "hidden": 7168, + "router": { + "bias": false, + "gate": { + "dtype": "bf16", + "logits": "fp32" + }, + "renormalize": true, + "scale": 2.5, + "scoring": "sqrtsoftplus", + "select": { + "kind": "hash", + "vocab": 129280 + } + }, + "routing": { + "distribution": "natural", + "seed": 0 + }, + "shared": { + "a1": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 128 + ], + "static": false + } + }, + "count": 1, + "inter": 3072, + "w1": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + }, + "w2": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + } + } + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn", + "deepseek-ai/DeepSeek-V4-Pro/ffn" + ] + }, + { + "type": "moe", + "args": { + "tokens": 12, + "activation": { + "kind": "silu", + "limit": 10.0 + }, + "experts": { + "a1": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "a2": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "inter": 3072, + "num": 384, + "top_k": 6, + "w1": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + }, + "w2": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + } + }, + "hidden": 7168, + "router": { + "bias": false, + "gate": { + "dtype": "bf16", + "logits": "fp32" + }, + "renormalize": true, + "scale": 2.5, + "scoring": "sqrtsoftplus", + "select": { + "kind": "hash", + "vocab": 129280 + } + }, + "routing": { + "distribution": "natural", + "seed": 0 + }, + "shared": { + "a1": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 128 + ], + "static": false + } + }, + "count": 1, + "inter": 3072, + "w1": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + }, + "w2": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + } + } + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn", + "deepseek-ai/DeepSeek-V4-Pro/ffn" + ] + }, + { + "type": "moe", + "args": { + "tokens": 16, + "activation": { + "kind": "silu", + "limit": 10.0 + }, + "experts": { + "a1": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "a2": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "inter": 3072, + "num": 384, + "top_k": 6, + "w1": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + }, + "w2": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + } + }, + "hidden": 7168, + "router": { + "bias": false, + "gate": { + "dtype": "bf16", + "logits": "fp32" + }, + "renormalize": true, + "scale": 2.5, + "scoring": "sqrtsoftplus", + "select": { + "kind": "hash", + "vocab": 129280 + } + }, + "routing": { + "distribution": "natural", + "seed": 0 + }, + "shared": { + "a1": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 128 + ], + "static": false + } + }, + "count": 1, + "inter": 3072, + "w1": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + }, + "w2": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + } + } + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn", + "deepseek-ai/DeepSeek-V4-Pro/ffn" + ] + }, + { + "type": "moe", + "args": { + "tokens": 24, + "activation": { + "kind": "silu", + "limit": 10.0 + }, + "experts": { + "a1": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "a2": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "inter": 3072, + "num": 384, + "top_k": 6, + "w1": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + }, + "w2": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + } + }, + "hidden": 7168, + "router": { + "bias": false, + "gate": { + "dtype": "bf16", + "logits": "fp32" + }, + "renormalize": true, + "scale": 2.5, + "scoring": "sqrtsoftplus", + "select": { + "kind": "hash", + "vocab": 129280 + } + }, + "routing": { + "distribution": "natural", + "seed": 0 + }, + "shared": { + "a1": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 128 + ], + "static": false + } + }, + "count": 1, + "inter": 3072, + "w1": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + }, + "w2": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + } + } + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn", + "deepseek-ai/DeepSeek-V4-Pro/ffn" + ] + }, + { + "type": "moe", + "args": { + "tokens": 32, + "activation": { + "kind": "silu", + "limit": 10.0 + }, + "experts": { + "a1": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "a2": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "inter": 3072, + "num": 384, + "top_k": 6, + "w1": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + }, + "w2": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + } + }, + "hidden": 7168, + "router": { + "bias": false, + "gate": { + "dtype": "bf16", + "logits": "fp32" + }, + "renormalize": true, + "scale": 2.5, + "scoring": "sqrtsoftplus", + "select": { + "kind": "hash", + "vocab": 129280 + } + }, + "routing": { + "distribution": "natural", + "seed": 0 + }, + "shared": { + "a1": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 128 + ], + "static": false + } + }, + "count": 1, + "inter": 3072, + "w1": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + }, + "w2": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + } + } + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn", + "deepseek-ai/DeepSeek-V4-Pro/ffn" + ] + }, + { + "type": "moe", + "args": { + "tokens": 48, + "activation": { + "kind": "silu", + "limit": 10.0 + }, + "experts": { + "a1": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "a2": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "inter": 3072, + "num": 384, + "top_k": 6, + "w1": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + }, + "w2": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + } + }, + "hidden": 7168, + "router": { + "bias": false, + "gate": { + "dtype": "bf16", + "logits": "fp32" + }, + "renormalize": true, + "scale": 2.5, + "scoring": "sqrtsoftplus", + "select": { + "kind": "hash", + "vocab": 129280 + } + }, + "routing": { + "distribution": "natural", + "seed": 0 + }, + "shared": { + "a1": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 128 + ], + "static": false + } + }, + "count": 1, + "inter": 3072, + "w1": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + }, + "w2": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + } + } + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn", + "deepseek-ai/DeepSeek-V4-Pro/ffn" + ] + }, + { + "type": "moe", + "args": { + "tokens": 64, + "activation": { + "kind": "silu", + "limit": 10.0 + }, + "experts": { + "a1": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "a2": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "inter": 3072, + "num": 384, + "top_k": 6, + "w1": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + }, + "w2": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + } + }, + "hidden": 7168, + "router": { + "bias": false, + "gate": { + "dtype": "bf16", + "logits": "fp32" + }, + "renormalize": true, + "scale": 2.5, + "scoring": "sqrtsoftplus", + "select": { + "kind": "hash", + "vocab": 129280 + } + }, + "routing": { + "distribution": "natural", + "seed": 0 + }, + "shared": { + "a1": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 128 + ], + "static": false + } + }, + "count": 1, + "inter": 3072, + "w1": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + }, + "w2": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + } + } + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn", + "deepseek-ai/DeepSeek-V4-Pro/ffn" + ] + }, + { + "type": "moe", + "args": { + "tokens": 96, + "activation": { + "kind": "silu", + "limit": 10.0 + }, + "experts": { + "a1": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "a2": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "inter": 3072, + "num": 384, + "top_k": 6, + "w1": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + }, + "w2": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + } + }, + "hidden": 7168, + "router": { + "bias": false, + "gate": { + "dtype": "bf16", + "logits": "fp32" + }, + "renormalize": true, + "scale": 2.5, + "scoring": "sqrtsoftplus", + "select": { + "kind": "hash", + "vocab": 129280 + } + }, + "routing": { + "distribution": "natural", + "seed": 0 + }, + "shared": { + "a1": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 128 + ], + "static": false + } + }, + "count": 1, + "inter": 3072, + "w1": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + }, + "w2": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + } + } + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn", + "deepseek-ai/DeepSeek-V4-Pro/ffn" + ] + }, + { + "type": "moe", + "args": { + "tokens": 128, + "activation": { + "kind": "silu", + "limit": 10.0 + }, + "experts": { + "a1": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "a2": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "inter": 3072, + "num": 384, + "top_k": 6, + "w1": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + }, + "w2": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + } + }, + "hidden": 7168, + "router": { + "bias": false, + "gate": { + "dtype": "bf16", + "logits": "fp32" + }, + "renormalize": true, + "scale": 2.5, + "scoring": "sqrtsoftplus", + "select": { + "kind": "hash", + "vocab": 129280 + } + }, + "routing": { + "distribution": "natural", + "seed": 0 + }, + "shared": { + "a1": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 128 + ], + "static": false + } + }, + "count": 1, + "inter": 3072, + "w1": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + }, + "w2": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + } + } + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn", + "deepseek-ai/DeepSeek-V4-Pro/ffn" + ] + }, + { + "type": "moe", + "args": { + "tokens": 192, + "activation": { + "kind": "silu", + "limit": 10.0 + }, + "experts": { + "a1": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "a2": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "inter": 3072, + "num": 384, + "top_k": 6, + "w1": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + }, + "w2": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + } + }, + "hidden": 7168, + "router": { + "bias": false, + "gate": { + "dtype": "bf16", + "logits": "fp32" + }, + "renormalize": true, + "scale": 2.5, + "scoring": "sqrtsoftplus", + "select": { + "kind": "hash", + "vocab": 129280 + } + }, + "routing": { + "distribution": "natural", + "seed": 0 + }, + "shared": { + "a1": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 128 + ], + "static": false + } + }, + "count": 1, + "inter": 3072, + "w1": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + }, + "w2": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + } + } + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn", + "deepseek-ai/DeepSeek-V4-Pro/ffn" + ] + }, + { + "type": "moe", + "args": { + "tokens": 256, + "activation": { + "kind": "silu", + "limit": 10.0 + }, + "experts": { + "a1": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "a2": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "inter": 3072, + "num": 384, + "top_k": 6, + "w1": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + }, + "w2": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + } + }, + "hidden": 7168, + "router": { + "bias": false, + "gate": { + "dtype": "bf16", + "logits": "fp32" + }, + "renormalize": true, + "scale": 2.5, + "scoring": "sqrtsoftplus", + "select": { + "kind": "hash", + "vocab": 129280 + } + }, + "routing": { + "distribution": "natural", + "seed": 0 + }, + "shared": { + "a1": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 128 + ], + "static": false + } + }, + "count": 1, + "inter": 3072, + "w1": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + }, + "w2": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + } + } + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn", + "deepseek-ai/DeepSeek-V4-Pro/ffn" + ] + }, + { + "type": "moe", + "args": { + "tokens": 384, + "activation": { + "kind": "silu", + "limit": 10.0 + }, + "experts": { + "a1": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "a2": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "inter": 3072, + "num": 384, + "top_k": 6, + "w1": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + }, + "w2": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + } + }, + "hidden": 7168, + "router": { + "bias": false, + "gate": { + "dtype": "bf16", + "logits": "fp32" + }, + "renormalize": true, + "scale": 2.5, + "scoring": "sqrtsoftplus", + "select": { + "kind": "hash", + "vocab": 129280 + } + }, + "routing": { + "distribution": "natural", + "seed": 0 + }, + "shared": { + "a1": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 128 + ], + "static": false + } + }, + "count": 1, + "inter": 3072, + "w1": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + }, + "w2": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + } + } + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn", + "deepseek-ai/DeepSeek-V4-Pro/ffn" + ] + }, + { + "type": "moe", + "args": { + "tokens": 512, + "activation": { + "kind": "silu", + "limit": 10.0 + }, + "experts": { + "a1": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "a2": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "inter": 3072, + "num": 384, + "top_k": 6, + "w1": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + }, + "w2": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + } + }, + "hidden": 7168, + "router": { + "bias": false, + "gate": { + "dtype": "bf16", + "logits": "fp32" + }, + "renormalize": true, + "scale": 2.5, + "scoring": "sqrtsoftplus", + "select": { + "kind": "hash", + "vocab": 129280 + } + }, + "routing": { + "distribution": "natural", + "seed": 0 + }, + "shared": { + "a1": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 128 + ], + "static": false + } + }, + "count": 1, + "inter": 3072, + "w1": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + }, + "w2": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + } + } + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn", + "deepseek-ai/DeepSeek-V4-Pro/ffn" + ] + }, + { + "type": "moe", + "args": { + "tokens": 768, + "activation": { + "kind": "silu", + "limit": 10.0 + }, + "experts": { + "a1": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "a2": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "inter": 3072, + "num": 384, + "top_k": 6, + "w1": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + }, + "w2": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + } + }, + "hidden": 7168, + "router": { + "bias": false, + "gate": { + "dtype": "bf16", + "logits": "fp32" + }, + "renormalize": true, + "scale": 2.5, + "scoring": "sqrtsoftplus", + "select": { + "kind": "hash", + "vocab": 129280 + } + }, + "routing": { + "distribution": "natural", + "seed": 0 + }, + "shared": { + "a1": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 128 + ], + "static": false + } + }, + "count": 1, + "inter": 3072, + "w1": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + }, + "w2": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + } + } + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn", + "deepseek-ai/DeepSeek-V4-Pro/ffn" + ] + }, + { + "type": "moe", + "args": { + "tokens": 1024, + "activation": { + "kind": "silu", + "limit": 10.0 + }, + "experts": { + "a1": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "a2": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "inter": 3072, + "num": 384, + "top_k": 6, + "w1": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + }, + "w2": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + } + }, + "hidden": 7168, + "router": { + "bias": false, + "gate": { + "dtype": "bf16", + "logits": "fp32" + }, + "renormalize": true, + "scale": 2.5, + "scoring": "sqrtsoftplus", + "select": { + "kind": "hash", + "vocab": 129280 + } + }, + "routing": { + "distribution": "natural", + "seed": 0 + }, + "shared": { + "a1": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 128 + ], + "static": false + } + }, + "count": 1, + "inter": 3072, + "w1": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + }, + "w2": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + } + } + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn", + "deepseek-ai/DeepSeek-V4-Pro/ffn" + ] + }, + { + "type": "moe", + "args": { + "tokens": 1536, + "activation": { + "kind": "silu", + "limit": 10.0 + }, + "experts": { + "a1": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "a2": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "inter": 3072, + "num": 384, + "top_k": 6, + "w1": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + }, + "w2": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + } + }, + "hidden": 7168, + "router": { + "bias": false, + "gate": { + "dtype": "bf16", + "logits": "fp32" + }, + "renormalize": true, + "scale": 2.5, + "scoring": "sqrtsoftplus", + "select": { + "kind": "hash", + "vocab": 129280 + } + }, + "routing": { + "distribution": "natural", + "seed": 0 + }, + "shared": { + "a1": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 128 + ], + "static": false + } + }, + "count": 1, + "inter": 3072, + "w1": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + }, + "w2": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + } + } + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn", + "deepseek-ai/DeepSeek-V4-Pro/ffn" + ] + }, + { + "type": "moe", + "args": { + "tokens": 2048, + "activation": { + "kind": "silu", + "limit": 10.0 + }, + "experts": { + "a1": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "a2": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "inter": 3072, + "num": 384, + "top_k": 6, + "w1": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + }, + "w2": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + } + }, + "hidden": 7168, + "router": { + "bias": false, + "gate": { + "dtype": "bf16", + "logits": "fp32" + }, + "renormalize": true, + "scale": 2.5, + "scoring": "sqrtsoftplus", + "select": { + "kind": "hash", + "vocab": 129280 + } + }, + "routing": { + "distribution": "natural", + "seed": 0 + }, + "shared": { + "a1": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 128 + ], + "static": false + } + }, + "count": 1, + "inter": 3072, + "w1": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + }, + "w2": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + } + } + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn", + "deepseek-ai/DeepSeek-V4-Pro/ffn" + ] + }, + { + "type": "moe", + "args": { + "tokens": 3072, + "activation": { + "kind": "silu", + "limit": 10.0 + }, + "experts": { + "a1": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "a2": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "inter": 3072, + "num": 384, + "top_k": 6, + "w1": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + }, + "w2": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + } + }, + "hidden": 7168, + "router": { + "bias": false, + "gate": { + "dtype": "bf16", + "logits": "fp32" + }, + "renormalize": true, + "scale": 2.5, + "scoring": "sqrtsoftplus", + "select": { + "kind": "hash", + "vocab": 129280 + } + }, + "routing": { + "distribution": "natural", + "seed": 0 + }, + "shared": { + "a1": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 128 + ], + "static": false + } + }, + "count": 1, + "inter": 3072, + "w1": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + }, + "w2": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + } + } + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn", + "deepseek-ai/DeepSeek-V4-Pro/ffn" + ] + }, + { + "type": "moe", + "args": { + "tokens": 4096, + "activation": { + "kind": "silu", + "limit": 10.0 + }, + "experts": { + "a1": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "a2": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "inter": 3072, + "num": 384, + "top_k": 6, + "w1": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + }, + "w2": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + } + }, + "hidden": 7168, + "router": { + "bias": false, + "gate": { + "dtype": "bf16", + "logits": "fp32" + }, + "renormalize": true, + "scale": 2.5, + "scoring": "sqrtsoftplus", + "select": { + "kind": "hash", + "vocab": 129280 + } + }, + "routing": { + "distribution": "natural", + "seed": 0 + }, + "shared": { + "a1": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 128 + ], + "static": false + } + }, + "count": 1, + "inter": 3072, + "w1": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + }, + "w2": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + } + } + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn", + "deepseek-ai/DeepSeek-V4-Pro/ffn" + ] + }, + { + "type": "moe", + "args": { + "tokens": 6144, + "activation": { + "kind": "silu", + "limit": 10.0 + }, + "experts": { + "a1": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "a2": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "inter": 3072, + "num": 384, + "top_k": 6, + "w1": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + }, + "w2": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + } + }, + "hidden": 7168, + "router": { + "bias": false, + "gate": { + "dtype": "bf16", + "logits": "fp32" + }, + "renormalize": true, + "scale": 2.5, + "scoring": "sqrtsoftplus", + "select": { + "kind": "hash", + "vocab": 129280 + } + }, + "routing": { + "distribution": "natural", + "seed": 0 + }, + "shared": { + "a1": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 128 + ], + "static": false + } + }, + "count": 1, + "inter": 3072, + "w1": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + }, + "w2": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + } + } + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn", + "deepseek-ai/DeepSeek-V4-Pro/ffn" + ] + }, + { + "type": "moe", + "args": { + "tokens": 8192, + "activation": { + "kind": "silu", + "limit": 10.0 + }, + "experts": { + "a1": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "a2": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "inter": 3072, + "num": 384, + "top_k": 6, + "w1": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + }, + "w2": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + } + }, + "hidden": 7168, + "router": { + "bias": false, + "gate": { + "dtype": "bf16", + "logits": "fp32" + }, + "renormalize": true, + "scale": 2.5, + "scoring": "sqrtsoftplus", + "select": { + "kind": "hash", + "vocab": 129280 + } + }, + "routing": { + "distribution": "natural", + "seed": 0 + }, + "shared": { + "a1": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 128 + ], + "static": false + } + }, + "count": 1, + "inter": 3072, + "w1": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + }, + "w2": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + } + } + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn", + "deepseek-ai/DeepSeek-V4-Pro/ffn" + ] + }, + { + "type": "moe", + "args": { + "tokens": 12288, + "activation": { + "kind": "silu", + "limit": 10.0 + }, + "experts": { + "a1": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "a2": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "inter": 3072, + "num": 384, + "top_k": 6, + "w1": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + }, + "w2": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + } + }, + "hidden": 7168, + "router": { + "bias": false, + "gate": { + "dtype": "bf16", + "logits": "fp32" + }, + "renormalize": true, + "scale": 2.5, + "scoring": "sqrtsoftplus", + "select": { + "kind": "hash", + "vocab": 129280 + } + }, + "routing": { + "distribution": "natural", + "seed": 0 + }, + "shared": { + "a1": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 128 + ], + "static": false + } + }, + "count": 1, + "inter": 3072, + "w1": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + }, + "w2": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + } + } + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn", + "deepseek-ai/DeepSeek-V4-Pro/ffn" + ] + }, + { + "type": "moe", + "args": { + "tokens": 16384, + "activation": { + "kind": "silu", + "limit": 10.0 + }, + "experts": { + "a1": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "a2": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "inter": 3072, + "num": 384, + "top_k": 6, + "w1": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + }, + "w2": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + } + }, + "hidden": 7168, + "router": { + "bias": false, + "gate": { + "dtype": "bf16", + "logits": "fp32" + }, + "renormalize": true, + "scale": 2.5, + "scoring": "sqrtsoftplus", + "select": { + "kind": "hash", + "vocab": 129280 + } + }, + "routing": { + "distribution": "natural", + "seed": 0 + }, + "shared": { + "a1": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 128 + ], + "static": false + } + }, + "count": 1, + "inter": 3072, + "w1": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + }, + "w2": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + } + } + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn", + "deepseek-ai/DeepSeek-V4-Pro/ffn" + ] + }, + { + "type": "moe", + "args": { + "tokens": 24576, + "activation": { + "kind": "silu", + "limit": 10.0 + }, + "experts": { + "a1": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "a2": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "inter": 3072, + "num": 384, + "top_k": 6, + "w1": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + }, + "w2": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + } + }, + "hidden": 7168, + "router": { + "bias": false, + "gate": { + "dtype": "bf16", + "logits": "fp32" + }, + "renormalize": true, + "scale": 2.5, + "scoring": "sqrtsoftplus", + "select": { + "kind": "hash", + "vocab": 129280 + } + }, + "routing": { + "distribution": "natural", + "seed": 0 + }, + "shared": { + "a1": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 128 + ], + "static": false + } + }, + "count": 1, + "inter": 3072, + "w1": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + }, + "w2": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + } + } + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn", + "deepseek-ai/DeepSeek-V4-Pro/ffn" + ] + }, + { + "type": "moe", + "args": { + "tokens": 32768, + "activation": { + "kind": "silu", + "limit": 10.0 + }, + "experts": { + "a1": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "a2": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "inter": 3072, + "num": 384, + "top_k": 6, + "w1": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + }, + "w2": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + } + }, + "hidden": 7168, + "router": { + "bias": false, + "gate": { + "dtype": "bf16", + "logits": "fp32" + }, + "renormalize": true, + "scale": 2.5, + "scoring": "sqrtsoftplus", + "select": { + "kind": "hash", + "vocab": 129280 + } + }, + "routing": { + "distribution": "natural", + "seed": 0 + }, + "shared": { + "a1": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 128 + ], + "static": false + } + }, + "count": 1, + "inter": 3072, + "w1": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + }, + "w2": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + } + } + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn", + "deepseek-ai/DeepSeek-V4-Pro/ffn" + ] + }, + { + "type": "moe", + "args": { + "tokens": 49152, + "activation": { + "kind": "silu", + "limit": 10.0 + }, + "experts": { + "a1": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "a2": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "inter": 3072, + "num": 384, + "top_k": 6, + "w1": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + }, + "w2": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + } + }, + "hidden": 7168, + "router": { + "bias": false, + "gate": { + "dtype": "bf16", + "logits": "fp32" + }, + "renormalize": true, + "scale": 2.5, + "scoring": "sqrtsoftplus", + "select": { + "kind": "hash", + "vocab": 129280 + } + }, + "routing": { + "distribution": "natural", + "seed": 0 + }, + "shared": { + "a1": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 128 + ], + "static": false + } + }, + "count": 1, + "inter": 3072, + "w1": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + }, + "w2": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + } + } + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn", + "deepseek-ai/DeepSeek-V4-Pro/ffn" + ] + }, + { + "type": "moe", + "args": { + "tokens": 65536, + "activation": { + "kind": "silu", + "limit": 10.0 + }, + "experts": { + "a1": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "a2": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "inter": 3072, + "num": 384, + "top_k": 6, + "w1": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + }, + "w2": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + } + }, + "hidden": 7168, + "router": { + "bias": false, + "gate": { + "dtype": "bf16", + "logits": "fp32" + }, + "renormalize": true, + "scale": 2.5, + "scoring": "sqrtsoftplus", + "select": { + "kind": "hash", + "vocab": 129280 + } + }, + "routing": { + "distribution": "natural", + "seed": 0 + }, + "shared": { + "a1": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 128 + ], + "static": false + } + }, + "count": 1, + "inter": 3072, + "w1": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + }, + "w2": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + } + } + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn", + "deepseek-ai/DeepSeek-V4-Pro/ffn" + ] + }, + { + "type": "moe", + "args": { + "tokens": 1, + "activation": { + "kind": "silu", + "limit": 10.0 + }, + "experts": { + "a1": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "a2": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "inter": 2304, + "num": 384, + "top_k": 6, + "w1": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + }, + "w2": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + } + }, + "hidden": 5120, + "router": { + "bias": true, + "gate": { + "dtype": "bf16", + "logits": "fp32" + }, + "renormalize": true, + "scale": 1.5, + "scoring": "sqrtsoftplus", + "select": { + "kind": "topk" + } + }, + "routing": { + "distribution": "natural", + "seed": 0 + }, + "shared": { + "a1": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "count": 1, + "inter": 2304, + "w1": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + }, + "w2": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + } + }, + "sources": [ + "deepseek-ai/DeepSeek-V4.1-Flash/ffn" + ] + }, + { + "type": "moe", + "args": { + "tokens": 2, + "activation": { + "kind": "silu", + "limit": 10.0 + }, + "experts": { + "a1": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "a2": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "inter": 2304, + "num": 384, + "top_k": 6, + "w1": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + }, + "w2": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + } + }, + "hidden": 5120, + "router": { + "bias": true, + "gate": { + "dtype": "bf16", + "logits": "fp32" + }, + "renormalize": true, + "scale": 1.5, + "scoring": "sqrtsoftplus", + "select": { + "kind": "topk" + } + }, + "routing": { + "distribution": "natural", + "seed": 0 + }, + "shared": { + "a1": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "count": 1, + "inter": 2304, + "w1": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + }, + "w2": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + } + }, + "sources": [ + "deepseek-ai/DeepSeek-V4.1-Flash/ffn" + ] + }, + { + "type": "moe", + "args": { + "tokens": 3, + "activation": { + "kind": "silu", + "limit": 10.0 + }, + "experts": { + "a1": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "a2": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "inter": 2304, + "num": 384, + "top_k": 6, + "w1": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + }, + "w2": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + } + }, + "hidden": 5120, + "router": { + "bias": true, + "gate": { + "dtype": "bf16", + "logits": "fp32" + }, + "renormalize": true, + "scale": 1.5, + "scoring": "sqrtsoftplus", + "select": { + "kind": "topk" + } + }, + "routing": { + "distribution": "natural", + "seed": 0 + }, + "shared": { + "a1": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "count": 1, + "inter": 2304, + "w1": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + }, + "w2": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + } + }, + "sources": [ + "deepseek-ai/DeepSeek-V4.1-Flash/ffn" + ] + }, + { + "type": "moe", + "args": { + "tokens": 4, + "activation": { + "kind": "silu", + "limit": 10.0 + }, + "experts": { + "a1": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "a2": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "inter": 2304, + "num": 384, + "top_k": 6, + "w1": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + }, + "w2": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + } + }, + "hidden": 5120, + "router": { + "bias": true, + "gate": { + "dtype": "bf16", + "logits": "fp32" + }, + "renormalize": true, + "scale": 1.5, + "scoring": "sqrtsoftplus", + "select": { + "kind": "topk" + } + }, + "routing": { + "distribution": "natural", + "seed": 0 + }, + "shared": { + "a1": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "count": 1, + "inter": 2304, + "w1": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + }, + "w2": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + } + }, + "sources": [ + "deepseek-ai/DeepSeek-V4.1-Flash/ffn" + ] + }, + { + "type": "moe", + "args": { + "tokens": 6, + "activation": { + "kind": "silu", + "limit": 10.0 + }, + "experts": { + "a1": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "a2": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "inter": 2304, + "num": 384, + "top_k": 6, + "w1": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + }, + "w2": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + } + }, + "hidden": 5120, + "router": { + "bias": true, + "gate": { + "dtype": "bf16", + "logits": "fp32" + }, + "renormalize": true, + "scale": 1.5, + "scoring": "sqrtsoftplus", + "select": { + "kind": "topk" + } + }, + "routing": { + "distribution": "natural", + "seed": 0 + }, + "shared": { + "a1": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "count": 1, + "inter": 2304, + "w1": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + }, + "w2": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + } + }, + "sources": [ + "deepseek-ai/DeepSeek-V4.1-Flash/ffn" + ] + }, + { + "type": "moe", + "args": { + "tokens": 8, + "activation": { + "kind": "silu", + "limit": 10.0 + }, + "experts": { + "a1": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "a2": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "inter": 2304, + "num": 384, + "top_k": 6, + "w1": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + }, + "w2": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + } + }, + "hidden": 5120, + "router": { + "bias": true, + "gate": { + "dtype": "bf16", + "logits": "fp32" + }, + "renormalize": true, + "scale": 1.5, + "scoring": "sqrtsoftplus", + "select": { + "kind": "topk" + } + }, + "routing": { + "distribution": "natural", + "seed": 0 + }, + "shared": { + "a1": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "count": 1, + "inter": 2304, + "w1": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + }, + "w2": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + } + }, + "sources": [ + "deepseek-ai/DeepSeek-V4.1-Flash/ffn" + ] + }, + { + "type": "moe", + "args": { + "tokens": 12, + "activation": { + "kind": "silu", + "limit": 10.0 + }, + "experts": { + "a1": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "a2": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "inter": 2304, + "num": 384, + "top_k": 6, + "w1": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + }, + "w2": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + } + }, + "hidden": 5120, + "router": { + "bias": true, + "gate": { + "dtype": "bf16", + "logits": "fp32" + }, + "renormalize": true, + "scale": 1.5, + "scoring": "sqrtsoftplus", + "select": { + "kind": "topk" + } + }, + "routing": { + "distribution": "natural", + "seed": 0 + }, + "shared": { + "a1": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "count": 1, + "inter": 2304, + "w1": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + }, + "w2": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + } + }, + "sources": [ + "deepseek-ai/DeepSeek-V4.1-Flash/ffn" + ] + }, + { + "type": "moe", + "args": { + "tokens": 16, + "activation": { + "kind": "silu", + "limit": 10.0 + }, + "experts": { + "a1": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "a2": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "inter": 2304, + "num": 384, + "top_k": 6, + "w1": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + }, + "w2": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + } + }, + "hidden": 5120, + "router": { + "bias": true, + "gate": { + "dtype": "bf16", + "logits": "fp32" + }, + "renormalize": true, + "scale": 1.5, + "scoring": "sqrtsoftplus", + "select": { + "kind": "topk" + } + }, + "routing": { + "distribution": "natural", + "seed": 0 + }, + "shared": { + "a1": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "count": 1, + "inter": 2304, + "w1": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + }, + "w2": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + } + }, + "sources": [ + "deepseek-ai/DeepSeek-V4.1-Flash/ffn" + ] + }, + { + "type": "moe", + "args": { + "tokens": 24, + "activation": { + "kind": "silu", + "limit": 10.0 + }, + "experts": { + "a1": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "a2": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "inter": 2304, + "num": 384, + "top_k": 6, + "w1": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + }, + "w2": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + } + }, + "hidden": 5120, + "router": { + "bias": true, + "gate": { + "dtype": "bf16", + "logits": "fp32" + }, + "renormalize": true, + "scale": 1.5, + "scoring": "sqrtsoftplus", + "select": { + "kind": "topk" + } + }, + "routing": { + "distribution": "natural", + "seed": 0 + }, + "shared": { + "a1": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "count": 1, + "inter": 2304, + "w1": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + }, + "w2": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + } + }, + "sources": [ + "deepseek-ai/DeepSeek-V4.1-Flash/ffn" + ] + }, + { + "type": "moe", + "args": { + "tokens": 32, + "activation": { + "kind": "silu", + "limit": 10.0 + }, + "experts": { + "a1": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "a2": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "inter": 2304, + "num": 384, + "top_k": 6, + "w1": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + }, + "w2": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + } + }, + "hidden": 5120, + "router": { + "bias": true, + "gate": { + "dtype": "bf16", + "logits": "fp32" + }, + "renormalize": true, + "scale": 1.5, + "scoring": "sqrtsoftplus", + "select": { + "kind": "topk" + } + }, + "routing": { + "distribution": "natural", + "seed": 0 + }, + "shared": { + "a1": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "count": 1, + "inter": 2304, + "w1": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + }, + "w2": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + } + }, + "sources": [ + "deepseek-ai/DeepSeek-V4.1-Flash/ffn" + ] + }, + { + "type": "moe", + "args": { + "tokens": 48, + "activation": { + "kind": "silu", + "limit": 10.0 + }, + "experts": { + "a1": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "a2": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "inter": 2304, + "num": 384, + "top_k": 6, + "w1": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + }, + "w2": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + } + }, + "hidden": 5120, + "router": { + "bias": true, + "gate": { + "dtype": "bf16", + "logits": "fp32" + }, + "renormalize": true, + "scale": 1.5, + "scoring": "sqrtsoftplus", + "select": { + "kind": "topk" + } + }, + "routing": { + "distribution": "natural", + "seed": 0 + }, + "shared": { + "a1": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "count": 1, + "inter": 2304, + "w1": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + }, + "w2": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + } + }, + "sources": [ + "deepseek-ai/DeepSeek-V4.1-Flash/ffn" + ] + }, + { + "type": "moe", + "args": { + "tokens": 64, + "activation": { + "kind": "silu", + "limit": 10.0 + }, + "experts": { + "a1": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "a2": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "inter": 2304, + "num": 384, + "top_k": 6, + "w1": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + }, + "w2": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + } + }, + "hidden": 5120, + "router": { + "bias": true, + "gate": { + "dtype": "bf16", + "logits": "fp32" + }, + "renormalize": true, + "scale": 1.5, + "scoring": "sqrtsoftplus", + "select": { + "kind": "topk" + } + }, + "routing": { + "distribution": "natural", + "seed": 0 + }, + "shared": { + "a1": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "count": 1, + "inter": 2304, + "w1": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + }, + "w2": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + } + }, + "sources": [ + "deepseek-ai/DeepSeek-V4.1-Flash/ffn" + ] + }, + { + "type": "moe", + "args": { + "tokens": 96, + "activation": { + "kind": "silu", + "limit": 10.0 + }, + "experts": { + "a1": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "a2": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "inter": 2304, + "num": 384, + "top_k": 6, + "w1": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + }, + "w2": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + } + }, + "hidden": 5120, + "router": { + "bias": true, + "gate": { + "dtype": "bf16", + "logits": "fp32" + }, + "renormalize": true, + "scale": 1.5, + "scoring": "sqrtsoftplus", + "select": { + "kind": "topk" + } + }, + "routing": { + "distribution": "natural", + "seed": 0 + }, + "shared": { + "a1": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "count": 1, + "inter": 2304, + "w1": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + }, + "w2": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + } + }, + "sources": [ + "deepseek-ai/DeepSeek-V4.1-Flash/ffn" + ] + }, + { + "type": "moe", + "args": { + "tokens": 128, + "activation": { + "kind": "silu", + "limit": 10.0 + }, + "experts": { + "a1": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "a2": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "inter": 2304, + "num": 384, + "top_k": 6, + "w1": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + }, + "w2": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + } + }, + "hidden": 5120, + "router": { + "bias": true, + "gate": { + "dtype": "bf16", + "logits": "fp32" + }, + "renormalize": true, + "scale": 1.5, + "scoring": "sqrtsoftplus", + "select": { + "kind": "topk" + } + }, + "routing": { + "distribution": "natural", + "seed": 0 + }, + "shared": { + "a1": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "count": 1, + "inter": 2304, + "w1": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + }, + "w2": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + } + }, + "sources": [ + "deepseek-ai/DeepSeek-V4.1-Flash/ffn" + ] + }, + { + "type": "moe", + "args": { + "tokens": 192, + "activation": { + "kind": "silu", + "limit": 10.0 + }, + "experts": { + "a1": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "a2": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "inter": 2304, + "num": 384, + "top_k": 6, + "w1": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + }, + "w2": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + } + }, + "hidden": 5120, + "router": { + "bias": true, + "gate": { + "dtype": "bf16", + "logits": "fp32" + }, + "renormalize": true, + "scale": 1.5, + "scoring": "sqrtsoftplus", + "select": { + "kind": "topk" + } + }, + "routing": { + "distribution": "natural", + "seed": 0 + }, + "shared": { + "a1": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "count": 1, + "inter": 2304, + "w1": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + }, + "w2": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + } + }, + "sources": [ + "deepseek-ai/DeepSeek-V4.1-Flash/ffn" + ] + }, + { + "type": "moe", + "args": { + "tokens": 256, + "activation": { + "kind": "silu", + "limit": 10.0 + }, + "experts": { + "a1": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "a2": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "inter": 2304, + "num": 384, + "top_k": 6, + "w1": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + }, + "w2": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + } + }, + "hidden": 5120, + "router": { + "bias": true, + "gate": { + "dtype": "bf16", + "logits": "fp32" + }, + "renormalize": true, + "scale": 1.5, + "scoring": "sqrtsoftplus", + "select": { + "kind": "topk" + } + }, + "routing": { + "distribution": "natural", + "seed": 0 + }, + "shared": { + "a1": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "count": 1, + "inter": 2304, + "w1": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + }, + "w2": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + } + }, + "sources": [ + "deepseek-ai/DeepSeek-V4.1-Flash/ffn" + ] + }, + { + "type": "moe", + "args": { + "tokens": 384, + "activation": { + "kind": "silu", + "limit": 10.0 + }, + "experts": { + "a1": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "a2": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "inter": 2304, + "num": 384, + "top_k": 6, + "w1": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + }, + "w2": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + } + }, + "hidden": 5120, + "router": { + "bias": true, + "gate": { + "dtype": "bf16", + "logits": "fp32" + }, + "renormalize": true, + "scale": 1.5, + "scoring": "sqrtsoftplus", + "select": { + "kind": "topk" + } + }, + "routing": { + "distribution": "natural", + "seed": 0 + }, + "shared": { + "a1": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "count": 1, + "inter": 2304, + "w1": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + }, + "w2": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + } + }, + "sources": [ + "deepseek-ai/DeepSeek-V4.1-Flash/ffn" + ] + }, + { + "type": "moe", + "args": { + "tokens": 512, + "activation": { + "kind": "silu", + "limit": 10.0 + }, + "experts": { + "a1": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "a2": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "inter": 2304, + "num": 384, + "top_k": 6, + "w1": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + }, + "w2": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + } + }, + "hidden": 5120, + "router": { + "bias": true, + "gate": { + "dtype": "bf16", + "logits": "fp32" + }, + "renormalize": true, + "scale": 1.5, + "scoring": "sqrtsoftplus", + "select": { + "kind": "topk" + } + }, + "routing": { + "distribution": "natural", + "seed": 0 + }, + "shared": { + "a1": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "count": 1, + "inter": 2304, + "w1": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + }, + "w2": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + } + }, + "sources": [ + "deepseek-ai/DeepSeek-V4.1-Flash/ffn" + ] + }, + { + "type": "moe", + "args": { + "tokens": 768, + "activation": { + "kind": "silu", + "limit": 10.0 + }, + "experts": { + "a1": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "a2": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "inter": 2304, + "num": 384, + "top_k": 6, + "w1": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + }, + "w2": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + } + }, + "hidden": 5120, + "router": { + "bias": true, + "gate": { + "dtype": "bf16", + "logits": "fp32" + }, + "renormalize": true, + "scale": 1.5, + "scoring": "sqrtsoftplus", + "select": { + "kind": "topk" + } + }, + "routing": { + "distribution": "natural", + "seed": 0 + }, + "shared": { + "a1": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "count": 1, + "inter": 2304, + "w1": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + }, + "w2": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + } + }, + "sources": [ + "deepseek-ai/DeepSeek-V4.1-Flash/ffn" + ] + }, + { + "type": "moe", + "args": { + "tokens": 1024, + "activation": { + "kind": "silu", + "limit": 10.0 + }, + "experts": { + "a1": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "a2": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "inter": 2304, + "num": 384, + "top_k": 6, + "w1": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + }, + "w2": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + } + }, + "hidden": 5120, + "router": { + "bias": true, + "gate": { + "dtype": "bf16", + "logits": "fp32" + }, + "renormalize": true, + "scale": 1.5, + "scoring": "sqrtsoftplus", + "select": { + "kind": "topk" + } + }, + "routing": { + "distribution": "natural", + "seed": 0 + }, + "shared": { + "a1": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "count": 1, + "inter": 2304, + "w1": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + }, + "w2": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + } + }, + "sources": [ + "deepseek-ai/DeepSeek-V4.1-Flash/ffn" + ] + }, + { + "type": "moe", + "args": { + "tokens": 1536, + "activation": { + "kind": "silu", + "limit": 10.0 + }, + "experts": { + "a1": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "a2": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "inter": 2304, + "num": 384, + "top_k": 6, + "w1": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + }, + "w2": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + } + }, + "hidden": 5120, + "router": { + "bias": true, + "gate": { + "dtype": "bf16", + "logits": "fp32" + }, + "renormalize": true, + "scale": 1.5, + "scoring": "sqrtsoftplus", + "select": { + "kind": "topk" + } + }, + "routing": { + "distribution": "natural", + "seed": 0 + }, + "shared": { + "a1": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "count": 1, + "inter": 2304, + "w1": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + }, + "w2": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + } + }, + "sources": [ + "deepseek-ai/DeepSeek-V4.1-Flash/ffn" + ] + }, + { + "type": "moe", + "args": { + "tokens": 2048, + "activation": { + "kind": "silu", + "limit": 10.0 + }, + "experts": { + "a1": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "a2": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "inter": 2304, + "num": 384, + "top_k": 6, + "w1": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + }, + "w2": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + } + }, + "hidden": 5120, + "router": { + "bias": true, + "gate": { + "dtype": "bf16", + "logits": "fp32" + }, + "renormalize": true, + "scale": 1.5, + "scoring": "sqrtsoftplus", + "select": { + "kind": "topk" + } + }, + "routing": { + "distribution": "natural", + "seed": 0 + }, + "shared": { + "a1": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "count": 1, + "inter": 2304, + "w1": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + }, + "w2": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + } + }, + "sources": [ + "deepseek-ai/DeepSeek-V4.1-Flash/ffn" + ] + }, + { + "type": "moe", + "args": { + "tokens": 3072, + "activation": { + "kind": "silu", + "limit": 10.0 + }, + "experts": { + "a1": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "a2": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "inter": 2304, + "num": 384, + "top_k": 6, + "w1": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + }, + "w2": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + } + }, + "hidden": 5120, + "router": { + "bias": true, + "gate": { + "dtype": "bf16", + "logits": "fp32" + }, + "renormalize": true, + "scale": 1.5, + "scoring": "sqrtsoftplus", + "select": { + "kind": "topk" + } + }, + "routing": { + "distribution": "natural", + "seed": 0 + }, + "shared": { + "a1": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "count": 1, + "inter": 2304, + "w1": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + }, + "w2": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + } + }, + "sources": [ + "deepseek-ai/DeepSeek-V4.1-Flash/ffn" + ] + }, + { + "type": "moe", + "args": { + "tokens": 4096, + "activation": { + "kind": "silu", + "limit": 10.0 + }, + "experts": { + "a1": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "a2": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "inter": 2304, + "num": 384, + "top_k": 6, + "w1": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + }, + "w2": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + } + }, + "hidden": 5120, + "router": { + "bias": true, + "gate": { + "dtype": "bf16", + "logits": "fp32" + }, + "renormalize": true, + "scale": 1.5, + "scoring": "sqrtsoftplus", + "select": { + "kind": "topk" + } + }, + "routing": { + "distribution": "natural", + "seed": 0 + }, + "shared": { + "a1": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "count": 1, + "inter": 2304, + "w1": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + }, + "w2": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + } + }, + "sources": [ + "deepseek-ai/DeepSeek-V4.1-Flash/ffn" + ] + }, + { + "type": "moe", + "args": { + "tokens": 6144, + "activation": { + "kind": "silu", + "limit": 10.0 + }, + "experts": { + "a1": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "a2": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "inter": 2304, + "num": 384, + "top_k": 6, + "w1": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + }, + "w2": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + } + }, + "hidden": 5120, + "router": { + "bias": true, + "gate": { + "dtype": "bf16", + "logits": "fp32" + }, + "renormalize": true, + "scale": 1.5, + "scoring": "sqrtsoftplus", + "select": { + "kind": "topk" + } + }, + "routing": { + "distribution": "natural", + "seed": 0 + }, + "shared": { + "a1": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "count": 1, + "inter": 2304, + "w1": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + }, + "w2": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + } + }, + "sources": [ + "deepseek-ai/DeepSeek-V4.1-Flash/ffn" + ] + }, + { + "type": "moe", + "args": { + "tokens": 8192, + "activation": { + "kind": "silu", + "limit": 10.0 + }, + "experts": { + "a1": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "a2": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "inter": 2304, + "num": 384, + "top_k": 6, + "w1": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + }, + "w2": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + } + }, + "hidden": 5120, + "router": { + "bias": true, + "gate": { + "dtype": "bf16", + "logits": "fp32" + }, + "renormalize": true, + "scale": 1.5, + "scoring": "sqrtsoftplus", + "select": { + "kind": "topk" + } + }, + "routing": { + "distribution": "natural", + "seed": 0 + }, + "shared": { + "a1": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "count": 1, + "inter": 2304, + "w1": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + }, + "w2": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + } + }, + "sources": [ + "deepseek-ai/DeepSeek-V4.1-Flash/ffn" + ] + }, + { + "type": "moe", + "args": { + "tokens": 12288, + "activation": { + "kind": "silu", + "limit": 10.0 + }, + "experts": { + "a1": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "a2": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "inter": 2304, + "num": 384, + "top_k": 6, + "w1": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + }, + "w2": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + } + }, + "hidden": 5120, + "router": { + "bias": true, + "gate": { + "dtype": "bf16", + "logits": "fp32" + }, + "renormalize": true, + "scale": 1.5, + "scoring": "sqrtsoftplus", + "select": { + "kind": "topk" + } + }, + "routing": { + "distribution": "natural", + "seed": 0 + }, + "shared": { + "a1": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "count": 1, + "inter": 2304, + "w1": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + }, + "w2": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + } + }, + "sources": [ + "deepseek-ai/DeepSeek-V4.1-Flash/ffn" + ] + }, + { + "type": "moe", + "args": { + "tokens": 16384, + "activation": { + "kind": "silu", + "limit": 10.0 + }, + "experts": { + "a1": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "a2": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "inter": 2304, + "num": 384, + "top_k": 6, + "w1": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + }, + "w2": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + } + }, + "hidden": 5120, + "router": { + "bias": true, + "gate": { + "dtype": "bf16", + "logits": "fp32" + }, + "renormalize": true, + "scale": 1.5, + "scoring": "sqrtsoftplus", + "select": { + "kind": "topk" + } + }, + "routing": { + "distribution": "natural", + "seed": 0 + }, + "shared": { + "a1": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "count": 1, + "inter": 2304, + "w1": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + }, + "w2": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + } + }, + "sources": [ + "deepseek-ai/DeepSeek-V4.1-Flash/ffn" + ] + }, + { + "type": "moe", + "args": { + "tokens": 24576, + "activation": { + "kind": "silu", + "limit": 10.0 + }, + "experts": { + "a1": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "a2": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "inter": 2304, + "num": 384, + "top_k": 6, + "w1": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + }, + "w2": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + } + }, + "hidden": 5120, + "router": { + "bias": true, + "gate": { + "dtype": "bf16", + "logits": "fp32" + }, + "renormalize": true, + "scale": 1.5, + "scoring": "sqrtsoftplus", + "select": { + "kind": "topk" + } + }, + "routing": { + "distribution": "natural", + "seed": 0 + }, + "shared": { + "a1": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "count": 1, + "inter": 2304, + "w1": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + }, + "w2": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + } + }, + "sources": [ + "deepseek-ai/DeepSeek-V4.1-Flash/ffn" + ] + }, + { + "type": "moe", + "args": { + "tokens": 32768, + "activation": { + "kind": "silu", + "limit": 10.0 + }, + "experts": { + "a1": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "a2": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "inter": 2304, + "num": 384, + "top_k": 6, + "w1": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + }, + "w2": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + } + }, + "hidden": 5120, + "router": { + "bias": true, + "gate": { + "dtype": "bf16", + "logits": "fp32" + }, + "renormalize": true, + "scale": 1.5, + "scoring": "sqrtsoftplus", + "select": { + "kind": "topk" + } + }, + "routing": { + "distribution": "natural", + "seed": 0 + }, + "shared": { + "a1": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "count": 1, + "inter": 2304, + "w1": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + }, + "w2": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + } + }, + "sources": [ + "deepseek-ai/DeepSeek-V4.1-Flash/ffn" + ] + }, + { + "type": "moe", + "args": { + "tokens": 49152, + "activation": { + "kind": "silu", + "limit": 10.0 + }, + "experts": { + "a1": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "a2": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "inter": 2304, + "num": 384, + "top_k": 6, + "w1": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + }, + "w2": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + } + }, + "hidden": 5120, + "router": { + "bias": true, + "gate": { + "dtype": "bf16", + "logits": "fp32" + }, + "renormalize": true, + "scale": 1.5, + "scoring": "sqrtsoftplus", + "select": { + "kind": "topk" + } + }, + "routing": { + "distribution": "natural", + "seed": 0 + }, + "shared": { + "a1": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "count": 1, + "inter": 2304, + "w1": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + }, + "w2": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + } + }, + "sources": [ + "deepseek-ai/DeepSeek-V4.1-Flash/ffn" + ] + }, + { + "type": "moe", + "args": { + "tokens": 65536, + "activation": { + "kind": "silu", + "limit": 10.0 + }, + "experts": { + "a1": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "a2": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "inter": 2304, + "num": 384, + "top_k": 6, + "w1": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + }, + "w2": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + } + }, + "hidden": 5120, + "router": { + "bias": true, + "gate": { + "dtype": "bf16", + "logits": "fp32" + }, + "renormalize": true, + "scale": 1.5, + "scoring": "sqrtsoftplus", + "select": { + "kind": "topk" + } + }, + "routing": { + "distribution": "natural", + "seed": 0 + }, + "shared": { + "a1": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "count": 1, + "inter": 2304, + "w1": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + }, + "w2": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + } + }, + "sources": [ + "deepseek-ai/DeepSeek-V4.1-Flash/ffn" + ] + } +] diff --git a/operatorx/testlists/moe_gemm.json b/operatorx/testlists/moe_gemm.json deleted file mode 100644 index 6f639178eb..0000000000 --- a/operatorx/testlists/moe_gemm.json +++ /dev/null @@ -1,16764 +0,0 @@ -[ - { - "type": "moe_gemm", - "args": { - "num_tokens": 32768, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "expert_parallel_size": 4, - "routed_tensor_parallel_size": 1, - "shared_tensor_parallel_size": 1, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 1214, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "expert_parallel_size": 8, - "routed_tensor_parallel_size": 1, - "shared_tensor_parallel_size": 1, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 350, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "expert_parallel_size": 8, - "routed_tensor_parallel_size": 1, - "shared_tensor_parallel_size": 1, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 6144, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "expert_parallel_size": 4, - "routed_tensor_parallel_size": 1, - "shared_tensor_parallel_size": 1, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 2, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "expert_parallel_size": 8, - "routed_tensor_parallel_size": 1, - "shared_tensor_parallel_size": 1, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 10240, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "expert_parallel_size": 4, - "routed_tensor_parallel_size": 1, - "shared_tensor_parallel_size": 1, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 15360, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "expert_parallel_size": 4, - "routed_tensor_parallel_size": 1, - "shared_tensor_parallel_size": 1, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 25600, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "expert_parallel_size": 4, - "routed_tensor_parallel_size": 1, - "shared_tensor_parallel_size": 1, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 4, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "expert_parallel_size": 8, - "routed_tensor_parallel_size": 1, - "shared_tensor_parallel_size": 1, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 6, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "expert_parallel_size": 8, - "routed_tensor_parallel_size": 1, - "shared_tensor_parallel_size": 1, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 10, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "expert_parallel_size": 8, - "routed_tensor_parallel_size": 1, - "shared_tensor_parallel_size": 1, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 18, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "expert_parallel_size": 8, - "routed_tensor_parallel_size": 1, - "shared_tensor_parallel_size": 1, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 36, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "expert_parallel_size": 8, - "routed_tensor_parallel_size": 1, - "shared_tensor_parallel_size": 1, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 72, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "expert_parallel_size": 8, - "routed_tensor_parallel_size": 1, - "shared_tensor_parallel_size": 1, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 2484, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "expert_parallel_size": 8, - "routed_tensor_parallel_size": 1, - "shared_tensor_parallel_size": 1, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 4096, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "expert_parallel_size": 8, - "routed_tensor_parallel_size": 1, - "shared_tensor_parallel_size": 1, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 1096, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "expert_parallel_size": 8, - "routed_tensor_parallel_size": 1, - "shared_tensor_parallel_size": 1, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 273, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "expert_parallel_size": 8, - "routed_tensor_parallel_size": 1, - "shared_tensor_parallel_size": 1, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 1, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "expert_parallel_size": 8, - "routed_tensor_parallel_size": 1, - "shared_tensor_parallel_size": 1, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 3, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "expert_parallel_size": 8, - "routed_tensor_parallel_size": 1, - "shared_tensor_parallel_size": 1, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 5, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "expert_parallel_size": 8, - "routed_tensor_parallel_size": 1, - "shared_tensor_parallel_size": 1, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 9, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "expert_parallel_size": 8, - "routed_tensor_parallel_size": 1, - "shared_tensor_parallel_size": 1, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 75, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "expert_parallel_size": 8, - "routed_tensor_parallel_size": 1, - "shared_tensor_parallel_size": 1, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 180, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "expert_parallel_size": 8, - "routed_tensor_parallel_size": 1, - "shared_tensor_parallel_size": 1, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 44, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "expert_parallel_size": 8, - "routed_tensor_parallel_size": 1, - "shared_tensor_parallel_size": 1, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 12, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "expert_parallel_size": 8, - "routed_tensor_parallel_size": 1, - "shared_tensor_parallel_size": 1, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 24, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "expert_parallel_size": 8, - "routed_tensor_parallel_size": 1, - "shared_tensor_parallel_size": 1, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 2192, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "expert_parallel_size": 8, - "routed_tensor_parallel_size": 1, - "shared_tensor_parallel_size": 1, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 3382, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "expert_parallel_size": 8, - "routed_tensor_parallel_size": 1, - "shared_tensor_parallel_size": 1, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 658, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "expert_parallel_size": 8, - "routed_tensor_parallel_size": 1, - "shared_tensor_parallel_size": 1, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 20, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "expert_parallel_size": 8, - "routed_tensor_parallel_size": 1, - "shared_tensor_parallel_size": 1, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 74, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "expert_parallel_size": 8, - "routed_tensor_parallel_size": 1, - "shared_tensor_parallel_size": 1, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 1606, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "expert_parallel_size": 8, - "routed_tensor_parallel_size": 1, - "shared_tensor_parallel_size": 1, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 279, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "expert_parallel_size": 8, - "routed_tensor_parallel_size": 1, - "shared_tensor_parallel_size": 1, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 1111, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "expert_parallel_size": 8, - "routed_tensor_parallel_size": 1, - "shared_tensor_parallel_size": 1, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 8192, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "expert_parallel_size": 4, - "routed_tensor_parallel_size": 1, - "shared_tensor_parallel_size": 1, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 16384, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "expert_parallel_size": 4, - "routed_tensor_parallel_size": 1, - "shared_tensor_parallel_size": 1, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 8, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "expert_parallel_size": 8, - "routed_tensor_parallel_size": 1, - "shared_tensor_parallel_size": 1, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 16, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "expert_parallel_size": 8, - "routed_tensor_parallel_size": 1, - "shared_tensor_parallel_size": 1, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 32, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "expert_parallel_size": 8, - "routed_tensor_parallel_size": 1, - "shared_tensor_parallel_size": 1, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 8192, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "expert_parallel_size": 8, - "routed_tensor_parallel_size": 1, - "shared_tensor_parallel_size": 1, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 2, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "expert_parallel_size": 1, - "routed_tensor_parallel_size": 8, - "shared_tensor_parallel_size": 8, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 32768, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "expert_parallel_size": 8, - "routed_tensor_parallel_size": 1, - "shared_tensor_parallel_size": 1, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 8, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "expert_parallel_size": 1, - "routed_tensor_parallel_size": 8, - "shared_tensor_parallel_size": 8, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 16, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "expert_parallel_size": 1, - "routed_tensor_parallel_size": 8, - "shared_tensor_parallel_size": 8, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 64, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "expert_parallel_size": 1, - "routed_tensor_parallel_size": 8, - "shared_tensor_parallel_size": 8, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 4096, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "expert_parallel_size": 8, - "routed_tensor_parallel_size": 1, - "shared_tensor_parallel_size": 1, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 1, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "expert_parallel_size": 1, - "routed_tensor_parallel_size": 8, - "shared_tensor_parallel_size": 8, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 10, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "expert_parallel_size": 1, - "routed_tensor_parallel_size": 8, - "shared_tensor_parallel_size": 8, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 42, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "expert_parallel_size": 1, - "routed_tensor_parallel_size": 8, - "shared_tensor_parallel_size": 8, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 32768, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "expert_parallel_size": 1, - "routed_tensor_parallel_size": 8, - "shared_tensor_parallel_size": 8, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 32, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "expert_parallel_size": 1, - "routed_tensor_parallel_size": 8, - "shared_tensor_parallel_size": 8, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 149, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "expert_parallel_size": 8, - "routed_tensor_parallel_size": 1, - "shared_tensor_parallel_size": 1, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 576, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "expert_parallel_size": 8, - "routed_tensor_parallel_size": 1, - "shared_tensor_parallel_size": 1, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 2176, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "expert_parallel_size": 8, - "routed_tensor_parallel_size": 1, - "shared_tensor_parallel_size": 1, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 8192, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "expert_parallel_size": 1, - "routed_tensor_parallel_size": 8, - "shared_tensor_parallel_size": 8, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 128, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "expert_parallel_size": 8, - "routed_tensor_parallel_size": 1, - "shared_tensor_parallel_size": 1, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 256, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "expert_parallel_size": 8, - "routed_tensor_parallel_size": 1, - "shared_tensor_parallel_size": 1, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 640, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "expert_parallel_size": 8, - "routed_tensor_parallel_size": 1, - "shared_tensor_parallel_size": 1, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 32768, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "expert_parallel_size": 2, - "routed_tensor_parallel_size": 1, - "shared_tensor_parallel_size": 1, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 1308, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "expert_parallel_size": 8, - "routed_tensor_parallel_size": 1, - "shared_tensor_parallel_size": 1, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 271, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "expert_parallel_size": 8, - "routed_tensor_parallel_size": 1, - "shared_tensor_parallel_size": 1, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 11264, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "expert_parallel_size": 2, - "routed_tensor_parallel_size": 1, - "shared_tensor_parallel_size": 1, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 10240, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "expert_parallel_size": 2, - "routed_tensor_parallel_size": 1, - "shared_tensor_parallel_size": 1, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 20480, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "expert_parallel_size": 2, - "routed_tensor_parallel_size": 1, - "shared_tensor_parallel_size": 1, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 25600, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "expert_parallel_size": 2, - "routed_tensor_parallel_size": 1, - "shared_tensor_parallel_size": 1, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 48, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "expert_parallel_size": 8, - "routed_tensor_parallel_size": 1, - "shared_tensor_parallel_size": 1, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 80, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "expert_parallel_size": 8, - "routed_tensor_parallel_size": 1, - "shared_tensor_parallel_size": 1, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 4684, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "expert_parallel_size": 8, - "routed_tensor_parallel_size": 1, - "shared_tensor_parallel_size": 1, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 8609, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "expert_parallel_size": 8, - "routed_tensor_parallel_size": 1, - "shared_tensor_parallel_size": 1, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 12926, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "expert_parallel_size": 8, - "routed_tensor_parallel_size": 1, - "shared_tensor_parallel_size": 1, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 588, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "expert_parallel_size": 8, - "routed_tensor_parallel_size": 1, - "shared_tensor_parallel_size": 1, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 6144, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "expert_parallel_size": 2, - "routed_tensor_parallel_size": 1, - "shared_tensor_parallel_size": 1, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 5120, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "expert_parallel_size": 2, - "routed_tensor_parallel_size": 1, - "shared_tensor_parallel_size": 1, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 15360, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "expert_parallel_size": 2, - "routed_tensor_parallel_size": 1, - "shared_tensor_parallel_size": 1, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 1, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "expert_parallel_size": 4, - "routed_tensor_parallel_size": 1, - "shared_tensor_parallel_size": 1, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 2, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "expert_parallel_size": 4, - "routed_tensor_parallel_size": 1, - "shared_tensor_parallel_size": 1, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 3, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "expert_parallel_size": 4, - "routed_tensor_parallel_size": 1, - "shared_tensor_parallel_size": 1, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 5, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "expert_parallel_size": 4, - "routed_tensor_parallel_size": 1, - "shared_tensor_parallel_size": 1, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 22, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "expert_parallel_size": 8, - "routed_tensor_parallel_size": 1, - "shared_tensor_parallel_size": 1, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 39, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "expert_parallel_size": 8, - "routed_tensor_parallel_size": 1, - "shared_tensor_parallel_size": 1, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 79, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "expert_parallel_size": 8, - "routed_tensor_parallel_size": 1, - "shared_tensor_parallel_size": 1, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 4405, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "expert_parallel_size": 8, - "routed_tensor_parallel_size": 1, - "shared_tensor_parallel_size": 1, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 8192, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "expert_parallel_size": 8, - "routed_tensor_parallel_size": 1, - "shared_tensor_parallel_size": 1, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 2305, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "expert_parallel_size": 8, - "routed_tensor_parallel_size": 1, - "shared_tensor_parallel_size": 1, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 4396, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "expert_parallel_size": 8, - "routed_tensor_parallel_size": 1, - "shared_tensor_parallel_size": 1, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 26, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "expert_parallel_size": 4, - "routed_tensor_parallel_size": 1, - "shared_tensor_parallel_size": 1, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 362, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "expert_parallel_size": 8, - "routed_tensor_parallel_size": 1, - "shared_tensor_parallel_size": 1, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 2394, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "expert_parallel_size": 8, - "routed_tensor_parallel_size": 1, - "shared_tensor_parallel_size": 1, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 35, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "expert_parallel_size": 4, - "routed_tensor_parallel_size": 1, - "shared_tensor_parallel_size": 1, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 21, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "expert_parallel_size": 8, - "routed_tensor_parallel_size": 1, - "shared_tensor_parallel_size": 1, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 294, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "expert_parallel_size": 8, - "routed_tensor_parallel_size": 1, - "shared_tensor_parallel_size": 1, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 1093, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "expert_parallel_size": 8, - "routed_tensor_parallel_size": 1, - "shared_tensor_parallel_size": 1, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 2048, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "expert_parallel_size": 8, - "routed_tensor_parallel_size": 1, - "shared_tensor_parallel_size": 1, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 10240, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "expert_parallel_size": 1, - "routed_tensor_parallel_size": 4, - "shared_tensor_parallel_size": 4, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 32768, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "expert_parallel_size": 1, - "routed_tensor_parallel_size": 4, - "shared_tensor_parallel_size": 4, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 40, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "expert_parallel_size": 1, - "routed_tensor_parallel_size": 8, - "shared_tensor_parallel_size": 8, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 6144, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "expert_parallel_size": 1, - "routed_tensor_parallel_size": 8, - "shared_tensor_parallel_size": 8, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 2560, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "expert_parallel_size": 8, - "routed_tensor_parallel_size": 1, - "shared_tensor_parallel_size": 1, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 288, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "expert_parallel_size": 8, - "routed_tensor_parallel_size": 1, - "shared_tensor_parallel_size": 1, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 11264, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "expert_parallel_size": 1, - "routed_tensor_parallel_size": 8, - "shared_tensor_parallel_size": 8, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 2112, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "expert_parallel_size": 8, - "routed_tensor_parallel_size": 1, - "shared_tensor_parallel_size": 1, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 1536, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "expert_parallel_size": 1, - "routed_tensor_parallel_size": 8, - "shared_tensor_parallel_size": 8, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 426, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "expert_parallel_size": 1, - "routed_tensor_parallel_size": 8, - "shared_tensor_parallel_size": 8, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 12288, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "expert_parallel_size": 1, - "routed_tensor_parallel_size": 4, - "shared_tensor_parallel_size": 4, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 48, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "expert_parallel_size": 1, - "routed_tensor_parallel_size": 8, - "shared_tensor_parallel_size": 8, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 16384, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "expert_parallel_size": 1, - "routed_tensor_parallel_size": 8, - "shared_tensor_parallel_size": 8, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 144, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "expert_parallel_size": 1, - "routed_tensor_parallel_size": 8, - "shared_tensor_parallel_size": 8, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 288, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "expert_parallel_size": 1, - "routed_tensor_parallel_size": 8, - "shared_tensor_parallel_size": 8, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 1024, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "expert_parallel_size": 1, - "routed_tensor_parallel_size": 8, - "shared_tensor_parallel_size": 8, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 512, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "expert_parallel_size": 1, - "routed_tensor_parallel_size": 8, - "shared_tensor_parallel_size": 8, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 51, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "expert_parallel_size": 1, - "routed_tensor_parallel_size": 8, - "shared_tensor_parallel_size": 8, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 1075, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "expert_parallel_size": 1, - "routed_tensor_parallel_size": 8, - "shared_tensor_parallel_size": 8, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 3072, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "expert_parallel_size": 8, - "routed_tensor_parallel_size": 1, - "shared_tensor_parallel_size": 1, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 4096, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "expert_parallel_size": 4, - "routed_tensor_parallel_size": 1, - "shared_tensor_parallel_size": 1, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 4, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "expert_parallel_size": 4, - "routed_tensor_parallel_size": 1, - "shared_tensor_parallel_size": 1, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 8, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "expert_parallel_size": 4, - "routed_tensor_parallel_size": 1, - "shared_tensor_parallel_size": 1, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 16, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "expert_parallel_size": 4, - "routed_tensor_parallel_size": 1, - "shared_tensor_parallel_size": 1, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 32, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "expert_parallel_size": 4, - "routed_tensor_parallel_size": 1, - "shared_tensor_parallel_size": 1, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 64, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "expert_parallel_size": 4, - "routed_tensor_parallel_size": 1, - "shared_tensor_parallel_size": 1, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 128, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "expert_parallel_size": 4, - "routed_tensor_parallel_size": 1, - "shared_tensor_parallel_size": 1, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 16384, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "expert_parallel_size": 8, - "routed_tensor_parallel_size": 1, - "shared_tensor_parallel_size": 1, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 32768, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "expert_parallel_size": 8, - "routed_tensor_parallel_size": 1, - "shared_tensor_parallel_size": 1, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 64, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "expert_parallel_size": 8, - "routed_tensor_parallel_size": 1, - "shared_tensor_parallel_size": 1, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 128, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "expert_parallel_size": 8, - "routed_tensor_parallel_size": 1, - "shared_tensor_parallel_size": 1, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 12, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "expert_parallel_size": 4, - "routed_tensor_parallel_size": 1, - "shared_tensor_parallel_size": 1, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 24, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "expert_parallel_size": 4, - "routed_tensor_parallel_size": 1, - "shared_tensor_parallel_size": 1, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 48, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "expert_parallel_size": 4, - "routed_tensor_parallel_size": 1, - "shared_tensor_parallel_size": 1, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 256, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "expert_parallel_size": 4, - "routed_tensor_parallel_size": 1, - "shared_tensor_parallel_size": 1, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 256, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "expert_parallel_size": 8, - "routed_tensor_parallel_size": 1, - "shared_tensor_parallel_size": 1, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 512, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "expert_parallel_size": 8, - "routed_tensor_parallel_size": 1, - "shared_tensor_parallel_size": 1, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 1024, - "hidden": 7168, - "intermediate": 3072, - "num_experts": 384, - "top_k": 6, - "dtype_act": "nvfp4", - "dtype_weight": "mxfp4", - "expert_parallel_size": 1, - "routed_tensor_parallel_size": 8, - "shared_tensor_parallel_size": 1, - "n_shared_experts": 1 - }, - "name": "dsv4" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 1, - "hidden": 7168, - "intermediate": 3072, - "num_experts": 384, - "top_k": 6, - "dtype_act": "nvfp4", - "dtype_weight": "mxfp4", - "expert_parallel_size": 1, - "routed_tensor_parallel_size": 8, - "shared_tensor_parallel_size": 1, - "n_shared_experts": 1 - }, - "name": "dsv4" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 2048, - "hidden": 7168, - "intermediate": 3072, - "num_experts": 384, - "top_k": 6, - "dtype_act": "nvfp4", - "dtype_weight": "mxfp4", - "expert_parallel_size": 1, - "routed_tensor_parallel_size": 8, - "shared_tensor_parallel_size": 1, - "n_shared_experts": 1 - }, - "name": "dsv4" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 2, - "hidden": 7168, - "intermediate": 3072, - "num_experts": 384, - "top_k": 6, - "dtype_act": "nvfp4", - "dtype_weight": "mxfp4", - "expert_parallel_size": 1, - "routed_tensor_parallel_size": 8, - "shared_tensor_parallel_size": 1, - "n_shared_experts": 1 - }, - "name": "dsv4" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 4096, - "hidden": 7168, - "intermediate": 3072, - "num_experts": 384, - "top_k": 6, - "dtype_act": "nvfp4", - "dtype_weight": "mxfp4", - "expert_parallel_size": 1, - "routed_tensor_parallel_size": 8, - "shared_tensor_parallel_size": 1, - "n_shared_experts": 1 - }, - "name": "dsv4" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 4, - "hidden": 7168, - "intermediate": 3072, - "num_experts": 384, - "top_k": 6, - "dtype_act": "nvfp4", - "dtype_weight": "mxfp4", - "expert_parallel_size": 1, - "routed_tensor_parallel_size": 8, - "shared_tensor_parallel_size": 1, - "n_shared_experts": 1 - }, - "name": "dsv4" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 8192, - "hidden": 7168, - "intermediate": 3072, - "num_experts": 384, - "top_k": 6, - "dtype_act": "nvfp4", - "dtype_weight": "mxfp4", - "expert_parallel_size": 1, - "routed_tensor_parallel_size": 8, - "shared_tensor_parallel_size": 1, - "n_shared_experts": 1 - }, - "name": "dsv4" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 8, - "hidden": 7168, - "intermediate": 3072, - "num_experts": 384, - "top_k": 6, - "dtype_act": "nvfp4", - "dtype_weight": "mxfp4", - "expert_parallel_size": 1, - "routed_tensor_parallel_size": 8, - "shared_tensor_parallel_size": 1, - "n_shared_experts": 1 - }, - "name": "dsv4" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 16384, - "hidden": 7168, - "intermediate": 3072, - "num_experts": 384, - "top_k": 6, - "dtype_act": "nvfp4", - "dtype_weight": "mxfp4", - "expert_parallel_size": 1, - "routed_tensor_parallel_size": 8, - "shared_tensor_parallel_size": 1, - "n_shared_experts": 1 - }, - "name": "dsv4" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 16, - "hidden": 7168, - "intermediate": 3072, - "num_experts": 384, - "top_k": 6, - "dtype_act": "nvfp4", - "dtype_weight": "mxfp4", - "expert_parallel_size": 1, - "routed_tensor_parallel_size": 8, - "shared_tensor_parallel_size": 1, - "n_shared_experts": 1 - }, - "name": "dsv4" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 32768, - "hidden": 7168, - "intermediate": 3072, - "num_experts": 384, - "top_k": 6, - "dtype_act": "nvfp4", - "dtype_weight": "mxfp4", - "expert_parallel_size": 1, - "routed_tensor_parallel_size": 8, - "shared_tensor_parallel_size": 1, - "n_shared_experts": 1 - }, - "name": "dsv4" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 32, - "hidden": 7168, - "intermediate": 3072, - "num_experts": 384, - "top_k": 6, - "dtype_act": "nvfp4", - "dtype_weight": "mxfp4", - "expert_parallel_size": 1, - "routed_tensor_parallel_size": 8, - "shared_tensor_parallel_size": 1, - "n_shared_experts": 1 - }, - "name": "dsv4" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 32768, - "hidden": 7168, - "intermediate": 3072, - "num_experts": 384, - "top_k": 6, - "dtype_act": "nvfp4", - "dtype_weight": "mxfp4", - "expert_parallel_size": 8, - "routed_tensor_parallel_size": 1, - "shared_tensor_parallel_size": 1, - "n_shared_experts": 1 - }, - "name": "dsv4" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 64, - "hidden": 7168, - "intermediate": 3072, - "num_experts": 384, - "top_k": 6, - "dtype_act": "nvfp4", - "dtype_weight": "mxfp4", - "expert_parallel_size": 8, - "routed_tensor_parallel_size": 1, - "shared_tensor_parallel_size": 1, - "n_shared_experts": 1 - }, - "name": "dsv4" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 128, - "hidden": 7168, - "intermediate": 3072, - "num_experts": 384, - "top_k": 6, - "dtype_act": "nvfp4", - "dtype_weight": "mxfp4", - "expert_parallel_size": 8, - "routed_tensor_parallel_size": 1, - "shared_tensor_parallel_size": 1, - "n_shared_experts": 1 - }, - "name": "dsv4" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 256, - "hidden": 7168, - "intermediate": 3072, - "num_experts": 384, - "top_k": 6, - "dtype_act": "nvfp4", - "dtype_weight": "mxfp4", - "expert_parallel_size": 8, - "routed_tensor_parallel_size": 1, - "shared_tensor_parallel_size": 1, - "n_shared_experts": 1 - }, - "name": "dsv4" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 512, - "hidden": 7168, - "intermediate": 3072, - "num_experts": 384, - "top_k": 6, - "dtype_act": "nvfp4", - "dtype_weight": "mxfp4", - "expert_parallel_size": 8, - "routed_tensor_parallel_size": 1, - "shared_tensor_parallel_size": 1, - "n_shared_experts": 1 - }, - "name": "dsv4" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 1024, - "hidden": 7168, - "intermediate": 3072, - "num_experts": 384, - "top_k": 6, - "dtype_act": "nvfp4", - "dtype_weight": "mxfp4", - "expert_parallel_size": 8, - "routed_tensor_parallel_size": 1, - "shared_tensor_parallel_size": 1, - "n_shared_experts": 1 - }, - "name": "dsv4" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 64, - "hidden": 7168, - "intermediate": 3072, - "num_experts": 384, - "top_k": 6, - "dtype_act": "nvfp4", - "dtype_weight": "mxfp4", - "expert_parallel_size": 1, - "routed_tensor_parallel_size": 8, - "shared_tensor_parallel_size": 1, - "n_shared_experts": 1 - }, - "name": "dsv4" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 2048, - "hidden": 7168, - "intermediate": 3072, - "num_experts": 384, - "top_k": 6, - "dtype_act": "nvfp4", - "dtype_weight": "mxfp4", - "expert_parallel_size": 8, - "routed_tensor_parallel_size": 1, - "shared_tensor_parallel_size": 1, - "n_shared_experts": 1 - }, - "name": "dsv4" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 4096, - "hidden": 7168, - "intermediate": 3072, - "num_experts": 384, - "top_k": 6, - "dtype_act": "nvfp4", - "dtype_weight": "mxfp4", - "expert_parallel_size": 8, - "routed_tensor_parallel_size": 1, - "shared_tensor_parallel_size": 1, - "n_shared_experts": 1 - }, - "name": "dsv4" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 32, - "hidden": 7168, - "intermediate": 3072, - "num_experts": 384, - "top_k": 6, - "dtype_act": "nvfp4", - "dtype_weight": "mxfp4", - "expert_parallel_size": 8, - "routed_tensor_parallel_size": 1, - "shared_tensor_parallel_size": 1, - "n_shared_experts": 1 - }, - "name": "dsv4" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 128, - "hidden": 7168, - "intermediate": 3072, - "num_experts": 384, - "top_k": 6, - "dtype_act": "nvfp4", - "dtype_weight": "mxfp4", - "expert_parallel_size": 1, - "routed_tensor_parallel_size": 8, - "shared_tensor_parallel_size": 1, - "n_shared_experts": 1 - }, - "name": "dsv4" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 8192, - "hidden": 7168, - "intermediate": 3072, - "num_experts": 384, - "top_k": 6, - "dtype_act": "nvfp4", - "dtype_weight": "mxfp4", - "expert_parallel_size": 8, - "routed_tensor_parallel_size": 1, - "shared_tensor_parallel_size": 1, - "n_shared_experts": 1 - }, - "name": "dsv4" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 4096, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "expert_parallel_size": 1, - "routed_tensor_parallel_size": 4, - "shared_tensor_parallel_size": 4, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 4, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "expert_parallel_size": 1, - "routed_tensor_parallel_size": 4, - "shared_tensor_parallel_size": 4, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 8192, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "expert_parallel_size": 1, - "routed_tensor_parallel_size": 4, - "shared_tensor_parallel_size": 4, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 8, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "expert_parallel_size": 1, - "routed_tensor_parallel_size": 4, - "shared_tensor_parallel_size": 4, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 16384, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "expert_parallel_size": 1, - "routed_tensor_parallel_size": 4, - "shared_tensor_parallel_size": 4, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 16, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "expert_parallel_size": 1, - "routed_tensor_parallel_size": 4, - "shared_tensor_parallel_size": 4, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 4096, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "expert_parallel_size": 1, - "routed_tensor_parallel_size": 8, - "shared_tensor_parallel_size": 8, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 4, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "expert_parallel_size": 1, - "routed_tensor_parallel_size": 8, - "shared_tensor_parallel_size": 8, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 32768, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "expert_parallel_size": 1, - "routed_tensor_parallel_size": 4, - "shared_tensor_parallel_size": 4, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 32, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "expert_parallel_size": 1, - "routed_tensor_parallel_size": 4, - "shared_tensor_parallel_size": 4, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 32768, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "expert_parallel_size": 1, - "routed_tensor_parallel_size": 8, - "shared_tensor_parallel_size": 8, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 512, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "expert_parallel_size": 4, - "routed_tensor_parallel_size": 1, - "shared_tensor_parallel_size": 1, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 8, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "expert_parallel_size": 1, - "routed_tensor_parallel_size": 8, - "shared_tensor_parallel_size": 8, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 256, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "expert_parallel_size": 1, - "routed_tensor_parallel_size": 8, - "shared_tensor_parallel_size": 8, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 4096, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "expert_parallel_size": 1, - "routed_tensor_parallel_size": 8, - "shared_tensor_parallel_size": 8, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 4, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "expert_parallel_size": 1, - "routed_tensor_parallel_size": 8, - "shared_tensor_parallel_size": 8, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 4, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "expert_parallel_size": 1, - "routed_tensor_parallel_size": 4, - "shared_tensor_parallel_size": 4, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 8, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "expert_parallel_size": 1, - "routed_tensor_parallel_size": 4, - "shared_tensor_parallel_size": 4, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 16, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "expert_parallel_size": 1, - "routed_tensor_parallel_size": 4, - "shared_tensor_parallel_size": 4, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 32, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "expert_parallel_size": 1, - "routed_tensor_parallel_size": 4, - "shared_tensor_parallel_size": 4, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 32768, - "hidden": 7168, - "intermediate": 3072, - "num_experts": 384, - "top_k": 6, - "dtype_act": "nvfp4", - "dtype_weight": "mxfp4", - "expert_parallel_size": 1, - "routed_tensor_parallel_size": 4, - "shared_tensor_parallel_size": 1, - "n_shared_experts": 1 - }, - "name": "dsv4" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 32, - "hidden": 7168, - "intermediate": 3072, - "num_experts": 384, - "top_k": 6, - "dtype_act": "nvfp4", - "dtype_weight": "mxfp4", - "expert_parallel_size": 1, - "routed_tensor_parallel_size": 4, - "shared_tensor_parallel_size": 1, - "n_shared_experts": 1 - }, - "name": "dsv4" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 32768, - "hidden": 7168, - "intermediate": 3072, - "num_experts": 384, - "top_k": 6, - "dtype_act": "nvfp4", - "dtype_weight": "mxfp4", - "expert_parallel_size": 4, - "routed_tensor_parallel_size": 1, - "shared_tensor_parallel_size": 1, - "n_shared_experts": 1 - }, - "name": "dsv4" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 512, - "hidden": 7168, - "intermediate": 3072, - "num_experts": 384, - "top_k": 6, - "dtype_act": "nvfp4", - "dtype_weight": "mxfp4", - "expert_parallel_size": 4, - "routed_tensor_parallel_size": 1, - "shared_tensor_parallel_size": 1, - "n_shared_experts": 1 - }, - "name": "dsv4" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 4096, - "hidden": 7168, - "intermediate": 3072, - "num_experts": 384, - "top_k": 6, - "dtype_act": "nvfp4", - "dtype_weight": "mxfp4", - "expert_parallel_size": 1, - "routed_tensor_parallel_size": 4, - "shared_tensor_parallel_size": 1, - "n_shared_experts": 1 - }, - "name": "dsv4" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 8, - "hidden": 7168, - "intermediate": 3072, - "num_experts": 384, - "top_k": 6, - "dtype_act": "nvfp4", - "dtype_weight": "mxfp4", - "expert_parallel_size": 1, - "routed_tensor_parallel_size": 4, - "shared_tensor_parallel_size": 1, - "n_shared_experts": 1 - }, - "name": "dsv4" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 8192, - "hidden": 7168, - "intermediate": 3072, - "num_experts": 384, - "top_k": 6, - "dtype_act": "nvfp4", - "dtype_weight": "mxfp4", - "expert_parallel_size": 1, - "routed_tensor_parallel_size": 4, - "shared_tensor_parallel_size": 1, - "n_shared_experts": 1 - }, - "name": "dsv4" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 16, - "hidden": 7168, - "intermediate": 3072, - "num_experts": 384, - "top_k": 6, - "dtype_act": "nvfp4", - "dtype_weight": "mxfp4", - "expert_parallel_size": 1, - "routed_tensor_parallel_size": 4, - "shared_tensor_parallel_size": 1, - "n_shared_experts": 1 - }, - "name": "dsv4" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 16384, - "hidden": 7168, - "intermediate": 3072, - "num_experts": 384, - "top_k": 6, - "dtype_act": "nvfp4", - "dtype_weight": "mxfp4", - "expert_parallel_size": 1, - "routed_tensor_parallel_size": 4, - "shared_tensor_parallel_size": 1, - "n_shared_experts": 1 - }, - "name": "dsv4" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 64, - "hidden": 7168, - "intermediate": 3072, - "num_experts": 384, - "top_k": 6, - "dtype_act": "nvfp4", - "dtype_weight": "mxfp4", - "expert_parallel_size": 1, - "routed_tensor_parallel_size": 4, - "shared_tensor_parallel_size": 1, - "n_shared_experts": 1 - }, - "name": "dsv4" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 4096, - "hidden": 4096, - "intermediate": 1024, - "num_experts": 512, - "top_k": 10, - "dtype_act": "bf16", - "dtype_weight": "bf16", - "expert_parallel_size": 1, - "routed_tensor_parallel_size": 8, - "shared_tensor_parallel_size": 8, - "n_shared_experts": 1 - }, - "name": "qwen3.5" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 4, - "hidden": 4096, - "intermediate": 1024, - "num_experts": 512, - "top_k": 10, - "dtype_act": "bf16", - "dtype_weight": "bf16", - "expert_parallel_size": 1, - "routed_tensor_parallel_size": 8, - "shared_tensor_parallel_size": 8, - "n_shared_experts": 1 - }, - "name": "qwen3.5" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 8192, - "hidden": 4096, - "intermediate": 1024, - "num_experts": 512, - "top_k": 10, - "dtype_act": "bf16", - "dtype_weight": "bf16", - "expert_parallel_size": 1, - "routed_tensor_parallel_size": 8, - "shared_tensor_parallel_size": 8, - "n_shared_experts": 1 - }, - "name": "qwen3.5" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 8, - "hidden": 4096, - "intermediate": 1024, - "num_experts": 512, - "top_k": 10, - "dtype_act": "bf16", - "dtype_weight": "bf16", - "expert_parallel_size": 1, - "routed_tensor_parallel_size": 8, - "shared_tensor_parallel_size": 8, - "n_shared_experts": 1 - }, - "name": "qwen3.5" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 16384, - "hidden": 4096, - "intermediate": 1024, - "num_experts": 512, - "top_k": 10, - "dtype_act": "bf16", - "dtype_weight": "bf16", - "expert_parallel_size": 1, - "routed_tensor_parallel_size": 8, - "shared_tensor_parallel_size": 8, - "n_shared_experts": 1 - }, - "name": "qwen3.5" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 16, - "hidden": 4096, - "intermediate": 1024, - "num_experts": 512, - "top_k": 10, - "dtype_act": "bf16", - "dtype_weight": "bf16", - "expert_parallel_size": 1, - "routed_tensor_parallel_size": 8, - "shared_tensor_parallel_size": 8, - "n_shared_experts": 1 - }, - "name": "qwen3.5" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 32768, - "hidden": 4096, - "intermediate": 1024, - "num_experts": 512, - "top_k": 10, - "dtype_act": "bf16", - "dtype_weight": "bf16", - "expert_parallel_size": 1, - "routed_tensor_parallel_size": 8, - "shared_tensor_parallel_size": 8, - "n_shared_experts": 1 - }, - "name": "qwen3.5" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 32, - "hidden": 4096, - "intermediate": 1024, - "num_experts": 512, - "top_k": 10, - "dtype_act": "bf16", - "dtype_weight": "bf16", - "expert_parallel_size": 1, - "routed_tensor_parallel_size": 8, - "shared_tensor_parallel_size": 8, - "n_shared_experts": 1 - }, - "name": "qwen3.5" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 64, - "hidden": 4096, - "intermediate": 1024, - "num_experts": 512, - "top_k": 10, - "dtype_act": "bf16", - "dtype_weight": "bf16", - "expert_parallel_size": 1, - "routed_tensor_parallel_size": 8, - "shared_tensor_parallel_size": 8, - "n_shared_experts": 1 - }, - "name": "qwen3.5" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 128, - "hidden": 4096, - "intermediate": 1024, - "num_experts": 512, - "top_k": 10, - "dtype_act": "bf16", - "dtype_weight": "bf16", - "expert_parallel_size": 1, - "routed_tensor_parallel_size": 8, - "shared_tensor_parallel_size": 8, - "n_shared_experts": 1 - }, - "name": "qwen3.5" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 4096, - "hidden": 4096, - "intermediate": 1024, - "num_experts": 512, - "top_k": 10, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "expert_parallel_size": 1, - "routed_tensor_parallel_size": 8, - "shared_tensor_parallel_size": 8, - "n_shared_experts": 1 - }, - "name": "qwen3.5" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 4, - "hidden": 4096, - "intermediate": 1024, - "num_experts": 512, - "top_k": 10, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "expert_parallel_size": 1, - "routed_tensor_parallel_size": 8, - "shared_tensor_parallel_size": 8, - "n_shared_experts": 1 - }, - "name": "qwen3.5" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 4096, - "hidden": 4096, - "intermediate": 1024, - "num_experts": 512, - "top_k": 10, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "expert_parallel_size": 1, - "routed_tensor_parallel_size": 4, - "shared_tensor_parallel_size": 4, - "n_shared_experts": 1 - }, - "name": "qwen3.5" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 4, - "hidden": 4096, - "intermediate": 1024, - "num_experts": 512, - "top_k": 10, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "expert_parallel_size": 1, - "routed_tensor_parallel_size": 4, - "shared_tensor_parallel_size": 4, - "n_shared_experts": 1 - }, - "name": "qwen3.5" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 8192, - "hidden": 4096, - "intermediate": 1024, - "num_experts": 512, - "top_k": 10, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "expert_parallel_size": 1, - "routed_tensor_parallel_size": 4, - "shared_tensor_parallel_size": 4, - "n_shared_experts": 1 - }, - "name": "qwen3.5" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 8, - "hidden": 4096, - "intermediate": 1024, - "num_experts": 512, - "top_k": 10, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "expert_parallel_size": 1, - "routed_tensor_parallel_size": 4, - "shared_tensor_parallel_size": 4, - "n_shared_experts": 1 - }, - "name": "qwen3.5" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 16384, - "hidden": 4096, - "intermediate": 1024, - "num_experts": 512, - "top_k": 10, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "expert_parallel_size": 1, - "routed_tensor_parallel_size": 4, - "shared_tensor_parallel_size": 4, - "n_shared_experts": 1 - }, - "name": "qwen3.5" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 16, - "hidden": 4096, - "intermediate": 1024, - "num_experts": 512, - "top_k": 10, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "expert_parallel_size": 1, - "routed_tensor_parallel_size": 4, - "shared_tensor_parallel_size": 4, - "n_shared_experts": 1 - }, - "name": "qwen3.5" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 32768, - "hidden": 4096, - "intermediate": 1024, - "num_experts": 512, - "top_k": 10, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "expert_parallel_size": 1, - "routed_tensor_parallel_size": 4, - "shared_tensor_parallel_size": 4, - "n_shared_experts": 1 - }, - "name": "qwen3.5" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 32, - "hidden": 4096, - "intermediate": 1024, - "num_experts": 512, - "top_k": 10, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "expert_parallel_size": 1, - "routed_tensor_parallel_size": 4, - "shared_tensor_parallel_size": 4, - "n_shared_experts": 1 - }, - "name": "qwen3.5" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 64, - "hidden": 4096, - "intermediate": 1024, - "num_experts": 512, - "top_k": 10, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "expert_parallel_size": 1, - "routed_tensor_parallel_size": 4, - "shared_tensor_parallel_size": 4, - "n_shared_experts": 1 - }, - "name": "qwen3.5" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 128, - "hidden": 4096, - "intermediate": 1024, - "num_experts": 512, - "top_k": 10, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "expert_parallel_size": 1, - "routed_tensor_parallel_size": 4, - "shared_tensor_parallel_size": 4, - "n_shared_experts": 1 - }, - "name": "qwen3.5" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 256, - "hidden": 4096, - "intermediate": 1024, - "num_experts": 512, - "top_k": 10, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "expert_parallel_size": 1, - "routed_tensor_parallel_size": 4, - "shared_tensor_parallel_size": 4, - "n_shared_experts": 1 - }, - "name": "qwen3.5" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 32768, - "hidden": 4096, - "intermediate": 1024, - "num_experts": 512, - "top_k": 10, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "expert_parallel_size": 1, - "routed_tensor_parallel_size": 8, - "shared_tensor_parallel_size": 8, - "n_shared_experts": 1 - }, - "name": "qwen3.5" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 4096, - "hidden": 4096, - "intermediate": 1024, - "num_experts": 512, - "top_k": 10, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "expert_parallel_size": 1, - "routed_tensor_parallel_size": 4, - "shared_tensor_parallel_size": 4, - "n_shared_experts": 1 - }, - "name": "qwen3.5" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 4, - "hidden": 4096, - "intermediate": 1024, - "num_experts": 512, - "top_k": 10, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "expert_parallel_size": 1, - "routed_tensor_parallel_size": 4, - "shared_tensor_parallel_size": 4, - "n_shared_experts": 1 - }, - "name": "qwen3.5" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 4096, - "hidden": 4096, - "intermediate": 1024, - "num_experts": 512, - "top_k": 10, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "expert_parallel_size": 1, - "routed_tensor_parallel_size": 2, - "shared_tensor_parallel_size": 2, - "n_shared_experts": 1 - }, - "name": "qwen3.5" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 4, - "hidden": 4096, - "intermediate": 1024, - "num_experts": 512, - "top_k": 10, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "expert_parallel_size": 1, - "routed_tensor_parallel_size": 2, - "shared_tensor_parallel_size": 2, - "n_shared_experts": 1 - }, - "name": "qwen3.5" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 8192, - "hidden": 4096, - "intermediate": 1024, - "num_experts": 512, - "top_k": 10, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "expert_parallel_size": 1, - "routed_tensor_parallel_size": 2, - "shared_tensor_parallel_size": 2, - "n_shared_experts": 1 - }, - "name": "qwen3.5" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 8, - "hidden": 4096, - "intermediate": 1024, - "num_experts": 512, - "top_k": 10, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "expert_parallel_size": 1, - "routed_tensor_parallel_size": 2, - "shared_tensor_parallel_size": 2, - "n_shared_experts": 1 - }, - "name": "qwen3.5" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 16384, - "hidden": 4096, - "intermediate": 1024, - "num_experts": 512, - "top_k": 10, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "expert_parallel_size": 1, - "routed_tensor_parallel_size": 2, - "shared_tensor_parallel_size": 2, - "n_shared_experts": 1 - }, - "name": "qwen3.5" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 16, - "hidden": 4096, - "intermediate": 1024, - "num_experts": 512, - "top_k": 10, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "expert_parallel_size": 1, - "routed_tensor_parallel_size": 2, - "shared_tensor_parallel_size": 2, - "n_shared_experts": 1 - }, - "name": "qwen3.5" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 32768, - "hidden": 4096, - "intermediate": 1024, - "num_experts": 512, - "top_k": 10, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "expert_parallel_size": 1, - "routed_tensor_parallel_size": 2, - "shared_tensor_parallel_size": 2, - "n_shared_experts": 1 - }, - "name": "qwen3.5" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 32, - "hidden": 4096, - "intermediate": 1024, - "num_experts": 512, - "top_k": 10, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "expert_parallel_size": 1, - "routed_tensor_parallel_size": 2, - "shared_tensor_parallel_size": 2, - "n_shared_experts": 1 - }, - "name": "qwen3.5" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 64, - "hidden": 4096, - "intermediate": 1024, - "num_experts": 512, - "top_k": 10, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "expert_parallel_size": 1, - "routed_tensor_parallel_size": 2, - "shared_tensor_parallel_size": 2, - "n_shared_experts": 1 - }, - "name": "qwen3.5" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 128, - "hidden": 4096, - "intermediate": 1024, - "num_experts": 512, - "top_k": 10, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "expert_parallel_size": 1, - "routed_tensor_parallel_size": 2, - "shared_tensor_parallel_size": 2, - "n_shared_experts": 1 - }, - "name": "qwen3.5" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 32768, - "hidden": 4096, - "intermediate": 1024, - "num_experts": 512, - "top_k": 10, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "expert_parallel_size": 1, - "routed_tensor_parallel_size": 4, - "shared_tensor_parallel_size": 4, - "n_shared_experts": 1 - }, - "name": "qwen3.5" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 8, - "hidden": 4096, - "intermediate": 1024, - "num_experts": 512, - "top_k": 10, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "expert_parallel_size": 1, - "routed_tensor_parallel_size": 4, - "shared_tensor_parallel_size": 4, - "n_shared_experts": 1 - }, - "name": "qwen3.5" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 4096, - "hidden": 6144, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "expert_parallel_size": 1, - "routed_tensor_parallel_size": 8, - "shared_tensor_parallel_size": 8, - "n_shared_experts": 1 - }, - "name": "glm5" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 4, - "hidden": 6144, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "expert_parallel_size": 1, - "routed_tensor_parallel_size": 8, - "shared_tensor_parallel_size": 8, - "n_shared_experts": 1 - }, - "name": "glm5" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 8192, - "hidden": 6144, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "expert_parallel_size": 1, - "routed_tensor_parallel_size": 8, - "shared_tensor_parallel_size": 8, - "n_shared_experts": 1 - }, - "name": "glm5" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 8, - "hidden": 6144, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "expert_parallel_size": 1, - "routed_tensor_parallel_size": 8, - "shared_tensor_parallel_size": 8, - "n_shared_experts": 1 - }, - "name": "glm5" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 16384, - "hidden": 6144, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "expert_parallel_size": 1, - "routed_tensor_parallel_size": 8, - "shared_tensor_parallel_size": 8, - "n_shared_experts": 1 - }, - "name": "glm5" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 16, - "hidden": 6144, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "expert_parallel_size": 1, - "routed_tensor_parallel_size": 8, - "shared_tensor_parallel_size": 8, - "n_shared_experts": 1 - }, - "name": "glm5" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 32768, - "hidden": 6144, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "expert_parallel_size": 1, - "routed_tensor_parallel_size": 8, - "shared_tensor_parallel_size": 8, - "n_shared_experts": 1 - }, - "name": "glm5" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 32, - "hidden": 6144, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "expert_parallel_size": 1, - "routed_tensor_parallel_size": 8, - "shared_tensor_parallel_size": 8, - "n_shared_experts": 1 - }, - "name": "glm5" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 64, - "hidden": 6144, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "expert_parallel_size": 1, - "routed_tensor_parallel_size": 8, - "shared_tensor_parallel_size": 8, - "n_shared_experts": 1 - }, - "name": "glm5" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 128, - "hidden": 6144, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "expert_parallel_size": 1, - "routed_tensor_parallel_size": 8, - "shared_tensor_parallel_size": 8, - "n_shared_experts": 1 - }, - "name": "glm5" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 256, - "hidden": 6144, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "expert_parallel_size": 1, - "routed_tensor_parallel_size": 8, - "shared_tensor_parallel_size": 8, - "n_shared_experts": 1 - }, - "name": "glm5" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 512, - "hidden": 6144, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "expert_parallel_size": 1, - "routed_tensor_parallel_size": 8, - "shared_tensor_parallel_size": 8, - "n_shared_experts": 1 - }, - "name": "glm5" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 4096, - "hidden": 6144, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "expert_parallel_size": 1, - "routed_tensor_parallel_size": 8, - "shared_tensor_parallel_size": 8, - "n_shared_experts": 1 - }, - "name": "glm5" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 4, - "hidden": 6144, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "expert_parallel_size": 1, - "routed_tensor_parallel_size": 8, - "shared_tensor_parallel_size": 8, - "n_shared_experts": 1 - }, - "name": "glm5" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 4096, - "hidden": 6144, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "expert_parallel_size": 1, - "routed_tensor_parallel_size": 4, - "shared_tensor_parallel_size": 4, - "n_shared_experts": 1 - }, - "name": "glm5" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 4, - "hidden": 6144, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "expert_parallel_size": 1, - "routed_tensor_parallel_size": 4, - "shared_tensor_parallel_size": 4, - "n_shared_experts": 1 - }, - "name": "glm5" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 8192, - "hidden": 6144, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "expert_parallel_size": 1, - "routed_tensor_parallel_size": 4, - "shared_tensor_parallel_size": 4, - "n_shared_experts": 1 - }, - "name": "glm5" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 8, - "hidden": 6144, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "expert_parallel_size": 1, - "routed_tensor_parallel_size": 4, - "shared_tensor_parallel_size": 4, - "n_shared_experts": 1 - }, - "name": "glm5" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 16384, - "hidden": 6144, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "expert_parallel_size": 1, - "routed_tensor_parallel_size": 4, - "shared_tensor_parallel_size": 4, - "n_shared_experts": 1 - }, - "name": "glm5" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 16, - "hidden": 6144, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "expert_parallel_size": 1, - "routed_tensor_parallel_size": 4, - "shared_tensor_parallel_size": 4, - "n_shared_experts": 1 - }, - "name": "glm5" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 32768, - "hidden": 6144, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "expert_parallel_size": 1, - "routed_tensor_parallel_size": 4, - "shared_tensor_parallel_size": 4, - "n_shared_experts": 1 - }, - "name": "glm5" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 32, - "hidden": 6144, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "expert_parallel_size": 1, - "routed_tensor_parallel_size": 4, - "shared_tensor_parallel_size": 4, - "n_shared_experts": 1 - }, - "name": "glm5" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 64, - "hidden": 6144, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "expert_parallel_size": 1, - "routed_tensor_parallel_size": 4, - "shared_tensor_parallel_size": 4, - "n_shared_experts": 1 - }, - "name": "glm5" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 128, - "hidden": 6144, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "expert_parallel_size": 1, - "routed_tensor_parallel_size": 4, - "shared_tensor_parallel_size": 4, - "n_shared_experts": 1 - }, - "name": "glm5" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 256, - "hidden": 6144, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "expert_parallel_size": 1, - "routed_tensor_parallel_size": 4, - "shared_tensor_parallel_size": 4, - "n_shared_experts": 1 - }, - "name": "glm5" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 32768, - "hidden": 6144, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "expert_parallel_size": 1, - "routed_tensor_parallel_size": 8, - "shared_tensor_parallel_size": 8, - "n_shared_experts": 1 - }, - "name": "glm5" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 8, - "hidden": 6144, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "expert_parallel_size": 1, - "routed_tensor_parallel_size": 8, - "shared_tensor_parallel_size": 8, - "n_shared_experts": 1 - }, - "name": "glm5" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 512, - "hidden": 6144, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "expert_parallel_size": 1, - "routed_tensor_parallel_size": 4, - "shared_tensor_parallel_size": 4, - "n_shared_experts": 1 - }, - "name": "glm5" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 8, - "hidden": 4096, - "intermediate": 1024, - "num_experts": 512, - "top_k": 10, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "expert_parallel_size": 1, - "routed_tensor_parallel_size": 8, - "shared_tensor_parallel_size": 8, - "n_shared_experts": 1 - }, - "name": "qwen3.5" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 512, - "hidden": 4096, - "intermediate": 1024, - "num_experts": 512, - "top_k": 10, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "expert_parallel_size": 1, - "routed_tensor_parallel_size": 4, - "shared_tensor_parallel_size": 4, - "n_shared_experts": 1 - }, - "name": "qwen3.5" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 8192, - "hidden": 4096, - "intermediate": 1024, - "num_experts": 512, - "top_k": 10, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "expert_parallel_size": 1, - "routed_tensor_parallel_size": 4, - "shared_tensor_parallel_size": 4, - "n_shared_experts": 1 - }, - "name": "qwen3.5" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 16384, - "hidden": 4096, - "intermediate": 1024, - "num_experts": 512, - "top_k": 10, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "expert_parallel_size": 1, - "routed_tensor_parallel_size": 4, - "shared_tensor_parallel_size": 4, - "n_shared_experts": 1 - }, - "name": "qwen3.5" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 16, - "hidden": 4096, - "intermediate": 1024, - "num_experts": 512, - "top_k": 10, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "expert_parallel_size": 1, - "routed_tensor_parallel_size": 4, - "shared_tensor_parallel_size": 4, - "n_shared_experts": 1 - }, - "name": "qwen3.5" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 32, - "hidden": 4096, - "intermediate": 1024, - "num_experts": 512, - "top_k": 10, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "expert_parallel_size": 1, - "routed_tensor_parallel_size": 4, - "shared_tensor_parallel_size": 4, - "n_shared_experts": 1 - }, - "name": "qwen3.5" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 64, - "hidden": 4096, - "intermediate": 1024, - "num_experts": 512, - "top_k": 10, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "expert_parallel_size": 1, - "routed_tensor_parallel_size": 4, - "shared_tensor_parallel_size": 4, - "n_shared_experts": 1 - }, - "name": "qwen3.5" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 128, - "hidden": 4096, - "intermediate": 1024, - "num_experts": 512, - "top_k": 10, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "expert_parallel_size": 1, - "routed_tensor_parallel_size": 4, - "shared_tensor_parallel_size": 4, - "n_shared_experts": 1 - }, - "name": "qwen3.5" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 4096, - "hidden": 4096, - "intermediate": 1024, - "num_experts": 512, - "top_k": 10, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "expert_parallel_size": 2, - "routed_tensor_parallel_size": 1, - "shared_tensor_parallel_size": 1, - "n_shared_experts": 1 - }, - "name": "qwen3.5" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 4, - "hidden": 4096, - "intermediate": 1024, - "num_experts": 512, - "top_k": 10, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "expert_parallel_size": 2, - "routed_tensor_parallel_size": 1, - "shared_tensor_parallel_size": 1, - "n_shared_experts": 1 - }, - "name": "qwen3.5" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 8192, - "hidden": 4096, - "intermediate": 1024, - "num_experts": 512, - "top_k": 10, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "expert_parallel_size": 2, - "routed_tensor_parallel_size": 1, - "shared_tensor_parallel_size": 1, - "n_shared_experts": 1 - }, - "name": "qwen3.5" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 8, - "hidden": 4096, - "intermediate": 1024, - "num_experts": 512, - "top_k": 10, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "expert_parallel_size": 2, - "routed_tensor_parallel_size": 1, - "shared_tensor_parallel_size": 1, - "n_shared_experts": 1 - }, - "name": "qwen3.5" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 16384, - "hidden": 4096, - "intermediate": 1024, - "num_experts": 512, - "top_k": 10, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "expert_parallel_size": 2, - "routed_tensor_parallel_size": 1, - "shared_tensor_parallel_size": 1, - "n_shared_experts": 1 - }, - "name": "qwen3.5" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 16, - "hidden": 4096, - "intermediate": 1024, - "num_experts": 512, - "top_k": 10, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "expert_parallel_size": 2, - "routed_tensor_parallel_size": 1, - "shared_tensor_parallel_size": 1, - "n_shared_experts": 1 - }, - "name": "qwen3.5" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 32768, - "hidden": 4096, - "intermediate": 1024, - "num_experts": 512, - "top_k": 10, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "expert_parallel_size": 2, - "routed_tensor_parallel_size": 1, - "shared_tensor_parallel_size": 1, - "n_shared_experts": 1 - }, - "name": "qwen3.5" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 32, - "hidden": 4096, - "intermediate": 1024, - "num_experts": 512, - "top_k": 10, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "expert_parallel_size": 2, - "routed_tensor_parallel_size": 1, - "shared_tensor_parallel_size": 1, - "n_shared_experts": 1 - }, - "name": "qwen3.5" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 64, - "hidden": 4096, - "intermediate": 1024, - "num_experts": 512, - "top_k": 10, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "expert_parallel_size": 2, - "routed_tensor_parallel_size": 1, - "shared_tensor_parallel_size": 1, - "n_shared_experts": 1 - }, - "name": "qwen3.5" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 128, - "hidden": 4096, - "intermediate": 1024, - "num_experts": 512, - "top_k": 10, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "expert_parallel_size": 2, - "routed_tensor_parallel_size": 1, - "shared_tensor_parallel_size": 1, - "n_shared_experts": 1 - }, - "name": "qwen3.5" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 256, - "hidden": 4096, - "intermediate": 1024, - "num_experts": 512, - "top_k": 10, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "expert_parallel_size": 1, - "routed_tensor_parallel_size": 4, - "shared_tensor_parallel_size": 4, - "n_shared_experts": 1 - }, - "name": "qwen3.5" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 256, - "hidden": 4096, - "intermediate": 1024, - "num_experts": 512, - "top_k": 10, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "expert_parallel_size": 2, - "routed_tensor_parallel_size": 1, - "shared_tensor_parallel_size": 1, - "n_shared_experts": 1 - }, - "name": "qwen3.5" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 4096, - "hidden": 4096, - "intermediate": 1024, - "num_experts": 512, - "top_k": 10, - "dtype_act": "bf16", - "dtype_weight": "bf16", - "expert_parallel_size": 1, - "routed_tensor_parallel_size": 4, - "shared_tensor_parallel_size": 4, - "n_shared_experts": 1 - }, - "name": "qwen3.5" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 4, - "hidden": 4096, - "intermediate": 1024, - "num_experts": 512, - "top_k": 10, - "dtype_act": "bf16", - "dtype_weight": "bf16", - "expert_parallel_size": 1, - "routed_tensor_parallel_size": 4, - "shared_tensor_parallel_size": 4, - "n_shared_experts": 1 - }, - "name": "qwen3.5" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 8192, - "hidden": 4096, - "intermediate": 1024, - "num_experts": 512, - "top_k": 10, - "dtype_act": "bf16", - "dtype_weight": "bf16", - "expert_parallel_size": 1, - "routed_tensor_parallel_size": 4, - "shared_tensor_parallel_size": 4, - "n_shared_experts": 1 - }, - "name": "qwen3.5" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 8, - "hidden": 4096, - "intermediate": 1024, - "num_experts": 512, - "top_k": 10, - "dtype_act": "bf16", - "dtype_weight": "bf16", - "expert_parallel_size": 1, - "routed_tensor_parallel_size": 4, - "shared_tensor_parallel_size": 4, - "n_shared_experts": 1 - }, - "name": "qwen3.5" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 16384, - "hidden": 4096, - "intermediate": 1024, - "num_experts": 512, - "top_k": 10, - "dtype_act": "bf16", - "dtype_weight": "bf16", - "expert_parallel_size": 1, - "routed_tensor_parallel_size": 4, - "shared_tensor_parallel_size": 4, - "n_shared_experts": 1 - }, - "name": "qwen3.5" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 16, - "hidden": 4096, - "intermediate": 1024, - "num_experts": 512, - "top_k": 10, - "dtype_act": "bf16", - "dtype_weight": "bf16", - "expert_parallel_size": 1, - "routed_tensor_parallel_size": 4, - "shared_tensor_parallel_size": 4, - "n_shared_experts": 1 - }, - "name": "qwen3.5" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 32768, - "hidden": 4096, - "intermediate": 1024, - "num_experts": 512, - "top_k": 10, - "dtype_act": "bf16", - "dtype_weight": "bf16", - "expert_parallel_size": 1, - "routed_tensor_parallel_size": 4, - "shared_tensor_parallel_size": 4, - "n_shared_experts": 1 - }, - "name": "qwen3.5" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 32, - "hidden": 4096, - "intermediate": 1024, - "num_experts": 512, - "top_k": 10, - "dtype_act": "bf16", - "dtype_weight": "bf16", - "expert_parallel_size": 1, - "routed_tensor_parallel_size": 4, - "shared_tensor_parallel_size": 4, - "n_shared_experts": 1 - }, - "name": "qwen3.5" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 64, - "hidden": 4096, - "intermediate": 1024, - "num_experts": 512, - "top_k": 10, - "dtype_act": "bf16", - "dtype_weight": "bf16", - "expert_parallel_size": 1, - "routed_tensor_parallel_size": 4, - "shared_tensor_parallel_size": 4, - "n_shared_experts": 1 - }, - "name": "qwen3.5" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 128, - "hidden": 4096, - "intermediate": 1024, - "num_experts": 512, - "top_k": 10, - "dtype_act": "bf16", - "dtype_weight": "bf16", - "expert_parallel_size": 1, - "routed_tensor_parallel_size": 4, - "shared_tensor_parallel_size": 4, - "n_shared_experts": 1 - }, - "name": "qwen3.5" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 4096, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 384, - "top_k": 8, - "dtype_act": "bf16", - "dtype_weight": "int4", - "expert_parallel_size": 1, - "routed_tensor_parallel_size": 8, - "shared_tensor_parallel_size": 8, - "n_shared_experts": 1 - }, - "name": "kimik2.5" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 4, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 384, - "top_k": 8, - "dtype_act": "bf16", - "dtype_weight": "int4", - "expert_parallel_size": 1, - "routed_tensor_parallel_size": 8, - "shared_tensor_parallel_size": 8, - "n_shared_experts": 1 - }, - "name": "kimik2.5" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 8192, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 384, - "top_k": 8, - "dtype_act": "bf16", - "dtype_weight": "int4", - "expert_parallel_size": 1, - "routed_tensor_parallel_size": 8, - "shared_tensor_parallel_size": 8, - "n_shared_experts": 1 - }, - "name": "kimik2.5" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 8, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 384, - "top_k": 8, - "dtype_act": "bf16", - "dtype_weight": "int4", - "expert_parallel_size": 1, - "routed_tensor_parallel_size": 8, - "shared_tensor_parallel_size": 8, - "n_shared_experts": 1 - }, - "name": "kimik2.5" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 16384, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 384, - "top_k": 8, - "dtype_act": "bf16", - "dtype_weight": "int4", - "expert_parallel_size": 1, - "routed_tensor_parallel_size": 8, - "shared_tensor_parallel_size": 8, - "n_shared_experts": 1 - }, - "name": "kimik2.5" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 16, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 384, - "top_k": 8, - "dtype_act": "bf16", - "dtype_weight": "int4", - "expert_parallel_size": 1, - "routed_tensor_parallel_size": 8, - "shared_tensor_parallel_size": 8, - "n_shared_experts": 1 - }, - "name": "kimik2.5" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 32768, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 384, - "top_k": 8, - "dtype_act": "bf16", - "dtype_weight": "int4", - "expert_parallel_size": 1, - "routed_tensor_parallel_size": 8, - "shared_tensor_parallel_size": 8, - "n_shared_experts": 1 - }, - "name": "kimik2.5" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 32, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 384, - "top_k": 8, - "dtype_act": "bf16", - "dtype_weight": "int4", - "expert_parallel_size": 1, - "routed_tensor_parallel_size": 8, - "shared_tensor_parallel_size": 8, - "n_shared_experts": 1 - }, - "name": "kimik2.5" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 64, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 384, - "top_k": 8, - "dtype_act": "bf16", - "dtype_weight": "int4", - "expert_parallel_size": 1, - "routed_tensor_parallel_size": 8, - "shared_tensor_parallel_size": 8, - "n_shared_experts": 1 - }, - "name": "kimik2.5" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 4096, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 384, - "top_k": 8, - "dtype_act": "bf16", - "dtype_weight": "int4", - "expert_parallel_size": 1, - "routed_tensor_parallel_size": 4, - "shared_tensor_parallel_size": 4, - "n_shared_experts": 1 - }, - "name": "kimik2.5" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 4, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 384, - "top_k": 8, - "dtype_act": "bf16", - "dtype_weight": "int4", - "expert_parallel_size": 1, - "routed_tensor_parallel_size": 4, - "shared_tensor_parallel_size": 4, - "n_shared_experts": 1 - }, - "name": "kimik2.5" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 8192, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 384, - "top_k": 8, - "dtype_act": "bf16", - "dtype_weight": "int4", - "expert_parallel_size": 1, - "routed_tensor_parallel_size": 4, - "shared_tensor_parallel_size": 4, - "n_shared_experts": 1 - }, - "name": "kimik2.5" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 8, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 384, - "top_k": 8, - "dtype_act": "bf16", - "dtype_weight": "int4", - "expert_parallel_size": 1, - "routed_tensor_parallel_size": 4, - "shared_tensor_parallel_size": 4, - "n_shared_experts": 1 - }, - "name": "kimik2.5" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 16384, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 384, - "top_k": 8, - "dtype_act": "bf16", - "dtype_weight": "int4", - "expert_parallel_size": 1, - "routed_tensor_parallel_size": 4, - "shared_tensor_parallel_size": 4, - "n_shared_experts": 1 - }, - "name": "kimik2.5" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 16, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 384, - "top_k": 8, - "dtype_act": "bf16", - "dtype_weight": "int4", - "expert_parallel_size": 1, - "routed_tensor_parallel_size": 4, - "shared_tensor_parallel_size": 4, - "n_shared_experts": 1 - }, - "name": "kimik2.5" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 32768, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 384, - "top_k": 8, - "dtype_act": "bf16", - "dtype_weight": "int4", - "expert_parallel_size": 1, - "routed_tensor_parallel_size": 4, - "shared_tensor_parallel_size": 4, - "n_shared_experts": 1 - }, - "name": "kimik2.5" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 32, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 384, - "top_k": 8, - "dtype_act": "bf16", - "dtype_weight": "int4", - "expert_parallel_size": 1, - "routed_tensor_parallel_size": 4, - "shared_tensor_parallel_size": 4, - "n_shared_experts": 1 - }, - "name": "kimik2.5" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 64, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 384, - "top_k": 8, - "dtype_act": "bf16", - "dtype_weight": "int4", - "expert_parallel_size": 1, - "routed_tensor_parallel_size": 4, - "shared_tensor_parallel_size": 4, - "n_shared_experts": 1 - }, - "name": "kimik2.5" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 4096, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 384, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "expert_parallel_size": 1, - "routed_tensor_parallel_size": 8, - "shared_tensor_parallel_size": 8, - "n_shared_experts": 1 - }, - "name": "kimik2.5" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 4, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 384, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "expert_parallel_size": 1, - "routed_tensor_parallel_size": 8, - "shared_tensor_parallel_size": 8, - "n_shared_experts": 1 - }, - "name": "kimik2.5" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 4096, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 384, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "expert_parallel_size": 1, - "routed_tensor_parallel_size": 4, - "shared_tensor_parallel_size": 4, - "n_shared_experts": 1 - }, - "name": "kimik2.5" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 4, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 384, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "expert_parallel_size": 1, - "routed_tensor_parallel_size": 4, - "shared_tensor_parallel_size": 4, - "n_shared_experts": 1 - }, - "name": "kimik2.5" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 8192, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 384, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "expert_parallel_size": 1, - "routed_tensor_parallel_size": 4, - "shared_tensor_parallel_size": 4, - "n_shared_experts": 1 - }, - "name": "kimik2.5" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 8, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 384, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "expert_parallel_size": 1, - "routed_tensor_parallel_size": 4, - "shared_tensor_parallel_size": 4, - "n_shared_experts": 1 - }, - "name": "kimik2.5" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 16384, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 384, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "expert_parallel_size": 1, - "routed_tensor_parallel_size": 4, - "shared_tensor_parallel_size": 4, - "n_shared_experts": 1 - }, - "name": "kimik2.5" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 16, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 384, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "expert_parallel_size": 1, - "routed_tensor_parallel_size": 4, - "shared_tensor_parallel_size": 4, - "n_shared_experts": 1 - }, - "name": "kimik2.5" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 32768, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 384, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "expert_parallel_size": 1, - "routed_tensor_parallel_size": 4, - "shared_tensor_parallel_size": 4, - "n_shared_experts": 1 - }, - "name": "kimik2.5" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 32, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 384, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "expert_parallel_size": 1, - "routed_tensor_parallel_size": 4, - "shared_tensor_parallel_size": 4, - "n_shared_experts": 1 - }, - "name": "kimik2.5" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 64, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 384, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "expert_parallel_size": 1, - "routed_tensor_parallel_size": 4, - "shared_tensor_parallel_size": 4, - "n_shared_experts": 1 - }, - "name": "kimik2.5" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 32768, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 384, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "expert_parallel_size": 1, - "routed_tensor_parallel_size": 8, - "shared_tensor_parallel_size": 8, - "n_shared_experts": 1 - }, - "name": "kimik2.5" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 128, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "expert_parallel_size": 1, - "routed_tensor_parallel_size": 8, - "shared_tensor_parallel_size": 8, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 256, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "expert_parallel_size": 1, - "routed_tensor_parallel_size": 8, - "shared_tensor_parallel_size": 8, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 8192, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "expert_parallel_size": 1, - "routed_tensor_parallel_size": 4, - "shared_tensor_parallel_size": 4, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 16384, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "expert_parallel_size": 1, - "routed_tensor_parallel_size": 4, - "shared_tensor_parallel_size": 4, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 512, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "expert_parallel_size": 8, - "routed_tensor_parallel_size": 1, - "shared_tensor_parallel_size": 1, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 1024, - "hidden": 7168, - "intermediate": 3072, - "num_experts": 384, - "top_k": 6, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "expert_parallel_size": 1, - "routed_tensor_parallel_size": 8, - "shared_tensor_parallel_size": 1, - "n_shared_experts": 1 - }, - "name": "dsv4" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 1, - "hidden": 7168, - "intermediate": 3072, - "num_experts": 384, - "top_k": 6, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "expert_parallel_size": 1, - "routed_tensor_parallel_size": 8, - "shared_tensor_parallel_size": 1, - "n_shared_experts": 1 - }, - "name": "dsv4" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 2048, - "hidden": 7168, - "intermediate": 3072, - "num_experts": 384, - "top_k": 6, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "expert_parallel_size": 1, - "routed_tensor_parallel_size": 8, - "shared_tensor_parallel_size": 1, - "n_shared_experts": 1 - }, - "name": "dsv4" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 2, - "hidden": 7168, - "intermediate": 3072, - "num_experts": 384, - "top_k": 6, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "expert_parallel_size": 1, - "routed_tensor_parallel_size": 8, - "shared_tensor_parallel_size": 1, - "n_shared_experts": 1 - }, - "name": "dsv4" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 4096, - "hidden": 7168, - "intermediate": 3072, - "num_experts": 384, - "top_k": 6, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "expert_parallel_size": 1, - "routed_tensor_parallel_size": 8, - "shared_tensor_parallel_size": 1, - "n_shared_experts": 1 - }, - "name": "dsv4" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 4, - "hidden": 7168, - "intermediate": 3072, - "num_experts": 384, - "top_k": 6, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "expert_parallel_size": 1, - "routed_tensor_parallel_size": 8, - "shared_tensor_parallel_size": 1, - "n_shared_experts": 1 - }, - "name": "dsv4" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 8192, - "hidden": 7168, - "intermediate": 3072, - "num_experts": 384, - "top_k": 6, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "expert_parallel_size": 1, - "routed_tensor_parallel_size": 8, - "shared_tensor_parallel_size": 1, - "n_shared_experts": 1 - }, - "name": "dsv4" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 8, - "hidden": 7168, - "intermediate": 3072, - "num_experts": 384, - "top_k": 6, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "expert_parallel_size": 1, - "routed_tensor_parallel_size": 8, - "shared_tensor_parallel_size": 1, - "n_shared_experts": 1 - }, - "name": "dsv4" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 16384, - "hidden": 7168, - "intermediate": 3072, - "num_experts": 384, - "top_k": 6, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "expert_parallel_size": 1, - "routed_tensor_parallel_size": 8, - "shared_tensor_parallel_size": 1, - "n_shared_experts": 1 - }, - "name": "dsv4" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 16, - "hidden": 7168, - "intermediate": 3072, - "num_experts": 384, - "top_k": 6, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "expert_parallel_size": 1, - "routed_tensor_parallel_size": 8, - "shared_tensor_parallel_size": 1, - "n_shared_experts": 1 - }, - "name": "dsv4" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 32768, - "hidden": 7168, - "intermediate": 3072, - "num_experts": 384, - "top_k": 6, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "expert_parallel_size": 1, - "routed_tensor_parallel_size": 8, - "shared_tensor_parallel_size": 1, - "n_shared_experts": 1 - }, - "name": "dsv4" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 32, - "hidden": 7168, - "intermediate": 3072, - "num_experts": 384, - "top_k": 6, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "expert_parallel_size": 1, - "routed_tensor_parallel_size": 8, - "shared_tensor_parallel_size": 1, - "n_shared_experts": 1 - }, - "name": "dsv4" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 64, - "hidden": 7168, - "intermediate": 3072, - "num_experts": 384, - "top_k": 6, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "expert_parallel_size": 1, - "routed_tensor_parallel_size": 8, - "shared_tensor_parallel_size": 1, - "n_shared_experts": 1 - }, - "name": "dsv4" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 128, - "hidden": 7168, - "intermediate": 3072, - "num_experts": 384, - "top_k": 6, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "expert_parallel_size": 1, - "routed_tensor_parallel_size": 8, - "shared_tensor_parallel_size": 1, - "n_shared_experts": 1 - }, - "name": "dsv4" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 256, - "hidden": 7168, - "intermediate": 3072, - "num_experts": 384, - "top_k": 6, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "expert_parallel_size": 1, - "routed_tensor_parallel_size": 8, - "shared_tensor_parallel_size": 1, - "n_shared_experts": 1 - }, - "name": "dsv4" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 1024, - "hidden": 7168, - "intermediate": 3072, - "num_experts": 384, - "top_k": 6, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "expert_parallel_size": 8, - "routed_tensor_parallel_size": 1, - "shared_tensor_parallel_size": 1, - "n_shared_experts": 1 - }, - "name": "dsv4" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 1, - "hidden": 7168, - "intermediate": 3072, - "num_experts": 384, - "top_k": 6, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "expert_parallel_size": 8, - "routed_tensor_parallel_size": 1, - "shared_tensor_parallel_size": 1, - "n_shared_experts": 1 - }, - "name": "dsv4" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 2048, - "hidden": 7168, - "intermediate": 3072, - "num_experts": 384, - "top_k": 6, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "expert_parallel_size": 8, - "routed_tensor_parallel_size": 1, - "shared_tensor_parallel_size": 1, - "n_shared_experts": 1 - }, - "name": "dsv4" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 2, - "hidden": 7168, - "intermediate": 3072, - "num_experts": 384, - "top_k": 6, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "expert_parallel_size": 8, - "routed_tensor_parallel_size": 1, - "shared_tensor_parallel_size": 1, - "n_shared_experts": 1 - }, - "name": "dsv4" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 4096, - "hidden": 7168, - "intermediate": 3072, - "num_experts": 384, - "top_k": 6, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "expert_parallel_size": 8, - "routed_tensor_parallel_size": 1, - "shared_tensor_parallel_size": 1, - "n_shared_experts": 1 - }, - "name": "dsv4" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 4, - "hidden": 7168, - "intermediate": 3072, - "num_experts": 384, - "top_k": 6, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "expert_parallel_size": 8, - "routed_tensor_parallel_size": 1, - "shared_tensor_parallel_size": 1, - "n_shared_experts": 1 - }, - "name": "dsv4" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 8192, - "hidden": 7168, - "intermediate": 3072, - "num_experts": 384, - "top_k": 6, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "expert_parallel_size": 8, - "routed_tensor_parallel_size": 1, - "shared_tensor_parallel_size": 1, - "n_shared_experts": 1 - }, - "name": "dsv4" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 8, - "hidden": 7168, - "intermediate": 3072, - "num_experts": 384, - "top_k": 6, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "expert_parallel_size": 8, - "routed_tensor_parallel_size": 1, - "shared_tensor_parallel_size": 1, - "n_shared_experts": 1 - }, - "name": "dsv4" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 16384, - "hidden": 7168, - "intermediate": 3072, - "num_experts": 384, - "top_k": 6, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "expert_parallel_size": 8, - "routed_tensor_parallel_size": 1, - "shared_tensor_parallel_size": 1, - "n_shared_experts": 1 - }, - "name": "dsv4" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 16, - "hidden": 7168, - "intermediate": 3072, - "num_experts": 384, - "top_k": 6, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "expert_parallel_size": 8, - "routed_tensor_parallel_size": 1, - "shared_tensor_parallel_size": 1, - "n_shared_experts": 1 - }, - "name": "dsv4" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 32768, - "hidden": 7168, - "intermediate": 3072, - "num_experts": 384, - "top_k": 6, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "expert_parallel_size": 8, - "routed_tensor_parallel_size": 1, - "shared_tensor_parallel_size": 1, - "n_shared_experts": 1 - }, - "name": "dsv4" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 32, - "hidden": 7168, - "intermediate": 3072, - "num_experts": 384, - "top_k": 6, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "expert_parallel_size": 8, - "routed_tensor_parallel_size": 1, - "shared_tensor_parallel_size": 1, - "n_shared_experts": 1 - }, - "name": "dsv4" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 64, - "hidden": 7168, - "intermediate": 3072, - "num_experts": 384, - "top_k": 6, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "expert_parallel_size": 8, - "routed_tensor_parallel_size": 1, - "shared_tensor_parallel_size": 1, - "n_shared_experts": 1 - }, - "name": "dsv4" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 128, - "hidden": 7168, - "intermediate": 3072, - "num_experts": 384, - "top_k": 6, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "expert_parallel_size": 8, - "routed_tensor_parallel_size": 1, - "shared_tensor_parallel_size": 1, - "n_shared_experts": 1 - }, - "name": "dsv4" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 256, - "hidden": 7168, - "intermediate": 3072, - "num_experts": 384, - "top_k": 6, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "expert_parallel_size": 8, - "routed_tensor_parallel_size": 1, - "shared_tensor_parallel_size": 1, - "n_shared_experts": 1 - }, - "name": "dsv4" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 512, - "hidden": 7168, - "intermediate": 3072, - "num_experts": 384, - "top_k": 6, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "expert_parallel_size": 1, - "routed_tensor_parallel_size": 8, - "shared_tensor_parallel_size": 1, - "n_shared_experts": 1 - }, - "name": "dsv4" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 512, - "hidden": 7168, - "intermediate": 3072, - "num_experts": 384, - "top_k": 6, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "expert_parallel_size": 8, - "routed_tensor_parallel_size": 1, - "shared_tensor_parallel_size": 1, - "n_shared_experts": 1 - }, - "name": "dsv4" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 1024, - "hidden": 7168, - "intermediate": 3072, - "num_experts": 384, - "top_k": 6, - "dtype_act": "nvfp4", - "dtype_weight": "mxfp4", - "expert_parallel_size": 1, - "routed_tensor_parallel_size": 4, - "shared_tensor_parallel_size": 1, - "n_shared_experts": 1 - }, - "name": "dsv4" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 1, - "hidden": 7168, - "intermediate": 3072, - "num_experts": 384, - "top_k": 6, - "dtype_act": "nvfp4", - "dtype_weight": "mxfp4", - "expert_parallel_size": 1, - "routed_tensor_parallel_size": 4, - "shared_tensor_parallel_size": 1, - "n_shared_experts": 1 - }, - "name": "dsv4" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 2048, - "hidden": 7168, - "intermediate": 3072, - "num_experts": 384, - "top_k": 6, - "dtype_act": "nvfp4", - "dtype_weight": "mxfp4", - "expert_parallel_size": 1, - "routed_tensor_parallel_size": 4, - "shared_tensor_parallel_size": 1, - "n_shared_experts": 1 - }, - "name": "dsv4" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 2, - "hidden": 7168, - "intermediate": 3072, - "num_experts": 384, - "top_k": 6, - "dtype_act": "nvfp4", - "dtype_weight": "mxfp4", - "expert_parallel_size": 1, - "routed_tensor_parallel_size": 4, - "shared_tensor_parallel_size": 1, - "n_shared_experts": 1 - }, - "name": "dsv4" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 4, - "hidden": 7168, - "intermediate": 3072, - "num_experts": 384, - "top_k": 6, - "dtype_act": "nvfp4", - "dtype_weight": "mxfp4", - "expert_parallel_size": 1, - "routed_tensor_parallel_size": 4, - "shared_tensor_parallel_size": 1, - "n_shared_experts": 1 - }, - "name": "dsv4" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 128, - "hidden": 7168, - "intermediate": 3072, - "num_experts": 384, - "top_k": 6, - "dtype_act": "nvfp4", - "dtype_weight": "mxfp4", - "expert_parallel_size": 1, - "routed_tensor_parallel_size": 4, - "shared_tensor_parallel_size": 1, - "n_shared_experts": 1 - }, - "name": "dsv4" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 256, - "hidden": 7168, - "intermediate": 3072, - "num_experts": 384, - "top_k": 6, - "dtype_act": "nvfp4", - "dtype_weight": "mxfp4", - "expert_parallel_size": 4, - "routed_tensor_parallel_size": 1, - "shared_tensor_parallel_size": 1, - "n_shared_experts": 1 - }, - "name": "dsv4" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 2048, - "hidden": 7168, - "intermediate": 3072, - "num_experts": 384, - "top_k": 6, - "dtype_act": "nvfp4", - "dtype_weight": "mxfp4", - "expert_parallel_size": 4, - "routed_tensor_parallel_size": 1, - "shared_tensor_parallel_size": 1, - "n_shared_experts": 1 - }, - "name": "dsv4" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 64, - "hidden": 7168, - "intermediate": 3072, - "num_experts": 384, - "top_k": 6, - "dtype_act": "nvfp4", - "dtype_weight": "mxfp4", - "expert_parallel_size": 4, - "routed_tensor_parallel_size": 1, - "shared_tensor_parallel_size": 1, - "n_shared_experts": 1 - }, - "name": "dsv4" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 128, - "hidden": 7168, - "intermediate": 3072, - "num_experts": 384, - "top_k": 6, - "dtype_act": "nvfp4", - "dtype_weight": "mxfp4", - "expert_parallel_size": 4, - "routed_tensor_parallel_size": 1, - "shared_tensor_parallel_size": 1, - "n_shared_experts": 1 - }, - "name": "dsv4" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 256, - "hidden": 7168, - "intermediate": 3072, - "num_experts": 384, - "top_k": 6, - "dtype_act": "nvfp4", - "dtype_weight": "mxfp4", - "expert_parallel_size": 1, - "routed_tensor_parallel_size": 4, - "shared_tensor_parallel_size": 1, - "n_shared_experts": 1 - }, - "name": "dsv4" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 512, - "hidden": 7168, - "intermediate": 3072, - "num_experts": 384, - "top_k": 6, - "dtype_act": "nvfp4", - "dtype_weight": "mxfp4", - "expert_parallel_size": 1, - "routed_tensor_parallel_size": 4, - "shared_tensor_parallel_size": 1, - "n_shared_experts": 1 - }, - "name": "dsv4" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 4096, - "hidden": 4096, - "intermediate": 1024, - "num_experts": 512, - "top_k": 10, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "expert_parallel_size": 8, - "routed_tensor_parallel_size": 1, - "shared_tensor_parallel_size": 1, - "n_shared_experts": 1 - }, - "name": "qwen3.5" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 4, - "hidden": 4096, - "intermediate": 1024, - "num_experts": 512, - "top_k": 10, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "expert_parallel_size": 8, - "routed_tensor_parallel_size": 1, - "shared_tensor_parallel_size": 1, - "n_shared_experts": 1 - }, - "name": "qwen3.5" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 8192, - "hidden": 4096, - "intermediate": 1024, - "num_experts": 512, - "top_k": 10, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "expert_parallel_size": 8, - "routed_tensor_parallel_size": 1, - "shared_tensor_parallel_size": 1, - "n_shared_experts": 1 - }, - "name": "qwen3.5" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 8, - "hidden": 4096, - "intermediate": 1024, - "num_experts": 512, - "top_k": 10, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "expert_parallel_size": 8, - "routed_tensor_parallel_size": 1, - "shared_tensor_parallel_size": 1, - "n_shared_experts": 1 - }, - "name": "qwen3.5" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 16384, - "hidden": 4096, - "intermediate": 1024, - "num_experts": 512, - "top_k": 10, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "expert_parallel_size": 8, - "routed_tensor_parallel_size": 1, - "shared_tensor_parallel_size": 1, - "n_shared_experts": 1 - }, - "name": "qwen3.5" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 16, - "hidden": 4096, - "intermediate": 1024, - "num_experts": 512, - "top_k": 10, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "expert_parallel_size": 8, - "routed_tensor_parallel_size": 1, - "shared_tensor_parallel_size": 1, - "n_shared_experts": 1 - }, - "name": "qwen3.5" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 32768, - "hidden": 4096, - "intermediate": 1024, - "num_experts": 512, - "top_k": 10, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "expert_parallel_size": 8, - "routed_tensor_parallel_size": 1, - "shared_tensor_parallel_size": 1, - "n_shared_experts": 1 - }, - "name": "qwen3.5" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 32, - "hidden": 4096, - "intermediate": 1024, - "num_experts": 512, - "top_k": 10, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "expert_parallel_size": 8, - "routed_tensor_parallel_size": 1, - "shared_tensor_parallel_size": 1, - "n_shared_experts": 1 - }, - "name": "qwen3.5" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 64, - "hidden": 4096, - "intermediate": 1024, - "num_experts": 512, - "top_k": 10, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "expert_parallel_size": 8, - "routed_tensor_parallel_size": 1, - "shared_tensor_parallel_size": 1, - "n_shared_experts": 1 - }, - "name": "qwen3.5" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 128, - "hidden": 4096, - "intermediate": 1024, - "num_experts": 512, - "top_k": 10, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "expert_parallel_size": 8, - "routed_tensor_parallel_size": 1, - "shared_tensor_parallel_size": 1, - "n_shared_experts": 1 - }, - "name": "qwen3.5" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 256, - "hidden": 4096, - "intermediate": 1024, - "num_experts": 512, - "top_k": 10, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "expert_parallel_size": 8, - "routed_tensor_parallel_size": 1, - "shared_tensor_parallel_size": 1, - "n_shared_experts": 1 - }, - "name": "qwen3.5" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 4, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "expert_parallel_size": 8, - "routed_tensor_parallel_size": 1, - "shared_tensor_parallel_size": 1, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 8, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "expert_parallel_size": 8, - "routed_tensor_parallel_size": 1, - "shared_tensor_parallel_size": 1, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 16384, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "expert_parallel_size": 8, - "routed_tensor_parallel_size": 1, - "shared_tensor_parallel_size": 1, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 16, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "expert_parallel_size": 8, - "routed_tensor_parallel_size": 1, - "shared_tensor_parallel_size": 1, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 32, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "expert_parallel_size": 8, - "routed_tensor_parallel_size": 1, - "shared_tensor_parallel_size": 1, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 64, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "expert_parallel_size": 8, - "routed_tensor_parallel_size": 1, - "shared_tensor_parallel_size": 1, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 1024, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "expert_parallel_size": 8, - "routed_tensor_parallel_size": 1, - "shared_tensor_parallel_size": 1, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 1, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "expert_parallel_size": 8, - "routed_tensor_parallel_size": 1, - "shared_tensor_parallel_size": 1, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 1, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "expert_parallel_size": 8, - "routed_tensor_parallel_size": 1, - "shared_tensor_parallel_size": 1, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 3, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "expert_parallel_size": 8, - "routed_tensor_parallel_size": 1, - "shared_tensor_parallel_size": 1, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 5, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "expert_parallel_size": 8, - "routed_tensor_parallel_size": 1, - "shared_tensor_parallel_size": 1, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 7, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "expert_parallel_size": 8, - "routed_tensor_parallel_size": 1, - "shared_tensor_parallel_size": 1, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 21, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "expert_parallel_size": 8, - "routed_tensor_parallel_size": 1, - "shared_tensor_parallel_size": 1, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 6144, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "expert_parallel_size": 16, - "routed_tensor_parallel_size": 1, - "shared_tensor_parallel_size": 1, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 4, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "expert_parallel_size": 16, - "routed_tensor_parallel_size": 1, - "shared_tensor_parallel_size": 1, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 9216, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "expert_parallel_size": 16, - "routed_tensor_parallel_size": 1, - "shared_tensor_parallel_size": 1, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 6, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "expert_parallel_size": 16, - "routed_tensor_parallel_size": 1, - "shared_tensor_parallel_size": 1, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 30720, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "expert_parallel_size": 16, - "routed_tensor_parallel_size": 1, - "shared_tensor_parallel_size": 1, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 20, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "expert_parallel_size": 16, - "routed_tensor_parallel_size": 1, - "shared_tensor_parallel_size": 1, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 32768, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "expert_parallel_size": 16, - "routed_tensor_parallel_size": 1, - "shared_tensor_parallel_size": 1, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 40, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "expert_parallel_size": 16, - "routed_tensor_parallel_size": 1, - "shared_tensor_parallel_size": 1, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 78, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "expert_parallel_size": 16, - "routed_tensor_parallel_size": 1, - "shared_tensor_parallel_size": 1, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 308, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "expert_parallel_size": 16, - "routed_tensor_parallel_size": 1, - "shared_tensor_parallel_size": 1, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 2, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "expert_parallel_size": 16, - "routed_tensor_parallel_size": 1, - "shared_tensor_parallel_size": 1, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 3, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "expert_parallel_size": 16, - "routed_tensor_parallel_size": 1, - "shared_tensor_parallel_size": 1, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 10, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "expert_parallel_size": 16, - "routed_tensor_parallel_size": 1, - "shared_tensor_parallel_size": 1, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 77, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "expert_parallel_size": 16, - "routed_tensor_parallel_size": 1, - "shared_tensor_parallel_size": 1, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 32768, - "hidden": 2880, - "intermediate": 2880, - "num_experts": 128, - "top_k": 4, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "expert_parallel_size": 1, - "routed_tensor_parallel_size": 1, - "shared_tensor_parallel_size": 1, - "n_shared_experts": 0 - }, - "name": "gptoss" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 256, - "hidden": 2880, - "intermediate": 2880, - "num_experts": 128, - "top_k": 4, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "expert_parallel_size": 1, - "routed_tensor_parallel_size": 1, - "shared_tensor_parallel_size": 1, - "n_shared_experts": 0 - }, - "name": "gptoss" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 32768, - "hidden": 2880, - "intermediate": 2880, - "num_experts": 128, - "top_k": 4, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "expert_parallel_size": 2, - "routed_tensor_parallel_size": 1, - "shared_tensor_parallel_size": 1, - "n_shared_experts": 0 - }, - "name": "gptoss" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 256, - "hidden": 2880, - "intermediate": 2880, - "num_experts": 128, - "top_k": 4, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "expert_parallel_size": 2, - "routed_tensor_parallel_size": 1, - "shared_tensor_parallel_size": 1, - "n_shared_experts": 0 - }, - "name": "gptoss" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 4096, - "hidden": 2880, - "intermediate": 2880, - "num_experts": 128, - "top_k": 4, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "expert_parallel_size": 1, - "routed_tensor_parallel_size": 2, - "shared_tensor_parallel_size": 2, - "n_shared_experts": 0 - }, - "name": "gptoss" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 4, - "hidden": 2880, - "intermediate": 2880, - "num_experts": 128, - "top_k": 4, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "expert_parallel_size": 1, - "routed_tensor_parallel_size": 2, - "shared_tensor_parallel_size": 2, - "n_shared_experts": 0 - }, - "name": "gptoss" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 8192, - "hidden": 2880, - "intermediate": 2880, - "num_experts": 128, - "top_k": 4, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "expert_parallel_size": 1, - "routed_tensor_parallel_size": 2, - "shared_tensor_parallel_size": 2, - "n_shared_experts": 0 - }, - "name": "gptoss" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 8, - "hidden": 2880, - "intermediate": 2880, - "num_experts": 128, - "top_k": 4, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "expert_parallel_size": 1, - "routed_tensor_parallel_size": 2, - "shared_tensor_parallel_size": 2, - "n_shared_experts": 0 - }, - "name": "gptoss" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 16384, - "hidden": 2880, - "intermediate": 2880, - "num_experts": 128, - "top_k": 4, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "expert_parallel_size": 1, - "routed_tensor_parallel_size": 2, - "shared_tensor_parallel_size": 2, - "n_shared_experts": 0 - }, - "name": "gptoss" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 16, - "hidden": 2880, - "intermediate": 2880, - "num_experts": 128, - "top_k": 4, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "expert_parallel_size": 1, - "routed_tensor_parallel_size": 2, - "shared_tensor_parallel_size": 2, - "n_shared_experts": 0 - }, - "name": "gptoss" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 32768, - "hidden": 2880, - "intermediate": 2880, - "num_experts": 128, - "top_k": 4, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "expert_parallel_size": 1, - "routed_tensor_parallel_size": 2, - "shared_tensor_parallel_size": 2, - "n_shared_experts": 0 - }, - "name": "gptoss" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 32, - "hidden": 2880, - "intermediate": 2880, - "num_experts": 128, - "top_k": 4, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "expert_parallel_size": 1, - "routed_tensor_parallel_size": 2, - "shared_tensor_parallel_size": 2, - "n_shared_experts": 0 - }, - "name": "gptoss" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 64, - "hidden": 2880, - "intermediate": 2880, - "num_experts": 128, - "top_k": 4, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "expert_parallel_size": 1, - "routed_tensor_parallel_size": 2, - "shared_tensor_parallel_size": 2, - "n_shared_experts": 0 - }, - "name": "gptoss" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 128, - "hidden": 2880, - "intermediate": 2880, - "num_experts": 128, - "top_k": 4, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "expert_parallel_size": 1, - "routed_tensor_parallel_size": 2, - "shared_tensor_parallel_size": 2, - "n_shared_experts": 0 - }, - "name": "gptoss" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 256, - "hidden": 2880, - "intermediate": 2880, - "num_experts": 128, - "top_k": 4, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "expert_parallel_size": 1, - "routed_tensor_parallel_size": 2, - "shared_tensor_parallel_size": 2, - "n_shared_experts": 0 - }, - "name": "gptoss" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 32768, - "hidden": 2880, - "intermediate": 2880, - "num_experts": 128, - "top_k": 4, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "expert_parallel_size": 4, - "routed_tensor_parallel_size": 1, - "shared_tensor_parallel_size": 1, - "n_shared_experts": 0 - }, - "name": "gptoss" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 64, - "hidden": 2880, - "intermediate": 2880, - "num_experts": 128, - "top_k": 4, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "expert_parallel_size": 4, - "routed_tensor_parallel_size": 1, - "shared_tensor_parallel_size": 1, - "n_shared_experts": 0 - }, - "name": "gptoss" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 4096, - "hidden": 2880, - "intermediate": 2880, - "num_experts": 128, - "top_k": 4, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "expert_parallel_size": 1, - "routed_tensor_parallel_size": 4, - "shared_tensor_parallel_size": 4, - "n_shared_experts": 0 - }, - "name": "gptoss" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 4, - "hidden": 2880, - "intermediate": 2880, - "num_experts": 128, - "top_k": 4, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "expert_parallel_size": 1, - "routed_tensor_parallel_size": 4, - "shared_tensor_parallel_size": 4, - "n_shared_experts": 0 - }, - "name": "gptoss" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 4096, - "hidden": 2880, - "intermediate": 2880, - "num_experts": 128, - "top_k": 4, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "expert_parallel_size": 1, - "routed_tensor_parallel_size": 8, - "shared_tensor_parallel_size": 8, - "n_shared_experts": 0 - }, - "name": "gptoss" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 4, - "hidden": 2880, - "intermediate": 2880, - "num_experts": 128, - "top_k": 4, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "expert_parallel_size": 1, - "routed_tensor_parallel_size": 8, - "shared_tensor_parallel_size": 8, - "n_shared_experts": 0 - }, - "name": "gptoss" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 4, - "hidden": 2880, - "intermediate": 2880, - "num_experts": 128, - "top_k": 4, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "expert_parallel_size": 1, - "routed_tensor_parallel_size": 1, - "shared_tensor_parallel_size": 1, - "n_shared_experts": 0 - }, - "name": "gptoss" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 8, - "hidden": 2880, - "intermediate": 2880, - "num_experts": 128, - "top_k": 4, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "expert_parallel_size": 1, - "routed_tensor_parallel_size": 1, - "shared_tensor_parallel_size": 1, - "n_shared_experts": 0 - }, - "name": "gptoss" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 16, - "hidden": 2880, - "intermediate": 2880, - "num_experts": 128, - "top_k": 4, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "expert_parallel_size": 1, - "routed_tensor_parallel_size": 1, - "shared_tensor_parallel_size": 1, - "n_shared_experts": 0 - }, - "name": "gptoss" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 32, - "hidden": 2880, - "intermediate": 2880, - "num_experts": 128, - "top_k": 4, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "expert_parallel_size": 1, - "routed_tensor_parallel_size": 1, - "shared_tensor_parallel_size": 1, - "n_shared_experts": 0 - }, - "name": "gptoss" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 64, - "hidden": 2880, - "intermediate": 2880, - "num_experts": 128, - "top_k": 4, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "expert_parallel_size": 1, - "routed_tensor_parallel_size": 1, - "shared_tensor_parallel_size": 1, - "n_shared_experts": 0 - }, - "name": "gptoss" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 128, - "hidden": 2880, - "intermediate": 2880, - "num_experts": 128, - "top_k": 4, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "expert_parallel_size": 1, - "routed_tensor_parallel_size": 1, - "shared_tensor_parallel_size": 1, - "n_shared_experts": 0 - }, - "name": "gptoss" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 32768, - "hidden": 2880, - "intermediate": 2880, - "num_experts": 128, - "top_k": 4, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "expert_parallel_size": 1, - "routed_tensor_parallel_size": 4, - "shared_tensor_parallel_size": 4, - "n_shared_experts": 0 - }, - "name": "gptoss" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 8, - "hidden": 2880, - "intermediate": 2880, - "num_experts": 128, - "top_k": 4, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "expert_parallel_size": 1, - "routed_tensor_parallel_size": 4, - "shared_tensor_parallel_size": 4, - "n_shared_experts": 0 - }, - "name": "gptoss" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 16, - "hidden": 2880, - "intermediate": 2880, - "num_experts": 128, - "top_k": 4, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "expert_parallel_size": 1, - "routed_tensor_parallel_size": 4, - "shared_tensor_parallel_size": 4, - "n_shared_experts": 0 - }, - "name": "gptoss" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 32, - "hidden": 2880, - "intermediate": 2880, - "num_experts": 128, - "top_k": 4, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "expert_parallel_size": 1, - "routed_tensor_parallel_size": 4, - "shared_tensor_parallel_size": 4, - "n_shared_experts": 0 - }, - "name": "gptoss" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 32768, - "hidden": 2880, - "intermediate": 2880, - "num_experts": 128, - "top_k": 4, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "expert_parallel_size": 1, - "routed_tensor_parallel_size": 8, - "shared_tensor_parallel_size": 8, - "n_shared_experts": 0 - }, - "name": "gptoss" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 4096, - "hidden": 2880, - "intermediate": 2880, - "num_experts": 128, - "top_k": 4, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "expert_parallel_size": 1, - "routed_tensor_parallel_size": 1, - "shared_tensor_parallel_size": 1, - "n_shared_experts": 0 - }, - "name": "gptoss" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 8192, - "hidden": 2880, - "intermediate": 2880, - "num_experts": 128, - "top_k": 4, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "expert_parallel_size": 1, - "routed_tensor_parallel_size": 1, - "shared_tensor_parallel_size": 1, - "n_shared_experts": 0 - }, - "name": "gptoss" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 16384, - "hidden": 2880, - "intermediate": 2880, - "num_experts": 128, - "top_k": 4, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "expert_parallel_size": 1, - "routed_tensor_parallel_size": 1, - "shared_tensor_parallel_size": 1, - "n_shared_experts": 0 - }, - "name": "gptoss" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 8192, - "hidden": 2880, - "intermediate": 2880, - "num_experts": 128, - "top_k": 4, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "expert_parallel_size": 1, - "routed_tensor_parallel_size": 4, - "shared_tensor_parallel_size": 4, - "n_shared_experts": 0 - }, - "name": "gptoss" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 16384, - "hidden": 2880, - "intermediate": 2880, - "num_experts": 128, - "top_k": 4, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "expert_parallel_size": 1, - "routed_tensor_parallel_size": 4, - "shared_tensor_parallel_size": 4, - "n_shared_experts": 0 - }, - "name": "gptoss" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 64, - "hidden": 2880, - "intermediate": 2880, - "num_experts": 128, - "top_k": 4, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "expert_parallel_size": 1, - "routed_tensor_parallel_size": 4, - "shared_tensor_parallel_size": 4, - "n_shared_experts": 0 - }, - "name": "gptoss" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 8192, - "hidden": 2880, - "intermediate": 2880, - "num_experts": 128, - "top_k": 4, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "expert_parallel_size": 1, - "routed_tensor_parallel_size": 8, - "shared_tensor_parallel_size": 8, - "n_shared_experts": 0 - }, - "name": "gptoss" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 8, - "hidden": 2880, - "intermediate": 2880, - "num_experts": 128, - "top_k": 4, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "expert_parallel_size": 1, - "routed_tensor_parallel_size": 8, - "shared_tensor_parallel_size": 8, - "n_shared_experts": 0 - }, - "name": "gptoss" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 32768, - "hidden": 3072, - "intermediate": 1536, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "expert_parallel_size": 2, - "routed_tensor_parallel_size": 1, - "shared_tensor_parallel_size": 1, - "n_shared_experts": 0 - }, - "name": "minimaxm2.5" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 512, - "hidden": 3072, - "intermediate": 1536, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "expert_parallel_size": 2, - "routed_tensor_parallel_size": 1, - "shared_tensor_parallel_size": 1, - "n_shared_experts": 0 - }, - "name": "minimaxm2.5" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 4096, - "hidden": 3072, - "intermediate": 1536, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "expert_parallel_size": 1, - "routed_tensor_parallel_size": 4, - "shared_tensor_parallel_size": 4, - "n_shared_experts": 0 - }, - "name": "minimaxm2.5" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 4, - "hidden": 3072, - "intermediate": 1536, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "expert_parallel_size": 1, - "routed_tensor_parallel_size": 4, - "shared_tensor_parallel_size": 4, - "n_shared_experts": 0 - }, - "name": "minimaxm2.5" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 8192, - "hidden": 3072, - "intermediate": 1536, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "expert_parallel_size": 1, - "routed_tensor_parallel_size": 4, - "shared_tensor_parallel_size": 4, - "n_shared_experts": 0 - }, - "name": "minimaxm2.5" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 8, - "hidden": 3072, - "intermediate": 1536, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "expert_parallel_size": 1, - "routed_tensor_parallel_size": 4, - "shared_tensor_parallel_size": 4, - "n_shared_experts": 0 - }, - "name": "minimaxm2.5" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 16384, - "hidden": 3072, - "intermediate": 1536, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "expert_parallel_size": 1, - "routed_tensor_parallel_size": 4, - "shared_tensor_parallel_size": 4, - "n_shared_experts": 0 - }, - "name": "minimaxm2.5" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 16, - "hidden": 3072, - "intermediate": 1536, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "expert_parallel_size": 1, - "routed_tensor_parallel_size": 4, - "shared_tensor_parallel_size": 4, - "n_shared_experts": 0 - }, - "name": "minimaxm2.5" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 32768, - "hidden": 3072, - "intermediate": 1536, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "expert_parallel_size": 1, - "routed_tensor_parallel_size": 4, - "shared_tensor_parallel_size": 4, - "n_shared_experts": 0 - }, - "name": "minimaxm2.5" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 32, - "hidden": 3072, - "intermediate": 1536, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "expert_parallel_size": 1, - "routed_tensor_parallel_size": 4, - "shared_tensor_parallel_size": 4, - "n_shared_experts": 0 - }, - "name": "minimaxm2.5" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 64, - "hidden": 3072, - "intermediate": 1536, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "expert_parallel_size": 1, - "routed_tensor_parallel_size": 4, - "shared_tensor_parallel_size": 4, - "n_shared_experts": 0 - }, - "name": "minimaxm2.5" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 128, - "hidden": 3072, - "intermediate": 1536, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "expert_parallel_size": 1, - "routed_tensor_parallel_size": 4, - "shared_tensor_parallel_size": 4, - "n_shared_experts": 0 - }, - "name": "minimaxm2.5" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 32768, - "hidden": 3072, - "intermediate": 1536, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "expert_parallel_size": 4, - "routed_tensor_parallel_size": 1, - "shared_tensor_parallel_size": 1, - "n_shared_experts": 0 - }, - "name": "minimaxm2.5" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 256, - "hidden": 3072, - "intermediate": 1536, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "expert_parallel_size": 4, - "routed_tensor_parallel_size": 1, - "shared_tensor_parallel_size": 1, - "n_shared_experts": 0 - }, - "name": "minimaxm2.5" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 512, - "hidden": 3072, - "intermediate": 1536, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "expert_parallel_size": 4, - "routed_tensor_parallel_size": 1, - "shared_tensor_parallel_size": 1, - "n_shared_experts": 0 - }, - "name": "minimaxm2.5" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 32768, - "hidden": 3072, - "intermediate": 1536, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "expert_parallel_size": 1, - "routed_tensor_parallel_size": 2, - "shared_tensor_parallel_size": 2, - "n_shared_experts": 0 - }, - "name": "minimaxm2.5" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 4, - "hidden": 3072, - "intermediate": 1536, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "expert_parallel_size": 1, - "routed_tensor_parallel_size": 2, - "shared_tensor_parallel_size": 2, - "n_shared_experts": 0 - }, - "name": "minimaxm2.5" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 8, - "hidden": 3072, - "intermediate": 1536, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "expert_parallel_size": 1, - "routed_tensor_parallel_size": 2, - "shared_tensor_parallel_size": 2, - "n_shared_experts": 0 - }, - "name": "minimaxm2.5" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 16, - "hidden": 3072, - "intermediate": 1536, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "expert_parallel_size": 1, - "routed_tensor_parallel_size": 2, - "shared_tensor_parallel_size": 2, - "n_shared_experts": 0 - }, - "name": "minimaxm2.5" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 32, - "hidden": 3072, - "intermediate": 1536, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "expert_parallel_size": 1, - "routed_tensor_parallel_size": 2, - "shared_tensor_parallel_size": 2, - "n_shared_experts": 0 - }, - "name": "minimaxm2.5" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 64, - "hidden": 3072, - "intermediate": 1536, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "expert_parallel_size": 1, - "routed_tensor_parallel_size": 2, - "shared_tensor_parallel_size": 2, - "n_shared_experts": 0 - }, - "name": "minimaxm2.5" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 128, - "hidden": 3072, - "intermediate": 1536, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "expert_parallel_size": 1, - "routed_tensor_parallel_size": 2, - "shared_tensor_parallel_size": 2, - "n_shared_experts": 0 - }, - "name": "minimaxm2.5" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 256, - "hidden": 3072, - "intermediate": 1536, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "expert_parallel_size": 1, - "routed_tensor_parallel_size": 2, - "shared_tensor_parallel_size": 2, - "n_shared_experts": 0 - }, - "name": "minimaxm2.5" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 512, - "hidden": 3072, - "intermediate": 1536, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "expert_parallel_size": 1, - "routed_tensor_parallel_size": 2, - "shared_tensor_parallel_size": 2, - "n_shared_experts": 0 - }, - "name": "minimaxm2.5" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 256, - "hidden": 3072, - "intermediate": 1536, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "expert_parallel_size": 1, - "routed_tensor_parallel_size": 4, - "shared_tensor_parallel_size": 4, - "n_shared_experts": 0 - }, - "name": "minimaxm2.5" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 512, - "hidden": 3072, - "intermediate": 1536, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "expert_parallel_size": 1, - "routed_tensor_parallel_size": 4, - "shared_tensor_parallel_size": 4, - "n_shared_experts": 0 - }, - "name": "minimaxm2.5" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 1024, - "hidden": 3072, - "intermediate": 1536, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "expert_parallel_size": 2, - "routed_tensor_parallel_size": 1, - "shared_tensor_parallel_size": 1, - "n_shared_experts": 0 - }, - "name": "minimaxm2.5" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 32768, - "hidden": 3072, - "intermediate": 1536, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "expert_parallel_size": 1, - "routed_tensor_parallel_size": 1, - "shared_tensor_parallel_size": 1, - "n_shared_experts": 0 - }, - "name": "minimaxm2.5" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 4, - "hidden": 3072, - "intermediate": 1536, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "expert_parallel_size": 1, - "routed_tensor_parallel_size": 1, - "shared_tensor_parallel_size": 1, - "n_shared_experts": 0 - }, - "name": "minimaxm2.5" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 8, - "hidden": 3072, - "intermediate": 1536, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "expert_parallel_size": 1, - "routed_tensor_parallel_size": 1, - "shared_tensor_parallel_size": 1, - "n_shared_experts": 0 - }, - "name": "minimaxm2.5" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 16, - "hidden": 3072, - "intermediate": 1536, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "expert_parallel_size": 1, - "routed_tensor_parallel_size": 1, - "shared_tensor_parallel_size": 1, - "n_shared_experts": 0 - }, - "name": "minimaxm2.5" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 4096, - "hidden": 3072, - "intermediate": 1536, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "expert_parallel_size": 1, - "routed_tensor_parallel_size": 1, - "shared_tensor_parallel_size": 1, - "n_shared_experts": 0 - }, - "name": "minimaxm2.5" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 4, - "hidden": 3072, - "intermediate": 1536, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "expert_parallel_size": 1, - "routed_tensor_parallel_size": 1, - "shared_tensor_parallel_size": 1, - "n_shared_experts": 0 - }, - "name": "minimaxm2.5" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 8192, - "hidden": 3072, - "intermediate": 1536, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "expert_parallel_size": 1, - "routed_tensor_parallel_size": 1, - "shared_tensor_parallel_size": 1, - "n_shared_experts": 0 - }, - "name": "minimaxm2.5" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 8, - "hidden": 3072, - "intermediate": 1536, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "expert_parallel_size": 1, - "routed_tensor_parallel_size": 1, - "shared_tensor_parallel_size": 1, - "n_shared_experts": 0 - }, - "name": "minimaxm2.5" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 16384, - "hidden": 3072, - "intermediate": 1536, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "expert_parallel_size": 1, - "routed_tensor_parallel_size": 1, - "shared_tensor_parallel_size": 1, - "n_shared_experts": 0 - }, - "name": "minimaxm2.5" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 16, - "hidden": 3072, - "intermediate": 1536, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "expert_parallel_size": 1, - "routed_tensor_parallel_size": 1, - "shared_tensor_parallel_size": 1, - "n_shared_experts": 0 - }, - "name": "minimaxm2.5" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 16384, - "hidden": 3072, - "intermediate": 1536, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "expert_parallel_size": 1, - "routed_tensor_parallel_size": 2, - "shared_tensor_parallel_size": 2, - "n_shared_experts": 0 - }, - "name": "minimaxm2.5" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 16, - "hidden": 3072, - "intermediate": 1536, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "expert_parallel_size": 1, - "routed_tensor_parallel_size": 2, - "shared_tensor_parallel_size": 2, - "n_shared_experts": 0 - }, - "name": "minimaxm2.5" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 32768, - "hidden": 3072, - "intermediate": 1536, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "expert_parallel_size": 2, - "routed_tensor_parallel_size": 1, - "shared_tensor_parallel_size": 1, - "n_shared_experts": 0 - }, - "name": "minimaxm2.5" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 128, - "hidden": 3072, - "intermediate": 1536, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "expert_parallel_size": 2, - "routed_tensor_parallel_size": 1, - "shared_tensor_parallel_size": 1, - "n_shared_experts": 0 - }, - "name": "minimaxm2.5" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 256, - "hidden": 3072, - "intermediate": 1536, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "expert_parallel_size": 2, - "routed_tensor_parallel_size": 1, - "shared_tensor_parallel_size": 1, - "n_shared_experts": 0 - }, - "name": "minimaxm2.5" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 512, - "hidden": 3072, - "intermediate": 1536, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "expert_parallel_size": 2, - "routed_tensor_parallel_size": 1, - "shared_tensor_parallel_size": 1, - "n_shared_experts": 0 - }, - "name": "minimaxm2.5" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 1024, - "hidden": 3072, - "intermediate": 1536, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "expert_parallel_size": 2, - "routed_tensor_parallel_size": 1, - "shared_tensor_parallel_size": 1, - "n_shared_experts": 0 - }, - "name": "minimaxm2.5" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 4096, - "hidden": 3072, - "intermediate": 1536, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "expert_parallel_size": 1, - "routed_tensor_parallel_size": 4, - "shared_tensor_parallel_size": 4, - "n_shared_experts": 0 - }, - "name": "minimaxm2.5" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 4, - "hidden": 3072, - "intermediate": 1536, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "expert_parallel_size": 1, - "routed_tensor_parallel_size": 4, - "shared_tensor_parallel_size": 4, - "n_shared_experts": 0 - }, - "name": "minimaxm2.5" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 8192, - "hidden": 3072, - "intermediate": 1536, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "expert_parallel_size": 1, - "routed_tensor_parallel_size": 4, - "shared_tensor_parallel_size": 4, - "n_shared_experts": 0 - }, - "name": "minimaxm2.5" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 8, - "hidden": 3072, - "intermediate": 1536, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "expert_parallel_size": 1, - "routed_tensor_parallel_size": 4, - "shared_tensor_parallel_size": 4, - "n_shared_experts": 0 - }, - "name": "minimaxm2.5" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 16384, - "hidden": 3072, - "intermediate": 1536, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "expert_parallel_size": 1, - "routed_tensor_parallel_size": 4, - "shared_tensor_parallel_size": 4, - "n_shared_experts": 0 - }, - "name": "minimaxm2.5" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 16, - "hidden": 3072, - "intermediate": 1536, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "expert_parallel_size": 1, - "routed_tensor_parallel_size": 4, - "shared_tensor_parallel_size": 4, - "n_shared_experts": 0 - }, - "name": "minimaxm2.5" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 32768, - "hidden": 3072, - "intermediate": 1536, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "expert_parallel_size": 4, - "routed_tensor_parallel_size": 1, - "shared_tensor_parallel_size": 1, - "n_shared_experts": 0 - }, - "name": "minimaxm2.5" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 64, - "hidden": 3072, - "intermediate": 1536, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "expert_parallel_size": 4, - "routed_tensor_parallel_size": 1, - "shared_tensor_parallel_size": 1, - "n_shared_experts": 0 - }, - "name": "minimaxm2.5" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 128, - "hidden": 3072, - "intermediate": 1536, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "expert_parallel_size": 4, - "routed_tensor_parallel_size": 1, - "shared_tensor_parallel_size": 1, - "n_shared_experts": 0 - }, - "name": "minimaxm2.5" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 4096, - "hidden": 3072, - "intermediate": 1536, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "expert_parallel_size": 1, - "routed_tensor_parallel_size": 8, - "shared_tensor_parallel_size": 8, - "n_shared_experts": 0 - }, - "name": "minimaxm2.5" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 4, - "hidden": 3072, - "intermediate": 1536, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "expert_parallel_size": 1, - "routed_tensor_parallel_size": 8, - "shared_tensor_parallel_size": 8, - "n_shared_experts": 0 - }, - "name": "minimaxm2.5" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 8192, - "hidden": 3072, - "intermediate": 1536, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "expert_parallel_size": 1, - "routed_tensor_parallel_size": 8, - "shared_tensor_parallel_size": 8, - "n_shared_experts": 0 - }, - "name": "minimaxm2.5" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 8, - "hidden": 3072, - "intermediate": 1536, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "expert_parallel_size": 1, - "routed_tensor_parallel_size": 8, - "shared_tensor_parallel_size": 8, - "n_shared_experts": 0 - }, - "name": "minimaxm2.5" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 32768, - "hidden": 3072, - "intermediate": 1536, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "expert_parallel_size": 1, - "routed_tensor_parallel_size": 1, - "shared_tensor_parallel_size": 1, - "n_shared_experts": 0 - }, - "name": "minimaxm2.5" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 32, - "hidden": 3072, - "intermediate": 1536, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "expert_parallel_size": 1, - "routed_tensor_parallel_size": 1, - "shared_tensor_parallel_size": 1, - "n_shared_experts": 0 - }, - "name": "minimaxm2.5" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 256, - "hidden": 3072, - "intermediate": 1536, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "expert_parallel_size": 1, - "routed_tensor_parallel_size": 1, - "shared_tensor_parallel_size": 1, - "n_shared_experts": 0 - }, - "name": "minimaxm2.5" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 32768, - "hidden": 3072, - "intermediate": 1536, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "expert_parallel_size": 1, - "routed_tensor_parallel_size": 4, - "shared_tensor_parallel_size": 4, - "n_shared_experts": 0 - }, - "name": "minimaxm2.5" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 32768, - "hidden": 3072, - "intermediate": 1536, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "expert_parallel_size": 1, - "routed_tensor_parallel_size": 8, - "shared_tensor_parallel_size": 8, - "n_shared_experts": 0 - }, - "name": "minimaxm2.5" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 2048, - "hidden": 3072, - "intermediate": 1536, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "expert_parallel_size": 2, - "routed_tensor_parallel_size": 1, - "shared_tensor_parallel_size": 1, - "n_shared_experts": 0 - }, - "name": "minimaxm2.5" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 64, - "hidden": 3072, - "intermediate": 1536, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "expert_parallel_size": 1, - "routed_tensor_parallel_size": 1, - "shared_tensor_parallel_size": 1, - "n_shared_experts": 0 - }, - "name": "minimaxm2.5" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 128, - "hidden": 3072, - "intermediate": 1536, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "expert_parallel_size": 1, - "routed_tensor_parallel_size": 1, - "shared_tensor_parallel_size": 1, - "n_shared_experts": 0 - }, - "name": "minimaxm2.5" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 16384, - "hidden": 2880, - "intermediate": 2880, - "num_experts": 128, - "top_k": 4, - "dtype_act": "bf16", - "dtype_weight": "mxfp4", - "expert_parallel_size": 1, - "routed_tensor_parallel_size": 8, - "shared_tensor_parallel_size": 8, - "n_shared_experts": 0 - }, - "name": "gptoss" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 16, - "hidden": 2880, - "intermediate": 2880, - "num_experts": 128, - "top_k": 4, - "dtype_act": "bf16", - "dtype_weight": "mxfp4", - "expert_parallel_size": 1, - "routed_tensor_parallel_size": 8, - "shared_tensor_parallel_size": 8, - "n_shared_experts": 0 - }, - "name": "gptoss" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 32768, - "hidden": 2880, - "intermediate": 2880, - "num_experts": 128, - "top_k": 4, - "dtype_act": "bf16", - "dtype_weight": "mxfp4", - "expert_parallel_size": 1, - "routed_tensor_parallel_size": 8, - "shared_tensor_parallel_size": 8, - "n_shared_experts": 0 - }, - "name": "gptoss" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 32, - "hidden": 2880, - "intermediate": 2880, - "num_experts": 128, - "top_k": 4, - "dtype_act": "bf16", - "dtype_weight": "mxfp4", - "expert_parallel_size": 1, - "routed_tensor_parallel_size": 8, - "shared_tensor_parallel_size": 8, - "n_shared_experts": 0 - }, - "name": "gptoss" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 64, - "hidden": 2880, - "intermediate": 2880, - "num_experts": 128, - "top_k": 4, - "dtype_act": "bf16", - "dtype_weight": "mxfp4", - "expert_parallel_size": 1, - "routed_tensor_parallel_size": 8, - "shared_tensor_parallel_size": 8, - "n_shared_experts": 0 - }, - "name": "gptoss" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 8, - "hidden": 2880, - "intermediate": 2880, - "num_experts": 128, - "top_k": 4, - "dtype_act": "bf16", - "dtype_weight": "mxfp4", - "expert_parallel_size": 1, - "routed_tensor_parallel_size": 8, - "shared_tensor_parallel_size": 8, - "n_shared_experts": 0 - }, - "name": "gptoss" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 4096, - "hidden": 3072, - "intermediate": 1536, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "expert_parallel_size": 4, - "routed_tensor_parallel_size": 1, - "shared_tensor_parallel_size": 1, - "n_shared_experts": 0 - }, - "name": "minimaxm2.5" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 4, - "hidden": 3072, - "intermediate": 1536, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "expert_parallel_size": 4, - "routed_tensor_parallel_size": 1, - "shared_tensor_parallel_size": 1, - "n_shared_experts": 0 - }, - "name": "minimaxm2.5" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 8192, - "hidden": 3072, - "intermediate": 1536, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "expert_parallel_size": 4, - "routed_tensor_parallel_size": 1, - "shared_tensor_parallel_size": 1, - "n_shared_experts": 0 - }, - "name": "minimaxm2.5" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 8, - "hidden": 3072, - "intermediate": 1536, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "expert_parallel_size": 4, - "routed_tensor_parallel_size": 1, - "shared_tensor_parallel_size": 1, - "n_shared_experts": 0 - }, - "name": "minimaxm2.5" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 16384, - "hidden": 3072, - "intermediate": 1536, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "expert_parallel_size": 4, - "routed_tensor_parallel_size": 1, - "shared_tensor_parallel_size": 1, - "n_shared_experts": 0 - }, - "name": "minimaxm2.5" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 16, - "hidden": 3072, - "intermediate": 1536, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "expert_parallel_size": 4, - "routed_tensor_parallel_size": 1, - "shared_tensor_parallel_size": 1, - "n_shared_experts": 0 - }, - "name": "minimaxm2.5" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 32, - "hidden": 3072, - "intermediate": 1536, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "expert_parallel_size": 4, - "routed_tensor_parallel_size": 1, - "shared_tensor_parallel_size": 1, - "n_shared_experts": 0 - }, - "name": "minimaxm2.5" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 64, - "hidden": 3072, - "intermediate": 1536, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "expert_parallel_size": 4, - "routed_tensor_parallel_size": 1, - "shared_tensor_parallel_size": 1, - "n_shared_experts": 0 - }, - "name": "minimaxm2.5" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 1024, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "expert_parallel_size": 1, - "routed_tensor_parallel_size": 16, - "shared_tensor_parallel_size": 16, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 2048, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "expert_parallel_size": 1, - "routed_tensor_parallel_size": 16, - "shared_tensor_parallel_size": 16, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 4096, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "expert_parallel_size": 1, - "routed_tensor_parallel_size": 16, - "shared_tensor_parallel_size": 16, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 8192, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "expert_parallel_size": 1, - "routed_tensor_parallel_size": 16, - "shared_tensor_parallel_size": 16, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 16384, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "expert_parallel_size": 1, - "routed_tensor_parallel_size": 16, - "shared_tensor_parallel_size": 16, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 32768, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "expert_parallel_size": 1, - "routed_tensor_parallel_size": 16, - "shared_tensor_parallel_size": 16, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 8, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "expert_parallel_size": 16, - "routed_tensor_parallel_size": 1, - "shared_tensor_parallel_size": 1, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 16, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "expert_parallel_size": 16, - "routed_tensor_parallel_size": 1, - "shared_tensor_parallel_size": 1, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 32, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "expert_parallel_size": 16, - "routed_tensor_parallel_size": 1, - "shared_tensor_parallel_size": 1, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 64, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "expert_parallel_size": 16, - "routed_tensor_parallel_size": 1, - "shared_tensor_parallel_size": 1, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 128, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "expert_parallel_size": 16, - "routed_tensor_parallel_size": 1, - "shared_tensor_parallel_size": 1, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 4096, - "hidden": 2880, - "intermediate": 2880, - "num_experts": 128, - "top_k": 4, - "dtype_act": "bf16", - "dtype_weight": "mxfp4", - "expert_parallel_size": 1, - "routed_tensor_parallel_size": 1, - "shared_tensor_parallel_size": 1, - "n_shared_experts": 0 - }, - "name": "gptoss" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 4, - "hidden": 2880, - "intermediate": 2880, - "num_experts": 128, - "top_k": 4, - "dtype_act": "bf16", - "dtype_weight": "mxfp4", - "expert_parallel_size": 1, - "routed_tensor_parallel_size": 1, - "shared_tensor_parallel_size": 1, - "n_shared_experts": 0 - }, - "name": "gptoss" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 8192, - "hidden": 2880, - "intermediate": 2880, - "num_experts": 128, - "top_k": 4, - "dtype_act": "bf16", - "dtype_weight": "mxfp4", - "expert_parallel_size": 1, - "routed_tensor_parallel_size": 1, - "shared_tensor_parallel_size": 1, - "n_shared_experts": 0 - }, - "name": "gptoss" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 8, - "hidden": 2880, - "intermediate": 2880, - "num_experts": 128, - "top_k": 4, - "dtype_act": "bf16", - "dtype_weight": "mxfp4", - "expert_parallel_size": 1, - "routed_tensor_parallel_size": 1, - "shared_tensor_parallel_size": 1, - "n_shared_experts": 0 - }, - "name": "gptoss" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 16384, - "hidden": 2880, - "intermediate": 2880, - "num_experts": 128, - "top_k": 4, - "dtype_act": "bf16", - "dtype_weight": "mxfp4", - "expert_parallel_size": 1, - "routed_tensor_parallel_size": 1, - "shared_tensor_parallel_size": 1, - "n_shared_experts": 0 - }, - "name": "gptoss" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 16, - "hidden": 2880, - "intermediate": 2880, - "num_experts": 128, - "top_k": 4, - "dtype_act": "bf16", - "dtype_weight": "mxfp4", - "expert_parallel_size": 1, - "routed_tensor_parallel_size": 1, - "shared_tensor_parallel_size": 1, - "n_shared_experts": 0 - }, - "name": "gptoss" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 32768, - "hidden": 2880, - "intermediate": 2880, - "num_experts": 128, - "top_k": 4, - "dtype_act": "bf16", - "dtype_weight": "mxfp4", - "expert_parallel_size": 1, - "routed_tensor_parallel_size": 1, - "shared_tensor_parallel_size": 1, - "n_shared_experts": 0 - }, - "name": "gptoss" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 32, - "hidden": 2880, - "intermediate": 2880, - "num_experts": 128, - "top_k": 4, - "dtype_act": "bf16", - "dtype_weight": "mxfp4", - "expert_parallel_size": 1, - "routed_tensor_parallel_size": 1, - "shared_tensor_parallel_size": 1, - "n_shared_experts": 0 - }, - "name": "gptoss" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 64, - "hidden": 2880, - "intermediate": 2880, - "num_experts": 128, - "top_k": 4, - "dtype_act": "bf16", - "dtype_weight": "mxfp4", - "expert_parallel_size": 1, - "routed_tensor_parallel_size": 1, - "shared_tensor_parallel_size": 1, - "n_shared_experts": 0 - }, - "name": "gptoss" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 4096, - "hidden": 2880, - "intermediate": 2880, - "num_experts": 128, - "top_k": 4, - "dtype_act": "bf16", - "dtype_weight": "mxfp4", - "expert_parallel_size": 2, - "routed_tensor_parallel_size": 1, - "shared_tensor_parallel_size": 1, - "n_shared_experts": 0 - }, - "name": "gptoss" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 4, - "hidden": 2880, - "intermediate": 2880, - "num_experts": 128, - "top_k": 4, - "dtype_act": "bf16", - "dtype_weight": "mxfp4", - "expert_parallel_size": 2, - "routed_tensor_parallel_size": 1, - "shared_tensor_parallel_size": 1, - "n_shared_experts": 0 - }, - "name": "gptoss" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 8192, - "hidden": 2880, - "intermediate": 2880, - "num_experts": 128, - "top_k": 4, - "dtype_act": "bf16", - "dtype_weight": "mxfp4", - "expert_parallel_size": 2, - "routed_tensor_parallel_size": 1, - "shared_tensor_parallel_size": 1, - "n_shared_experts": 0 - }, - "name": "gptoss" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 8, - "hidden": 2880, - "intermediate": 2880, - "num_experts": 128, - "top_k": 4, - "dtype_act": "bf16", - "dtype_weight": "mxfp4", - "expert_parallel_size": 2, - "routed_tensor_parallel_size": 1, - "shared_tensor_parallel_size": 1, - "n_shared_experts": 0 - }, - "name": "gptoss" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 16384, - "hidden": 2880, - "intermediate": 2880, - "num_experts": 128, - "top_k": 4, - "dtype_act": "bf16", - "dtype_weight": "mxfp4", - "expert_parallel_size": 2, - "routed_tensor_parallel_size": 1, - "shared_tensor_parallel_size": 1, - "n_shared_experts": 0 - }, - "name": "gptoss" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 16, - "hidden": 2880, - "intermediate": 2880, - "num_experts": 128, - "top_k": 4, - "dtype_act": "bf16", - "dtype_weight": "mxfp4", - "expert_parallel_size": 2, - "routed_tensor_parallel_size": 1, - "shared_tensor_parallel_size": 1, - "n_shared_experts": 0 - }, - "name": "gptoss" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 32768, - "hidden": 2880, - "intermediate": 2880, - "num_experts": 128, - "top_k": 4, - "dtype_act": "bf16", - "dtype_weight": "mxfp4", - "expert_parallel_size": 2, - "routed_tensor_parallel_size": 1, - "shared_tensor_parallel_size": 1, - "n_shared_experts": 0 - }, - "name": "gptoss" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 32, - "hidden": 2880, - "intermediate": 2880, - "num_experts": 128, - "top_k": 4, - "dtype_act": "bf16", - "dtype_weight": "mxfp4", - "expert_parallel_size": 2, - "routed_tensor_parallel_size": 1, - "shared_tensor_parallel_size": 1, - "n_shared_experts": 0 - }, - "name": "gptoss" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 64, - "hidden": 2880, - "intermediate": 2880, - "num_experts": 128, - "top_k": 4, - "dtype_act": "bf16", - "dtype_weight": "mxfp4", - "expert_parallel_size": 2, - "routed_tensor_parallel_size": 1, - "shared_tensor_parallel_size": 1, - "n_shared_experts": 0 - }, - "name": "gptoss" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 4096, - "hidden": 2880, - "intermediate": 2880, - "num_experts": 128, - "top_k": 4, - "dtype_act": "bf16", - "dtype_weight": "mxfp4", - "expert_parallel_size": 4, - "routed_tensor_parallel_size": 1, - "shared_tensor_parallel_size": 1, - "n_shared_experts": 0 - }, - "name": "gptoss" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 4, - "hidden": 2880, - "intermediate": 2880, - "num_experts": 128, - "top_k": 4, - "dtype_act": "bf16", - "dtype_weight": "mxfp4", - "expert_parallel_size": 4, - "routed_tensor_parallel_size": 1, - "shared_tensor_parallel_size": 1, - "n_shared_experts": 0 - }, - "name": "gptoss" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 8192, - "hidden": 2880, - "intermediate": 2880, - "num_experts": 128, - "top_k": 4, - "dtype_act": "bf16", - "dtype_weight": "mxfp4", - "expert_parallel_size": 4, - "routed_tensor_parallel_size": 1, - "shared_tensor_parallel_size": 1, - "n_shared_experts": 0 - }, - "name": "gptoss" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 8, - "hidden": 2880, - "intermediate": 2880, - "num_experts": 128, - "top_k": 4, - "dtype_act": "bf16", - "dtype_weight": "mxfp4", - "expert_parallel_size": 4, - "routed_tensor_parallel_size": 1, - "shared_tensor_parallel_size": 1, - "n_shared_experts": 0 - }, - "name": "gptoss" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 16384, - "hidden": 2880, - "intermediate": 2880, - "num_experts": 128, - "top_k": 4, - "dtype_act": "bf16", - "dtype_weight": "mxfp4", - "expert_parallel_size": 4, - "routed_tensor_parallel_size": 1, - "shared_tensor_parallel_size": 1, - "n_shared_experts": 0 - }, - "name": "gptoss" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 16, - "hidden": 2880, - "intermediate": 2880, - "num_experts": 128, - "top_k": 4, - "dtype_act": "bf16", - "dtype_weight": "mxfp4", - "expert_parallel_size": 4, - "routed_tensor_parallel_size": 1, - "shared_tensor_parallel_size": 1, - "n_shared_experts": 0 - }, - "name": "gptoss" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 32768, - "hidden": 2880, - "intermediate": 2880, - "num_experts": 128, - "top_k": 4, - "dtype_act": "bf16", - "dtype_weight": "mxfp4", - "expert_parallel_size": 4, - "routed_tensor_parallel_size": 1, - "shared_tensor_parallel_size": 1, - "n_shared_experts": 0 - }, - "name": "gptoss" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 32, - "hidden": 2880, - "intermediate": 2880, - "num_experts": 128, - "top_k": 4, - "dtype_act": "bf16", - "dtype_weight": "mxfp4", - "expert_parallel_size": 4, - "routed_tensor_parallel_size": 1, - "shared_tensor_parallel_size": 1, - "n_shared_experts": 0 - }, - "name": "gptoss" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 4096, - "hidden": 2880, - "intermediate": 2880, - "num_experts": 128, - "top_k": 4, - "dtype_act": "bf16", - "dtype_weight": "mxfp4", - "expert_parallel_size": 8, - "routed_tensor_parallel_size": 1, - "shared_tensor_parallel_size": 1, - "n_shared_experts": 0 - }, - "name": "gptoss" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 4, - "hidden": 2880, - "intermediate": 2880, - "num_experts": 128, - "top_k": 4, - "dtype_act": "bf16", - "dtype_weight": "mxfp4", - "expert_parallel_size": 8, - "routed_tensor_parallel_size": 1, - "shared_tensor_parallel_size": 1, - "n_shared_experts": 0 - }, - "name": "gptoss" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 8192, - "hidden": 2880, - "intermediate": 2880, - "num_experts": 128, - "top_k": 4, - "dtype_act": "bf16", - "dtype_weight": "mxfp4", - "expert_parallel_size": 8, - "routed_tensor_parallel_size": 1, - "shared_tensor_parallel_size": 1, - "n_shared_experts": 0 - }, - "name": "gptoss" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 8, - "hidden": 2880, - "intermediate": 2880, - "num_experts": 128, - "top_k": 4, - "dtype_act": "bf16", - "dtype_weight": "mxfp4", - "expert_parallel_size": 8, - "routed_tensor_parallel_size": 1, - "shared_tensor_parallel_size": 1, - "n_shared_experts": 0 - }, - "name": "gptoss" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 64, - "hidden": 2880, - "intermediate": 2880, - "num_experts": 128, - "top_k": 4, - "dtype_act": "bf16", - "dtype_weight": "mxfp4", - "expert_parallel_size": 4, - "routed_tensor_parallel_size": 1, - "shared_tensor_parallel_size": 1, - "n_shared_experts": 0 - }, - "name": "gptoss" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 32768, - "hidden": 2880, - "intermediate": 2880, - "num_experts": 128, - "top_k": 4, - "dtype_act": "bf16", - "dtype_weight": "mxfp4", - "expert_parallel_size": 8, - "routed_tensor_parallel_size": 1, - "shared_tensor_parallel_size": 1, - "n_shared_experts": 0 - }, - "name": "gptoss" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 4096, - "hidden": 3072, - "intermediate": 1536, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "expert_parallel_size": 1, - "routed_tensor_parallel_size": 8, - "shared_tensor_parallel_size": 8, - "n_shared_experts": 0 - }, - "name": "minimaxm2.5" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 4, - "hidden": 3072, - "intermediate": 1536, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "expert_parallel_size": 1, - "routed_tensor_parallel_size": 8, - "shared_tensor_parallel_size": 8, - "n_shared_experts": 0 - }, - "name": "minimaxm2.5" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 8192, - "hidden": 3072, - "intermediate": 1536, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "expert_parallel_size": 1, - "routed_tensor_parallel_size": 8, - "shared_tensor_parallel_size": 8, - "n_shared_experts": 0 - }, - "name": "minimaxm2.5" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 8, - "hidden": 3072, - "intermediate": 1536, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "expert_parallel_size": 1, - "routed_tensor_parallel_size": 8, - "shared_tensor_parallel_size": 8, - "n_shared_experts": 0 - }, - "name": "minimaxm2.5" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 16384, - "hidden": 3072, - "intermediate": 1536, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "expert_parallel_size": 1, - "routed_tensor_parallel_size": 8, - "shared_tensor_parallel_size": 8, - "n_shared_experts": 0 - }, - "name": "minimaxm2.5" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 16, - "hidden": 3072, - "intermediate": 1536, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "expert_parallel_size": 1, - "routed_tensor_parallel_size": 8, - "shared_tensor_parallel_size": 8, - "n_shared_experts": 0 - }, - "name": "minimaxm2.5" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 32768, - "hidden": 3072, - "intermediate": 1536, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "expert_parallel_size": 1, - "routed_tensor_parallel_size": 8, - "shared_tensor_parallel_size": 8, - "n_shared_experts": 0 - }, - "name": "minimaxm2.5" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 32, - "hidden": 3072, - "intermediate": 1536, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "expert_parallel_size": 1, - "routed_tensor_parallel_size": 8, - "shared_tensor_parallel_size": 8, - "n_shared_experts": 0 - }, - "name": "minimaxm2.5" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 64, - "hidden": 3072, - "intermediate": 1536, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "expert_parallel_size": 1, - "routed_tensor_parallel_size": 8, - "shared_tensor_parallel_size": 8, - "n_shared_experts": 0 - }, - "name": "minimaxm2.5" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 128, - "hidden": 3072, - "intermediate": 1536, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "expert_parallel_size": 1, - "routed_tensor_parallel_size": 8, - "shared_tensor_parallel_size": 8, - "n_shared_experts": 0 - }, - "name": "minimaxm2.5" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 360, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "expert_parallel_size": 32, - "routed_tensor_parallel_size": 1, - "shared_tensor_parallel_size": 1, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 12288, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "expert_parallel_size": 4, - "routed_tensor_parallel_size": 1, - "shared_tensor_parallel_size": 1, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 24576, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "expert_parallel_size": 4, - "routed_tensor_parallel_size": 1, - "shared_tensor_parallel_size": 1, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 8602, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "expert_parallel_size": 16, - "routed_tensor_parallel_size": 1, - "shared_tensor_parallel_size": 1, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 4506, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "expert_parallel_size": 32, - "routed_tensor_parallel_size": 1, - "shared_tensor_parallel_size": 1, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 4301, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "expert_parallel_size": 8, - "routed_tensor_parallel_size": 1, - "shared_tensor_parallel_size": 1, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 666, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "expert_parallel_size": 32, - "routed_tensor_parallel_size": 1, - "shared_tensor_parallel_size": 1, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 5120, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "expert_parallel_size": 4, - "routed_tensor_parallel_size": 1, - "shared_tensor_parallel_size": 1, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 4301, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "expert_parallel_size": 16, - "routed_tensor_parallel_size": 1, - "shared_tensor_parallel_size": 1, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 2253, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "expert_parallel_size": 32, - "routed_tensor_parallel_size": 1, - "shared_tensor_parallel_size": 1, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 2458, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "expert_parallel_size": 16, - "routed_tensor_parallel_size": 1, - "shared_tensor_parallel_size": 1, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 1332, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "expert_parallel_size": 32, - "routed_tensor_parallel_size": 1, - "shared_tensor_parallel_size": 1, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 11, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "expert_parallel_size": 8, - "routed_tensor_parallel_size": 1, - "shared_tensor_parallel_size": 1, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 19, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "expert_parallel_size": 8, - "routed_tensor_parallel_size": 1, - "shared_tensor_parallel_size": 1, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 333, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "expert_parallel_size": 32, - "routed_tensor_parallel_size": 1, - "shared_tensor_parallel_size": 1, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 1229, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "expert_parallel_size": 32, - "routed_tensor_parallel_size": 1, - "shared_tensor_parallel_size": 1, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 2253, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "expert_parallel_size": 16, - "routed_tensor_parallel_size": 1, - "shared_tensor_parallel_size": 1, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 4096, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "expert_parallel_size": 16, - "routed_tensor_parallel_size": 1, - "shared_tensor_parallel_size": 1, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 1230, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "expert_parallel_size": 32, - "routed_tensor_parallel_size": 1, - "shared_tensor_parallel_size": 1, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 24, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "expert_parallel_size": 8, - "routed_tensor_parallel_size": 1, - "shared_tensor_parallel_size": 1, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 18432, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "expert_parallel_size": 8, - "routed_tensor_parallel_size": 1, - "shared_tensor_parallel_size": 1, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 12, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "expert_parallel_size": 8, - "routed_tensor_parallel_size": 1, - "shared_tensor_parallel_size": 1, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 9216, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "expert_parallel_size": 8, - "routed_tensor_parallel_size": 1, - "shared_tensor_parallel_size": 1, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 6, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "expert_parallel_size": 8, - "routed_tensor_parallel_size": 1, - "shared_tensor_parallel_size": 1, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 2151, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "expert_parallel_size": 16, - "routed_tensor_parallel_size": 1, - "shared_tensor_parallel_size": 1, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 1127, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "expert_parallel_size": 32, - "routed_tensor_parallel_size": 1, - "shared_tensor_parallel_size": 1, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 256, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "expert_parallel_size": 32, - "routed_tensor_parallel_size": 1, - "shared_tensor_parallel_size": 1, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 27648, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "expert_parallel_size": 8, - "routed_tensor_parallel_size": 1, - "shared_tensor_parallel_size": 1, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 9, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "expert_parallel_size": 8, - "routed_tensor_parallel_size": 1, - "shared_tensor_parallel_size": 1, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 1332, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "expert_parallel_size": 16, - "routed_tensor_parallel_size": 1, - "shared_tensor_parallel_size": 1, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 666, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "expert_parallel_size": 16, - "routed_tensor_parallel_size": 1, - "shared_tensor_parallel_size": 1, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 666, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "expert_parallel_size": 32, - "routed_tensor_parallel_size": 1, - "shared_tensor_parallel_size": 1, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 180, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "expert_parallel_size": 32, - "routed_tensor_parallel_size": 1, - "shared_tensor_parallel_size": 1, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 10, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "expert_parallel_size": 8, - "routed_tensor_parallel_size": 1, - "shared_tensor_parallel_size": 1, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 1229, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "expert_parallel_size": 16, - "routed_tensor_parallel_size": 1, - "shared_tensor_parallel_size": 1, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 615, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "expert_parallel_size": 16, - "routed_tensor_parallel_size": 1, - "shared_tensor_parallel_size": 1, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 333, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "expert_parallel_size": 32, - "routed_tensor_parallel_size": 1, - "shared_tensor_parallel_size": 1, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 2, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "expert_parallel_size": 8, - "routed_tensor_parallel_size": 1, - "shared_tensor_parallel_size": 1, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 4096, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "expert_parallel_size": 1, - "routed_tensor_parallel_size": 4, - "shared_tensor_parallel_size": 4, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 1024, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "expert_parallel_size": 48, - "routed_tensor_parallel_size": 1, - "shared_tensor_parallel_size": 1, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 2048, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "expert_parallel_size": 48, - "routed_tensor_parallel_size": 1, - "shared_tensor_parallel_size": 1, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 4096, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "expert_parallel_size": 48, - "routed_tensor_parallel_size": 1, - "shared_tensor_parallel_size": 1, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 1024, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "expert_parallel_size": 32, - "routed_tensor_parallel_size": 1, - "shared_tensor_parallel_size": 1, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 2048, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "expert_parallel_size": 32, - "routed_tensor_parallel_size": 1, - "shared_tensor_parallel_size": 1, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 4096, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "expert_parallel_size": 32, - "routed_tensor_parallel_size": 1, - "shared_tensor_parallel_size": 1, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 6144, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "expert_parallel_size": 32, - "routed_tensor_parallel_size": 1, - "shared_tensor_parallel_size": 1, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 512, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "expert_parallel_size": 32, - "routed_tensor_parallel_size": 1, - "shared_tensor_parallel_size": 1, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 2048, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "expert_parallel_size": 24, - "routed_tensor_parallel_size": 1, - "shared_tensor_parallel_size": 1, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 4096, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "expert_parallel_size": 24, - "routed_tensor_parallel_size": 1, - "shared_tensor_parallel_size": 1, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 6144, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "expert_parallel_size": 24, - "routed_tensor_parallel_size": 1, - "shared_tensor_parallel_size": 1, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 128, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "expert_parallel_size": 32, - "routed_tensor_parallel_size": 1, - "shared_tensor_parallel_size": 1, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 512, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "expert_parallel_size": 32, - "routed_tensor_parallel_size": 1, - "shared_tensor_parallel_size": 1, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 2048, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "expert_parallel_size": 32, - "routed_tensor_parallel_size": 1, - "shared_tensor_parallel_size": 1, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 4096, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "expert_parallel_size": 32, - "routed_tensor_parallel_size": 1, - "shared_tensor_parallel_size": 1, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 8192, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "expert_parallel_size": 32, - "routed_tensor_parallel_size": 1, - "shared_tensor_parallel_size": 1, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 2048, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "expert_parallel_size": 48, - "routed_tensor_parallel_size": 1, - "shared_tensor_parallel_size": 1, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 4096, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "expert_parallel_size": 48, - "routed_tensor_parallel_size": 1, - "shared_tensor_parallel_size": 1, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 512, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "expert_parallel_size": 48, - "routed_tensor_parallel_size": 1, - "shared_tensor_parallel_size": 1, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 6452, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "expert_parallel_size": 4, - "routed_tensor_parallel_size": 1, - "shared_tensor_parallel_size": 1, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 8192, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "expert_parallel_size": 2, - "routed_tensor_parallel_size": 1, - "shared_tensor_parallel_size": 1, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 12288, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "expert_parallel_size": 2, - "routed_tensor_parallel_size": 1, - "shared_tensor_parallel_size": 1, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 24576, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "expert_parallel_size": 2, - "routed_tensor_parallel_size": 1, - "shared_tensor_parallel_size": 1, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 4506, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "expert_parallel_size": 16, - "routed_tensor_parallel_size": 1, - "shared_tensor_parallel_size": 1, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 2458, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "expert_parallel_size": 32, - "routed_tensor_parallel_size": 1, - "shared_tensor_parallel_size": 1, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 616, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "expert_parallel_size": 32, - "routed_tensor_parallel_size": 1, - "shared_tensor_parallel_size": 1, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 4506, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "expert_parallel_size": 8, - "routed_tensor_parallel_size": 1, - "shared_tensor_parallel_size": 1, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 1332, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "expert_parallel_size": 16, - "routed_tensor_parallel_size": 1, - "shared_tensor_parallel_size": 1, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 2254, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "expert_parallel_size": 16, - "routed_tensor_parallel_size": 1, - "shared_tensor_parallel_size": 1, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 6, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "expert_parallel_size": 4, - "routed_tensor_parallel_size": 1, - "shared_tensor_parallel_size": 1, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 1229, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "expert_parallel_size": 16, - "routed_tensor_parallel_size": 1, - "shared_tensor_parallel_size": 1, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 1076, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "expert_parallel_size": 4, - "routed_tensor_parallel_size": 1, - "shared_tensor_parallel_size": 1, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 8192, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "expert_parallel_size": 48, - "routed_tensor_parallel_size": 1, - "shared_tensor_parallel_size": 1, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 8192, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "expert_parallel_size": 4, - "routed_tensor_parallel_size": 1, - "shared_tensor_parallel_size": 1, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 24576, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "expert_parallel_size": 4, - "routed_tensor_parallel_size": 1, - "shared_tensor_parallel_size": 1, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 32768, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "expert_parallel_size": 4, - "routed_tensor_parallel_size": 1, - "shared_tensor_parallel_size": 1, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 360, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "expert_parallel_size": 32, - "routed_tensor_parallel_size": 1, - "shared_tensor_parallel_size": 1, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 1128, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "expert_parallel_size": 32, - "routed_tensor_parallel_size": 1, - "shared_tensor_parallel_size": 1, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 4506, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "expert_parallel_size": 16, - "routed_tensor_parallel_size": 1, - "shared_tensor_parallel_size": 1, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 4096, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "expert_parallel_size": 4, - "routed_tensor_parallel_size": 1, - "shared_tensor_parallel_size": 1, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 1229, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "expert_parallel_size": 32, - "routed_tensor_parallel_size": 1, - "shared_tensor_parallel_size": 1, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 2253, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "expert_parallel_size": 16, - "routed_tensor_parallel_size": 1, - "shared_tensor_parallel_size": 1, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 4301, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "expert_parallel_size": 8, - "routed_tensor_parallel_size": 1, - "shared_tensor_parallel_size": 1, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 6144, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "expert_parallel_size": 8, - "routed_tensor_parallel_size": 1, - "shared_tensor_parallel_size": 1, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 2458, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "expert_parallel_size": 16, - "routed_tensor_parallel_size": 1, - "shared_tensor_parallel_size": 1, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 2458, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "expert_parallel_size": 8, - "routed_tensor_parallel_size": 1, - "shared_tensor_parallel_size": 1, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 2151, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "expert_parallel_size": 8, - "routed_tensor_parallel_size": 1, - "shared_tensor_parallel_size": 1, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 1024, - "hidden": 2880, - "intermediate": 2880, - "num_experts": 128, - "top_k": 4, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "expert_parallel_size": 1, - "routed_tensor_parallel_size": 1, - "shared_tensor_parallel_size": 1, - "n_shared_experts": 0 - }, - "name": "gptoss" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 2048, - "hidden": 2880, - "intermediate": 2880, - "num_experts": 128, - "top_k": 4, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "expert_parallel_size": 1, - "routed_tensor_parallel_size": 1, - "shared_tensor_parallel_size": 1, - "n_shared_experts": 0 - }, - "name": "gptoss" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 1, - "hidden": 2880, - "intermediate": 2880, - "num_experts": 128, - "top_k": 4, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "expert_parallel_size": 1, - "routed_tensor_parallel_size": 4, - "shared_tensor_parallel_size": 4, - "n_shared_experts": 0 - }, - "name": "gptoss" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 2, - "hidden": 2880, - "intermediate": 2880, - "num_experts": 128, - "top_k": 4, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "expert_parallel_size": 1, - "routed_tensor_parallel_size": 4, - "shared_tensor_parallel_size": 4, - "n_shared_experts": 0 - }, - "name": "gptoss" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 128, - "hidden": 2880, - "intermediate": 2880, - "num_experts": 128, - "top_k": 4, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "expert_parallel_size": 1, - "routed_tensor_parallel_size": 4, - "shared_tensor_parallel_size": 4, - "n_shared_experts": 0 - }, - "name": "gptoss" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 512, - "hidden": 2880, - "intermediate": 2880, - "num_experts": 128, - "top_k": 4, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "expert_parallel_size": 2, - "routed_tensor_parallel_size": 1, - "shared_tensor_parallel_size": 1, - "n_shared_experts": 0 - }, - "name": "gptoss" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 1024, - "hidden": 2880, - "intermediate": 2880, - "num_experts": 128, - "top_k": 4, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "expert_parallel_size": 2, - "routed_tensor_parallel_size": 1, - "shared_tensor_parallel_size": 1, - "n_shared_experts": 0 - }, - "name": "gptoss" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 2048, - "hidden": 2880, - "intermediate": 2880, - "num_experts": 128, - "top_k": 4, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "expert_parallel_size": 2, - "routed_tensor_parallel_size": 1, - "shared_tensor_parallel_size": 1, - "n_shared_experts": 0 - }, - "name": "gptoss" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 2560, - "hidden": 2880, - "intermediate": 2880, - "num_experts": 128, - "top_k": 4, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "expert_parallel_size": 2, - "routed_tensor_parallel_size": 1, - "shared_tensor_parallel_size": 1, - "n_shared_experts": 0 - }, - "name": "gptoss" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 256, - "hidden": 2880, - "intermediate": 2880, - "num_experts": 128, - "top_k": 4, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "expert_parallel_size": 4, - "routed_tensor_parallel_size": 1, - "shared_tensor_parallel_size": 1, - "n_shared_experts": 0 - }, - "name": "gptoss" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 1024, - "hidden": 2880, - "intermediate": 2880, - "num_experts": 128, - "top_k": 4, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "expert_parallel_size": 4, - "routed_tensor_parallel_size": 1, - "shared_tensor_parallel_size": 1, - "n_shared_experts": 0 - }, - "name": "gptoss" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 1536, - "hidden": 2880, - "intermediate": 2880, - "num_experts": 128, - "top_k": 4, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "expert_parallel_size": 4, - "routed_tensor_parallel_size": 1, - "shared_tensor_parallel_size": 1, - "n_shared_experts": 0 - }, - "name": "gptoss" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 1, - "hidden": 2880, - "intermediate": 2880, - "num_experts": 128, - "top_k": 4, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "expert_parallel_size": 1, - "routed_tensor_parallel_size": 8, - "shared_tensor_parallel_size": 8, - "n_shared_experts": 0 - }, - "name": "gptoss" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 512, - "hidden": 2880, - "intermediate": 2880, - "num_experts": 128, - "top_k": 4, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "expert_parallel_size": 1, - "routed_tensor_parallel_size": 4, - "shared_tensor_parallel_size": 4, - "n_shared_experts": 0 - }, - "name": "gptoss" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 384, - "hidden": 2880, - "intermediate": 2880, - "num_experts": 128, - "top_k": 4, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "expert_parallel_size": 1, - "routed_tensor_parallel_size": 2, - "shared_tensor_parallel_size": 2, - "n_shared_experts": 0 - }, - "name": "gptoss" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 128, - "hidden": 2880, - "intermediate": 2880, - "num_experts": 128, - "top_k": 4, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "expert_parallel_size": 2, - "routed_tensor_parallel_size": 1, - "shared_tensor_parallel_size": 1, - "n_shared_experts": 0 - }, - "name": "gptoss" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 25, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "expert_parallel_size": 8, - "routed_tensor_parallel_size": 1, - "shared_tensor_parallel_size": 1, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 42, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "expert_parallel_size": 8, - "routed_tensor_parallel_size": 1, - "shared_tensor_parallel_size": 1, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 1, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "expert_parallel_size": 8, - "routed_tensor_parallel_size": 1, - "shared_tensor_parallel_size": 1, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 16, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "expert_parallel_size": 1, - "routed_tensor_parallel_size": 8, - "shared_tensor_parallel_size": 8, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 64, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "expert_parallel_size": 1, - "routed_tensor_parallel_size": 8, - "shared_tensor_parallel_size": 8, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 1024, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "expert_parallel_size": 8, - "routed_tensor_parallel_size": 1, - "shared_tensor_parallel_size": 1, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 144, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "expert_parallel_size": 8, - "routed_tensor_parallel_size": 1, - "shared_tensor_parallel_size": 1, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 160, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "expert_parallel_size": 8, - "routed_tensor_parallel_size": 1, - "shared_tensor_parallel_size": 1, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 320, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "expert_parallel_size": 8, - "routed_tensor_parallel_size": 1, - "shared_tensor_parallel_size": 1, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 2048, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "expert_parallel_size": 8, - "routed_tensor_parallel_size": 1, - "shared_tensor_parallel_size": 1, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 204, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "expert_parallel_size": 8, - "routed_tensor_parallel_size": 1, - "shared_tensor_parallel_size": 1, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 85, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "expert_parallel_size": 8, - "routed_tensor_parallel_size": 1, - "shared_tensor_parallel_size": 1, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 32, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "expert_parallel_size": 1, - "routed_tensor_parallel_size": 8, - "shared_tensor_parallel_size": 8, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 128, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "expert_parallel_size": 1, - "routed_tensor_parallel_size": 8, - "shared_tensor_parallel_size": 8, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 4096, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 384, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "expert_parallel_size": 4, - "routed_tensor_parallel_size": 1, - "shared_tensor_parallel_size": 1, - "n_shared_experts": 1 - }, - "name": "kimik2.5" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 32768, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 384, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "expert_parallel_size": 4, - "routed_tensor_parallel_size": 1, - "shared_tensor_parallel_size": 1, - "n_shared_experts": 1 - }, - "name": "kimik2.5" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 1, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 384, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "expert_parallel_size": 8, - "routed_tensor_parallel_size": 1, - "shared_tensor_parallel_size": 1, - "n_shared_experts": 1 - }, - "name": "kimik2.5" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 48, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 384, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "expert_parallel_size": 8, - "routed_tensor_parallel_size": 1, - "shared_tensor_parallel_size": 1, - "n_shared_experts": 1 - }, - "name": "kimik2.5" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 90, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 384, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "expert_parallel_size": 8, - "routed_tensor_parallel_size": 1, - "shared_tensor_parallel_size": 1, - "n_shared_experts": 1 - }, - "name": "kimik2.5" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 167, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 384, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "expert_parallel_size": 8, - "routed_tensor_parallel_size": 1, - "shared_tensor_parallel_size": 1, - "n_shared_experts": 1 - }, - "name": "kimik2.5" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 5120, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 384, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "expert_parallel_size": 4, - "routed_tensor_parallel_size": 1, - "shared_tensor_parallel_size": 1, - "n_shared_experts": 1 - }, - "name": "kimik2.5" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 15360, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 384, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "expert_parallel_size": 4, - "routed_tensor_parallel_size": 1, - "shared_tensor_parallel_size": 1, - "n_shared_experts": 1 - }, - "name": "kimik2.5" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 30720, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 384, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "expert_parallel_size": 4, - "routed_tensor_parallel_size": 1, - "shared_tensor_parallel_size": 1, - "n_shared_experts": 1 - }, - "name": "kimik2.5" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 1, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 384, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "expert_parallel_size": 4, - "routed_tensor_parallel_size": 1, - "shared_tensor_parallel_size": 1, - "n_shared_experts": 1 - }, - "name": "kimik2.5" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 3, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 384, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "expert_parallel_size": 4, - "routed_tensor_parallel_size": 1, - "shared_tensor_parallel_size": 1, - "n_shared_experts": 1 - }, - "name": "kimik2.5" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 6, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 384, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "expert_parallel_size": 4, - "routed_tensor_parallel_size": 1, - "shared_tensor_parallel_size": 1, - "n_shared_experts": 1 - }, - "name": "kimik2.5" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 11, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 384, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "expert_parallel_size": 4, - "routed_tensor_parallel_size": 1, - "shared_tensor_parallel_size": 1, - "n_shared_experts": 1 - }, - "name": "kimik2.5" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 666, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 384, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "expert_parallel_size": 16, - "routed_tensor_parallel_size": 1, - "shared_tensor_parallel_size": 1, - "n_shared_experts": 1 - }, - "name": "kimik2.5" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 2253, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 384, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "expert_parallel_size": 32, - "routed_tensor_parallel_size": 1, - "shared_tensor_parallel_size": 1, - "n_shared_experts": 1 - }, - "name": "kimik2.5" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 4301, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 384, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "expert_parallel_size": 8, - "routed_tensor_parallel_size": 1, - "shared_tensor_parallel_size": 1, - "n_shared_experts": 1 - }, - "name": "kimik2.5" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 6452, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 384, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "expert_parallel_size": 8, - "routed_tensor_parallel_size": 1, - "shared_tensor_parallel_size": 1, - "n_shared_experts": 1 - }, - "name": "kimik2.5" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 4301, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 384, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "expert_parallel_size": 16, - "routed_tensor_parallel_size": 1, - "shared_tensor_parallel_size": 1, - "n_shared_experts": 1 - }, - "name": "kimik2.5" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 4301, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 384, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "expert_parallel_size": 32, - "routed_tensor_parallel_size": 1, - "shared_tensor_parallel_size": 1, - "n_shared_experts": 1 - }, - "name": "kimik2.5" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 39, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 384, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "expert_parallel_size": 4, - "routed_tensor_parallel_size": 1, - "shared_tensor_parallel_size": 1, - "n_shared_experts": 1 - }, - "name": "kimik2.5" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 12, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 384, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "expert_parallel_size": 4, - "routed_tensor_parallel_size": 1, - "shared_tensor_parallel_size": 1, - "n_shared_experts": 1 - }, - "name": "kimik2.5" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 21, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 384, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "expert_parallel_size": 4, - "routed_tensor_parallel_size": 1, - "shared_tensor_parallel_size": 1, - "n_shared_experts": 1 - }, - "name": "kimik2.5" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 333, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 384, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "expert_parallel_size": 16, - "routed_tensor_parallel_size": 1, - "shared_tensor_parallel_size": 1, - "n_shared_experts": 1 - }, - "name": "kimik2.5" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 615, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 384, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "expert_parallel_size": 16, - "routed_tensor_parallel_size": 1, - "shared_tensor_parallel_size": 1, - "n_shared_experts": 1 - }, - "name": "kimik2.5" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 2151, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 384, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "expert_parallel_size": 8, - "routed_tensor_parallel_size": 1, - "shared_tensor_parallel_size": 1, - "n_shared_experts": 1 - }, - "name": "kimik2.5" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 2253, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 384, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "expert_parallel_size": 16, - "routed_tensor_parallel_size": 1, - "shared_tensor_parallel_size": 1, - "n_shared_experts": 1 - }, - "name": "kimik2.5" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 256, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 384, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "expert_parallel_size": 16, - "routed_tensor_parallel_size": 1, - "shared_tensor_parallel_size": 1, - "n_shared_experts": 1 - }, - "name": "kimik2.5" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 512, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 384, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "expert_parallel_size": 16, - "routed_tensor_parallel_size": 1, - "shared_tensor_parallel_size": 1, - "n_shared_experts": 1 - }, - "name": "kimik2.5" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 1024, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 384, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "expert_parallel_size": 16, - "routed_tensor_parallel_size": 1, - "shared_tensor_parallel_size": 1, - "n_shared_experts": 1 - }, - "name": "kimik2.5" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 2048, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 384, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "expert_parallel_size": 16, - "routed_tensor_parallel_size": 1, - "shared_tensor_parallel_size": 1, - "n_shared_experts": 1 - }, - "name": "kimik2.5" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 3072, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 384, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "expert_parallel_size": 16, - "routed_tensor_parallel_size": 1, - "shared_tensor_parallel_size": 1, - "n_shared_experts": 1 - }, - "name": "kimik2.5" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 4096, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 384, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "expert_parallel_size": 16, - "routed_tensor_parallel_size": 1, - "shared_tensor_parallel_size": 1, - "n_shared_experts": 1 - }, - "name": "kimik2.5" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 8192, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 384, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "expert_parallel_size": 4, - "routed_tensor_parallel_size": 1, - "shared_tensor_parallel_size": 1, - "n_shared_experts": 1 - }, - "name": "kimik2.5" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 16384, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 384, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "expert_parallel_size": 4, - "routed_tensor_parallel_size": 1, - "shared_tensor_parallel_size": 1, - "n_shared_experts": 1 - }, - "name": "kimik2.5" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 2, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 384, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "expert_parallel_size": 4, - "routed_tensor_parallel_size": 1, - "shared_tensor_parallel_size": 1, - "n_shared_experts": 1 - }, - "name": "kimik2.5" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 4, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 384, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "expert_parallel_size": 4, - "routed_tensor_parallel_size": 1, - "shared_tensor_parallel_size": 1, - "n_shared_experts": 1 - }, - "name": "kimik2.5" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 8, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 384, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "expert_parallel_size": 4, - "routed_tensor_parallel_size": 1, - "shared_tensor_parallel_size": 1, - "n_shared_experts": 1 - }, - "name": "kimik2.5" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 16, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 384, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "expert_parallel_size": 4, - "routed_tensor_parallel_size": 1, - "shared_tensor_parallel_size": 1, - "n_shared_experts": 1 - }, - "name": "kimik2.5" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 32, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 384, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "expert_parallel_size": 4, - "routed_tensor_parallel_size": 1, - "shared_tensor_parallel_size": 1, - "n_shared_experts": 1 - }, - "name": "kimik2.5" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 2048, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 384, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "expert_parallel_size": 8, - "routed_tensor_parallel_size": 1, - "shared_tensor_parallel_size": 1, - "n_shared_experts": 1 - }, - "name": "kimik2.5" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 1536, - "hidden": 7168, - "intermediate": 3072, - "num_experts": 384, - "top_k": 6, - "dtype_act": "nvfp4", - "dtype_weight": "mxfp4", - "expert_parallel_size": 16, - "routed_tensor_parallel_size": 1, - "shared_tensor_parallel_size": 1, - "n_shared_experts": 1 - }, - "name": "dsv4" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 1024, - "hidden": 7168, - "intermediate": 3072, - "num_experts": 384, - "top_k": 6, - "dtype_act": "nvfp4", - "dtype_weight": "mxfp4", - "expert_parallel_size": 16, - "routed_tensor_parallel_size": 1, - "shared_tensor_parallel_size": 1, - "n_shared_experts": 1 - }, - "name": "dsv4" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 4096, - "hidden": 7168, - "intermediate": 3072, - "num_experts": 384, - "top_k": 6, - "dtype_act": "nvfp4", - "dtype_weight": "mxfp4", - "expert_parallel_size": 16, - "routed_tensor_parallel_size": 1, - "shared_tensor_parallel_size": 1, - "n_shared_experts": 1 - }, - "name": "dsv4" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 8192, - "hidden": 7168, - "intermediate": 3072, - "num_experts": 384, - "top_k": 6, - "dtype_act": "nvfp4", - "dtype_weight": "mxfp4", - "expert_parallel_size": 16, - "routed_tensor_parallel_size": 1, - "shared_tensor_parallel_size": 1, - "n_shared_experts": 1 - }, - "name": "dsv4" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 21504, - "hidden": 7168, - "intermediate": 3072, - "num_experts": 384, - "top_k": 6, - "dtype_act": "nvfp4", - "dtype_weight": "mxfp4", - "expert_parallel_size": 12, - "routed_tensor_parallel_size": 1, - "shared_tensor_parallel_size": 1, - "n_shared_experts": 1 - }, - "name": "dsv4" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 21, - "hidden": 7168, - "intermediate": 3072, - "num_experts": 384, - "top_k": 6, - "dtype_act": "nvfp4", - "dtype_weight": "mxfp4", - "expert_parallel_size": 1, - "routed_tensor_parallel_size": 4, - "shared_tensor_parallel_size": 1, - "n_shared_experts": 1 - }, - "name": "dsv4" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 512, - "hidden": 7168, - "intermediate": 3072, - "num_experts": 384, - "top_k": 6, - "dtype_act": "nvfp4", - "dtype_weight": "mxfp4", - "expert_parallel_size": 16, - "routed_tensor_parallel_size": 1, - "shared_tensor_parallel_size": 1, - "n_shared_experts": 1 - }, - "name": "dsv4" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 4096, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "mxfp4", - "expert_parallel_size": 1, - "routed_tensor_parallel_size": 4, - "shared_tensor_parallel_size": 4, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 4, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "mxfp4", - "expert_parallel_size": 1, - "routed_tensor_parallel_size": 4, - "shared_tensor_parallel_size": 4, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 8192, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "mxfp4", - "expert_parallel_size": 1, - "routed_tensor_parallel_size": 4, - "shared_tensor_parallel_size": 4, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 8, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "mxfp4", - "expert_parallel_size": 1, - "routed_tensor_parallel_size": 4, - "shared_tensor_parallel_size": 4, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 16384, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "mxfp4", - "expert_parallel_size": 1, - "routed_tensor_parallel_size": 4, - "shared_tensor_parallel_size": 4, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 16, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "mxfp4", - "expert_parallel_size": 1, - "routed_tensor_parallel_size": 4, - "shared_tensor_parallel_size": 4, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 32768, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "mxfp4", - "expert_parallel_size": 1, - "routed_tensor_parallel_size": 4, - "shared_tensor_parallel_size": 4, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 32, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "mxfp4", - "expert_parallel_size": 1, - "routed_tensor_parallel_size": 4, - "shared_tensor_parallel_size": 4, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 64, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "mxfp4", - "expert_parallel_size": 1, - "routed_tensor_parallel_size": 4, - "shared_tensor_parallel_size": 4, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 4096, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "mxfp4", - "expert_parallel_size": 1, - "routed_tensor_parallel_size": 8, - "shared_tensor_parallel_size": 8, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 4, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "mxfp4", - "expert_parallel_size": 1, - "routed_tensor_parallel_size": 8, - "shared_tensor_parallel_size": 8, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 8192, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "mxfp4", - "expert_parallel_size": 1, - "routed_tensor_parallel_size": 8, - "shared_tensor_parallel_size": 8, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 8, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "mxfp4", - "expert_parallel_size": 1, - "routed_tensor_parallel_size": 8, - "shared_tensor_parallel_size": 8, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 16384, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "mxfp4", - "expert_parallel_size": 1, - "routed_tensor_parallel_size": 8, - "shared_tensor_parallel_size": 8, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 16, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "mxfp4", - "expert_parallel_size": 1, - "routed_tensor_parallel_size": 8, - "shared_tensor_parallel_size": 8, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 32768, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "mxfp4", - "expert_parallel_size": 1, - "routed_tensor_parallel_size": 8, - "shared_tensor_parallel_size": 8, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 32, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "mxfp4", - "expert_parallel_size": 1, - "routed_tensor_parallel_size": 8, - "shared_tensor_parallel_size": 8, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 64, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "mxfp4", - "expert_parallel_size": 1, - "routed_tensor_parallel_size": 8, - "shared_tensor_parallel_size": 8, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 1, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "mxfp4", - "expert_parallel_size": 1, - "routed_tensor_parallel_size": 8, - "shared_tensor_parallel_size": 8, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 2, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "mxfp4", - "expert_parallel_size": 1, - "routed_tensor_parallel_size": 8, - "shared_tensor_parallel_size": 8, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 12, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "mxfp4", - "expert_parallel_size": 1, - "routed_tensor_parallel_size": 8, - "shared_tensor_parallel_size": 8, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 128, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "mxfp4", - "expert_parallel_size": 1, - "routed_tensor_parallel_size": 8, - "shared_tensor_parallel_size": 8, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 256, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "mxfp4", - "expert_parallel_size": 1, - "routed_tensor_parallel_size": 8, - "shared_tensor_parallel_size": 8, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 128, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "mxfp4", - "expert_parallel_size": 1, - "routed_tensor_parallel_size": 4, - "shared_tensor_parallel_size": 4, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 256, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "mxfp4", - "expert_parallel_size": 1, - "routed_tensor_parallel_size": 4, - "shared_tensor_parallel_size": 4, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 512, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "mxfp4", - "expert_parallel_size": 1, - "routed_tensor_parallel_size": 4, - "shared_tensor_parallel_size": 4, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 512, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "mxfp4", - "expert_parallel_size": 1, - "routed_tensor_parallel_size": 8, - "shared_tensor_parallel_size": 8, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 64, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "expert_parallel_size": 1, - "routed_tensor_parallel_size": 4, - "shared_tensor_parallel_size": 4, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 256, - "hidden": 4096, - "intermediate": 1024, - "num_experts": 512, - "top_k": 10, - "dtype_act": "bf16", - "dtype_weight": "bf16", - "expert_parallel_size": 1, - "routed_tensor_parallel_size": 8, - "shared_tensor_parallel_size": 8, - "n_shared_experts": 1 - }, - "name": "qwen3.5" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 512, - "hidden": 4096, - "intermediate": 1024, - "num_experts": 512, - "top_k": 10, - "dtype_act": "bf16", - "dtype_weight": "bf16", - "expert_parallel_size": 1, - "routed_tensor_parallel_size": 8, - "shared_tensor_parallel_size": 8, - "n_shared_experts": 1 - }, - "name": "qwen3.5" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 8192, - "hidden": 4096, - "intermediate": 1024, - "num_experts": 512, - "top_k": 10, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "expert_parallel_size": 1, - "routed_tensor_parallel_size": 8, - "shared_tensor_parallel_size": 8, - "n_shared_experts": 1 - }, - "name": "qwen3.5" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 16384, - "hidden": 4096, - "intermediate": 1024, - "num_experts": 512, - "top_k": 10, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "expert_parallel_size": 1, - "routed_tensor_parallel_size": 8, - "shared_tensor_parallel_size": 8, - "n_shared_experts": 1 - }, - "name": "qwen3.5" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 16, - "hidden": 4096, - "intermediate": 1024, - "num_experts": 512, - "top_k": 10, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "expert_parallel_size": 1, - "routed_tensor_parallel_size": 8, - "shared_tensor_parallel_size": 8, - "n_shared_experts": 1 - }, - "name": "qwen3.5" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 32, - "hidden": 4096, - "intermediate": 1024, - "num_experts": 512, - "top_k": 10, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "expert_parallel_size": 1, - "routed_tensor_parallel_size": 8, - "shared_tensor_parallel_size": 8, - "n_shared_experts": 1 - }, - "name": "qwen3.5" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 64, - "hidden": 4096, - "intermediate": 1024, - "num_experts": 512, - "top_k": 10, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "expert_parallel_size": 1, - "routed_tensor_parallel_size": 8, - "shared_tensor_parallel_size": 8, - "n_shared_experts": 1 - }, - "name": "qwen3.5" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 32768, - "hidden": 4096, - "intermediate": 1024, - "num_experts": 512, - "top_k": 10, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "expert_parallel_size": 2, - "routed_tensor_parallel_size": 1, - "shared_tensor_parallel_size": 1, - "n_shared_experts": 1 - }, - "name": "qwen3.5" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 128, - "hidden": 4096, - "intermediate": 1024, - "num_experts": 512, - "top_k": 10, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "expert_parallel_size": 2, - "routed_tensor_parallel_size": 1, - "shared_tensor_parallel_size": 1, - "n_shared_experts": 1 - }, - "name": "qwen3.5" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 256, - "hidden": 4096, - "intermediate": 1024, - "num_experts": 512, - "top_k": 10, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "expert_parallel_size": 2, - "routed_tensor_parallel_size": 1, - "shared_tensor_parallel_size": 1, - "n_shared_experts": 1 - }, - "name": "qwen3.5" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 4, - "hidden": 4096, - "intermediate": 1024, - "num_experts": 512, - "top_k": 10, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "expert_parallel_size": 2, - "routed_tensor_parallel_size": 1, - "shared_tensor_parallel_size": 1, - "n_shared_experts": 1 - }, - "name": "qwen3.5" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 8, - "hidden": 4096, - "intermediate": 1024, - "num_experts": 512, - "top_k": 10, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "expert_parallel_size": 2, - "routed_tensor_parallel_size": 1, - "shared_tensor_parallel_size": 1, - "n_shared_experts": 1 - }, - "name": "qwen3.5" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 16, - "hidden": 4096, - "intermediate": 1024, - "num_experts": 512, - "top_k": 10, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "expert_parallel_size": 2, - "routed_tensor_parallel_size": 1, - "shared_tensor_parallel_size": 1, - "n_shared_experts": 1 - }, - "name": "qwen3.5" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 32, - "hidden": 4096, - "intermediate": 1024, - "num_experts": 512, - "top_k": 10, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "expert_parallel_size": 2, - "routed_tensor_parallel_size": 1, - "shared_tensor_parallel_size": 1, - "n_shared_experts": 1 - }, - "name": "qwen3.5" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 512, - "hidden": 4096, - "intermediate": 1024, - "num_experts": 512, - "top_k": 10, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "expert_parallel_size": 8, - "routed_tensor_parallel_size": 1, - "shared_tensor_parallel_size": 1, - "n_shared_experts": 1 - }, - "name": "qwen3.5" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 512, - "hidden": 4096, - "intermediate": 1024, - "num_experts": 512, - "top_k": 10, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "expert_parallel_size": 2, - "routed_tensor_parallel_size": 1, - "shared_tensor_parallel_size": 1, - "n_shared_experts": 1 - }, - "name": "qwen3.5" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 64, - "hidden": 4096, - "intermediate": 1024, - "num_experts": 512, - "top_k": 10, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "expert_parallel_size": 2, - "routed_tensor_parallel_size": 1, - "shared_tensor_parallel_size": 1, - "n_shared_experts": 1 - }, - "name": "qwen3.5" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 4096, - "hidden": 4096, - "intermediate": 1024, - "num_experts": 512, - "top_k": 10, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "expert_parallel_size": 1, - "routed_tensor_parallel_size": 2, - "shared_tensor_parallel_size": 2, - "n_shared_experts": 1 - }, - "name": "qwen3.5" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 4, - "hidden": 4096, - "intermediate": 1024, - "num_experts": 512, - "top_k": 10, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "expert_parallel_size": 1, - "routed_tensor_parallel_size": 2, - "shared_tensor_parallel_size": 2, - "n_shared_experts": 1 - }, - "name": "qwen3.5" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 8192, - "hidden": 4096, - "intermediate": 1024, - "num_experts": 512, - "top_k": 10, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "expert_parallel_size": 1, - "routed_tensor_parallel_size": 2, - "shared_tensor_parallel_size": 2, - "n_shared_experts": 1 - }, - "name": "qwen3.5" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 8, - "hidden": 4096, - "intermediate": 1024, - "num_experts": 512, - "top_k": 10, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "expert_parallel_size": 1, - "routed_tensor_parallel_size": 2, - "shared_tensor_parallel_size": 2, - "n_shared_experts": 1 - }, - "name": "qwen3.5" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 16384, - "hidden": 4096, - "intermediate": 1024, - "num_experts": 512, - "top_k": 10, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "expert_parallel_size": 1, - "routed_tensor_parallel_size": 2, - "shared_tensor_parallel_size": 2, - "n_shared_experts": 1 - }, - "name": "qwen3.5" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 16, - "hidden": 4096, - "intermediate": 1024, - "num_experts": 512, - "top_k": 10, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "expert_parallel_size": 1, - "routed_tensor_parallel_size": 2, - "shared_tensor_parallel_size": 2, - "n_shared_experts": 1 - }, - "name": "qwen3.5" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 32768, - "hidden": 4096, - "intermediate": 1024, - "num_experts": 512, - "top_k": 10, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "expert_parallel_size": 1, - "routed_tensor_parallel_size": 2, - "shared_tensor_parallel_size": 2, - "n_shared_experts": 1 - }, - "name": "qwen3.5" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 32, - "hidden": 4096, - "intermediate": 1024, - "num_experts": 512, - "top_k": 10, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "expert_parallel_size": 1, - "routed_tensor_parallel_size": 2, - "shared_tensor_parallel_size": 2, - "n_shared_experts": 1 - }, - "name": "qwen3.5" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 64, - "hidden": 4096, - "intermediate": 1024, - "num_experts": 512, - "top_k": 10, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "expert_parallel_size": 1, - "routed_tensor_parallel_size": 2, - "shared_tensor_parallel_size": 2, - "n_shared_experts": 1 - }, - "name": "qwen3.5" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 128, - "hidden": 4096, - "intermediate": 1024, - "num_experts": 512, - "top_k": 10, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "expert_parallel_size": 1, - "routed_tensor_parallel_size": 2, - "shared_tensor_parallel_size": 2, - "n_shared_experts": 1 - }, - "name": "qwen3.5" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 256, - "hidden": 4096, - "intermediate": 1024, - "num_experts": 512, - "top_k": 10, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "expert_parallel_size": 1, - "routed_tensor_parallel_size": 2, - "shared_tensor_parallel_size": 2, - "n_shared_experts": 1 - }, - "name": "qwen3.5" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 128, - "hidden": 4096, - "intermediate": 1024, - "num_experts": 512, - "top_k": 10, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "expert_parallel_size": 1, - "routed_tensor_parallel_size": 8, - "shared_tensor_parallel_size": 8, - "n_shared_experts": 1 - }, - "name": "qwen3.5" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 256, - "hidden": 4096, - "intermediate": 1024, - "num_experts": 512, - "top_k": 10, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "expert_parallel_size": 1, - "routed_tensor_parallel_size": 8, - "shared_tensor_parallel_size": 8, - "n_shared_experts": 1 - }, - "name": "qwen3.5" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 512, - "hidden": 4096, - "intermediate": 1024, - "num_experts": 512, - "top_k": 10, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "expert_parallel_size": 1, - "routed_tensor_parallel_size": 8, - "shared_tensor_parallel_size": 8, - "n_shared_experts": 1 - }, - "name": "qwen3.5" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 4096, - "hidden": 4096, - "intermediate": 1024, - "num_experts": 512, - "top_k": 10, - "dtype_act": "fp8", - "dtype_weight": "mxfp4", - "expert_parallel_size": 1, - "routed_tensor_parallel_size": 2, - "shared_tensor_parallel_size": 2, - "n_shared_experts": 1 - }, - "name": "qwen3.5" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 4, - "hidden": 4096, - "intermediate": 1024, - "num_experts": 512, - "top_k": 10, - "dtype_act": "fp8", - "dtype_weight": "mxfp4", - "expert_parallel_size": 1, - "routed_tensor_parallel_size": 2, - "shared_tensor_parallel_size": 2, - "n_shared_experts": 1 - }, - "name": "qwen3.5" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 8192, - "hidden": 4096, - "intermediate": 1024, - "num_experts": 512, - "top_k": 10, - "dtype_act": "fp8", - "dtype_weight": "mxfp4", - "expert_parallel_size": 1, - "routed_tensor_parallel_size": 2, - "shared_tensor_parallel_size": 2, - "n_shared_experts": 1 - }, - "name": "qwen3.5" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 8, - "hidden": 4096, - "intermediate": 1024, - "num_experts": 512, - "top_k": 10, - "dtype_act": "fp8", - "dtype_weight": "mxfp4", - "expert_parallel_size": 1, - "routed_tensor_parallel_size": 2, - "shared_tensor_parallel_size": 2, - "n_shared_experts": 1 - }, - "name": "qwen3.5" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 16384, - "hidden": 4096, - "intermediate": 1024, - "num_experts": 512, - "top_k": 10, - "dtype_act": "fp8", - "dtype_weight": "mxfp4", - "expert_parallel_size": 1, - "routed_tensor_parallel_size": 2, - "shared_tensor_parallel_size": 2, - "n_shared_experts": 1 - }, - "name": "qwen3.5" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 16, - "hidden": 4096, - "intermediate": 1024, - "num_experts": 512, - "top_k": 10, - "dtype_act": "fp8", - "dtype_weight": "mxfp4", - "expert_parallel_size": 1, - "routed_tensor_parallel_size": 2, - "shared_tensor_parallel_size": 2, - "n_shared_experts": 1 - }, - "name": "qwen3.5" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 32768, - "hidden": 4096, - "intermediate": 1024, - "num_experts": 512, - "top_k": 10, - "dtype_act": "fp8", - "dtype_weight": "mxfp4", - "expert_parallel_size": 1, - "routed_tensor_parallel_size": 2, - "shared_tensor_parallel_size": 2, - "n_shared_experts": 1 - }, - "name": "qwen3.5" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 32, - "hidden": 4096, - "intermediate": 1024, - "num_experts": 512, - "top_k": 10, - "dtype_act": "fp8", - "dtype_weight": "mxfp4", - "expert_parallel_size": 1, - "routed_tensor_parallel_size": 2, - "shared_tensor_parallel_size": 2, - "n_shared_experts": 1 - }, - "name": "qwen3.5" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 64, - "hidden": 4096, - "intermediate": 1024, - "num_experts": 512, - "top_k": 10, - "dtype_act": "fp8", - "dtype_weight": "mxfp4", - "expert_parallel_size": 1, - "routed_tensor_parallel_size": 2, - "shared_tensor_parallel_size": 2, - "n_shared_experts": 1 - }, - "name": "qwen3.5" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 128, - "hidden": 4096, - "intermediate": 1024, - "num_experts": 512, - "top_k": 10, - "dtype_act": "fp8", - "dtype_weight": "mxfp4", - "expert_parallel_size": 1, - "routed_tensor_parallel_size": 2, - "shared_tensor_parallel_size": 2, - "n_shared_experts": 1 - }, - "name": "qwen3.5" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 256, - "hidden": 4096, - "intermediate": 1024, - "num_experts": 512, - "top_k": 10, - "dtype_act": "fp8", - "dtype_weight": "mxfp4", - "expert_parallel_size": 1, - "routed_tensor_parallel_size": 2, - "shared_tensor_parallel_size": 2, - "n_shared_experts": 1 - }, - "name": "qwen3.5" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 4096, - "hidden": 4096, - "intermediate": 1024, - "num_experts": 512, - "top_k": 10, - "dtype_act": "fp8", - "dtype_weight": "mxfp4", - "expert_parallel_size": 1, - "routed_tensor_parallel_size": 4, - "shared_tensor_parallel_size": 4, - "n_shared_experts": 1 - }, - "name": "qwen3.5" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 4, - "hidden": 4096, - "intermediate": 1024, - "num_experts": 512, - "top_k": 10, - "dtype_act": "fp8", - "dtype_weight": "mxfp4", - "expert_parallel_size": 1, - "routed_tensor_parallel_size": 4, - "shared_tensor_parallel_size": 4, - "n_shared_experts": 1 - }, - "name": "qwen3.5" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 8192, - "hidden": 4096, - "intermediate": 1024, - "num_experts": 512, - "top_k": 10, - "dtype_act": "fp8", - "dtype_weight": "mxfp4", - "expert_parallel_size": 1, - "routed_tensor_parallel_size": 4, - "shared_tensor_parallel_size": 4, - "n_shared_experts": 1 - }, - "name": "qwen3.5" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 8, - "hidden": 4096, - "intermediate": 1024, - "num_experts": 512, - "top_k": 10, - "dtype_act": "fp8", - "dtype_weight": "mxfp4", - "expert_parallel_size": 1, - "routed_tensor_parallel_size": 4, - "shared_tensor_parallel_size": 4, - "n_shared_experts": 1 - }, - "name": "qwen3.5" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 16384, - "hidden": 4096, - "intermediate": 1024, - "num_experts": 512, - "top_k": 10, - "dtype_act": "fp8", - "dtype_weight": "mxfp4", - "expert_parallel_size": 1, - "routed_tensor_parallel_size": 4, - "shared_tensor_parallel_size": 4, - "n_shared_experts": 1 - }, - "name": "qwen3.5" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 16, - "hidden": 4096, - "intermediate": 1024, - "num_experts": 512, - "top_k": 10, - "dtype_act": "fp8", - "dtype_weight": "mxfp4", - "expert_parallel_size": 1, - "routed_tensor_parallel_size": 4, - "shared_tensor_parallel_size": 4, - "n_shared_experts": 1 - }, - "name": "qwen3.5" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 32768, - "hidden": 4096, - "intermediate": 1024, - "num_experts": 512, - "top_k": 10, - "dtype_act": "fp8", - "dtype_weight": "mxfp4", - "expert_parallel_size": 1, - "routed_tensor_parallel_size": 4, - "shared_tensor_parallel_size": 4, - "n_shared_experts": 1 - }, - "name": "qwen3.5" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 4096, - "hidden": 6144, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "expert_parallel_size": 1, - "routed_tensor_parallel_size": 4, - "shared_tensor_parallel_size": 4, - "n_shared_experts": 1 - }, - "name": "glm5" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 8, - "hidden": 6144, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "expert_parallel_size": 1, - "routed_tensor_parallel_size": 4, - "shared_tensor_parallel_size": 4, - "n_shared_experts": 1 - }, - "name": "glm5" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 8192, - "hidden": 6144, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "expert_parallel_size": 1, - "routed_tensor_parallel_size": 4, - "shared_tensor_parallel_size": 4, - "n_shared_experts": 1 - }, - "name": "glm5" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 16, - "hidden": 6144, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "expert_parallel_size": 1, - "routed_tensor_parallel_size": 4, - "shared_tensor_parallel_size": 4, - "n_shared_experts": 1 - }, - "name": "glm5" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 16384, - "hidden": 6144, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "expert_parallel_size": 1, - "routed_tensor_parallel_size": 4, - "shared_tensor_parallel_size": 4, - "n_shared_experts": 1 - }, - "name": "glm5" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 32, - "hidden": 6144, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "expert_parallel_size": 1, - "routed_tensor_parallel_size": 4, - "shared_tensor_parallel_size": 4, - "n_shared_experts": 1 - }, - "name": "glm5" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 32768, - "hidden": 6144, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "expert_parallel_size": 1, - "routed_tensor_parallel_size": 4, - "shared_tensor_parallel_size": 4, - "n_shared_experts": 1 - }, - "name": "glm5" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 64, - "hidden": 6144, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "expert_parallel_size": 1, - "routed_tensor_parallel_size": 4, - "shared_tensor_parallel_size": 4, - "n_shared_experts": 1 - }, - "name": "glm5" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 128, - "hidden": 6144, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "expert_parallel_size": 1, - "routed_tensor_parallel_size": 4, - "shared_tensor_parallel_size": 4, - "n_shared_experts": 1 - }, - "name": "glm5" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 256, - "hidden": 6144, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "expert_parallel_size": 1, - "routed_tensor_parallel_size": 4, - "shared_tensor_parallel_size": 4, - "n_shared_experts": 1 - }, - "name": "glm5" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 4, - "hidden": 6144, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "expert_parallel_size": 1, - "routed_tensor_parallel_size": 4, - "shared_tensor_parallel_size": 4, - "n_shared_experts": 1 - }, - "name": "glm5" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 4096, - "hidden": 6144, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "mxfp4", - "expert_parallel_size": 1, - "routed_tensor_parallel_size": 2, - "shared_tensor_parallel_size": 2, - "n_shared_experts": 1 - }, - "name": "glm5.1" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 4, - "hidden": 6144, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "mxfp4", - "expert_parallel_size": 1, - "routed_tensor_parallel_size": 2, - "shared_tensor_parallel_size": 2, - "n_shared_experts": 1 - }, - "name": "glm5.1" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 8192, - "hidden": 6144, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "mxfp4", - "expert_parallel_size": 1, - "routed_tensor_parallel_size": 2, - "shared_tensor_parallel_size": 2, - "n_shared_experts": 1 - }, - "name": "glm5.1" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 8, - "hidden": 6144, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "mxfp4", - "expert_parallel_size": 1, - "routed_tensor_parallel_size": 2, - "shared_tensor_parallel_size": 2, - "n_shared_experts": 1 - }, - "name": "glm5.1" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 16384, - "hidden": 6144, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "mxfp4", - "expert_parallel_size": 1, - "routed_tensor_parallel_size": 2, - "shared_tensor_parallel_size": 2, - "n_shared_experts": 1 - }, - "name": "glm5.1" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 16, - "hidden": 6144, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "mxfp4", - "expert_parallel_size": 1, - "routed_tensor_parallel_size": 2, - "shared_tensor_parallel_size": 2, - "n_shared_experts": 1 - }, - "name": "glm5.1" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 32768, - "hidden": 6144, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "mxfp4", - "expert_parallel_size": 1, - "routed_tensor_parallel_size": 2, - "shared_tensor_parallel_size": 2, - "n_shared_experts": 1 - }, - "name": "glm5.1" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 32, - "hidden": 6144, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "mxfp4", - "expert_parallel_size": 1, - "routed_tensor_parallel_size": 2, - "shared_tensor_parallel_size": 2, - "n_shared_experts": 1 - }, - "name": "glm5.1" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 64, - "hidden": 6144, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "mxfp4", - "expert_parallel_size": 1, - "routed_tensor_parallel_size": 2, - "shared_tensor_parallel_size": 2, - "n_shared_experts": 1 - }, - "name": "glm5.1" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 128, - "hidden": 6144, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "mxfp4", - "expert_parallel_size": 1, - "routed_tensor_parallel_size": 2, - "shared_tensor_parallel_size": 2, - "n_shared_experts": 1 - }, - "name": "glm5.1" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 256, - "hidden": 6144, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "mxfp4", - "expert_parallel_size": 1, - "routed_tensor_parallel_size": 2, - "shared_tensor_parallel_size": 2, - "n_shared_experts": 1 - }, - "name": "glm5.1" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 4096, - "hidden": 6144, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "mxfp4", - "expert_parallel_size": 1, - "routed_tensor_parallel_size": 4, - "shared_tensor_parallel_size": 4, - "n_shared_experts": 1 - }, - "name": "glm5.1" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 4, - "hidden": 6144, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "mxfp4", - "expert_parallel_size": 1, - "routed_tensor_parallel_size": 4, - "shared_tensor_parallel_size": 4, - "n_shared_experts": 1 - }, - "name": "glm5.1" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 8192, - "hidden": 6144, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "mxfp4", - "expert_parallel_size": 1, - "routed_tensor_parallel_size": 4, - "shared_tensor_parallel_size": 4, - "n_shared_experts": 1 - }, - "name": "glm5.1" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 8, - "hidden": 6144, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "mxfp4", - "expert_parallel_size": 1, - "routed_tensor_parallel_size": 4, - "shared_tensor_parallel_size": 4, - "n_shared_experts": 1 - }, - "name": "glm5.1" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 16384, - "hidden": 6144, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "mxfp4", - "expert_parallel_size": 1, - "routed_tensor_parallel_size": 4, - "shared_tensor_parallel_size": 4, - "n_shared_experts": 1 - }, - "name": "glm5.1" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 16, - "hidden": 6144, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "mxfp4", - "expert_parallel_size": 1, - "routed_tensor_parallel_size": 4, - "shared_tensor_parallel_size": 4, - "n_shared_experts": 1 - }, - "name": "glm5.1" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 32768, - "hidden": 6144, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "mxfp4", - "expert_parallel_size": 1, - "routed_tensor_parallel_size": 4, - "shared_tensor_parallel_size": 4, - "n_shared_experts": 1 - }, - "name": "glm5.1" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 32, - "hidden": 6144, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "mxfp4", - "expert_parallel_size": 1, - "routed_tensor_parallel_size": 4, - "shared_tensor_parallel_size": 4, - "n_shared_experts": 1 - }, - "name": "glm5.1" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 64, - "hidden": 6144, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "mxfp4", - "expert_parallel_size": 1, - "routed_tensor_parallel_size": 4, - "shared_tensor_parallel_size": 4, - "n_shared_experts": 1 - }, - "name": "glm5.1" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 128, - "hidden": 6144, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "mxfp4", - "expert_parallel_size": 1, - "routed_tensor_parallel_size": 4, - "shared_tensor_parallel_size": 4, - "n_shared_experts": 1 - }, - "name": "glm5.1" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 256, - "hidden": 6144, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "mxfp4", - "expert_parallel_size": 1, - "routed_tensor_parallel_size": 4, - "shared_tensor_parallel_size": 4, - "n_shared_experts": 1 - }, - "name": "glm5.1" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 4096, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 384, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "mxfp4", - "expert_parallel_size": 1, - "routed_tensor_parallel_size": 8, - "shared_tensor_parallel_size": 8, - "n_shared_experts": 1 - }, - "name": "kimik2.5" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 4, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 384, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "mxfp4", - "expert_parallel_size": 1, - "routed_tensor_parallel_size": 8, - "shared_tensor_parallel_size": 8, - "n_shared_experts": 1 - }, - "name": "kimik2.5" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 8192, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 384, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "mxfp4", - "expert_parallel_size": 1, - "routed_tensor_parallel_size": 8, - "shared_tensor_parallel_size": 8, - "n_shared_experts": 1 - }, - "name": "kimik2.5" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 8, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 384, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "mxfp4", - "expert_parallel_size": 1, - "routed_tensor_parallel_size": 8, - "shared_tensor_parallel_size": 8, - "n_shared_experts": 1 - }, - "name": "kimik2.5" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 16384, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 384, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "mxfp4", - "expert_parallel_size": 1, - "routed_tensor_parallel_size": 8, - "shared_tensor_parallel_size": 8, - "n_shared_experts": 1 - }, - "name": "kimik2.5" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 16, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 384, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "mxfp4", - "expert_parallel_size": 1, - "routed_tensor_parallel_size": 8, - "shared_tensor_parallel_size": 8, - "n_shared_experts": 1 - }, - "name": "kimik2.5" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 32768, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 384, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "mxfp4", - "expert_parallel_size": 1, - "routed_tensor_parallel_size": 8, - "shared_tensor_parallel_size": 8, - "n_shared_experts": 1 - }, - "name": "kimik2.5" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 32, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 384, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "mxfp4", - "expert_parallel_size": 1, - "routed_tensor_parallel_size": 8, - "shared_tensor_parallel_size": 8, - "n_shared_experts": 1 - }, - "name": "kimik2.5" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 64, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 384, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "mxfp4", - "expert_parallel_size": 1, - "routed_tensor_parallel_size": 8, - "shared_tensor_parallel_size": 8, - "n_shared_experts": 1 - }, - "name": "kimik2.5" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 4096, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 384, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "mxfp4", - "expert_parallel_size": 1, - "routed_tensor_parallel_size": 4, - "shared_tensor_parallel_size": 4, - "n_shared_experts": 1 - }, - "name": "kimik2.5" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 4, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 384, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "mxfp4", - "expert_parallel_size": 1, - "routed_tensor_parallel_size": 4, - "shared_tensor_parallel_size": 4, - "n_shared_experts": 1 - }, - "name": "kimik2.5" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 8192, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 384, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "mxfp4", - "expert_parallel_size": 1, - "routed_tensor_parallel_size": 4, - "shared_tensor_parallel_size": 4, - "n_shared_experts": 1 - }, - "name": "kimik2.5" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 8, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 384, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "mxfp4", - "expert_parallel_size": 1, - "routed_tensor_parallel_size": 4, - "shared_tensor_parallel_size": 4, - "n_shared_experts": 1 - }, - "name": "kimik2.5" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 16384, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 384, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "mxfp4", - "expert_parallel_size": 1, - "routed_tensor_parallel_size": 4, - "shared_tensor_parallel_size": 4, - "n_shared_experts": 1 - }, - "name": "kimik2.5" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 16, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 384, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "mxfp4", - "expert_parallel_size": 1, - "routed_tensor_parallel_size": 4, - "shared_tensor_parallel_size": 4, - "n_shared_experts": 1 - }, - "name": "kimik2.5" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 32768, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 384, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "mxfp4", - "expert_parallel_size": 1, - "routed_tensor_parallel_size": 4, - "shared_tensor_parallel_size": 4, - "n_shared_experts": 1 - }, - "name": "kimik2.5" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 32, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 384, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "mxfp4", - "expert_parallel_size": 1, - "routed_tensor_parallel_size": 4, - "shared_tensor_parallel_size": 4, - "n_shared_experts": 1 - }, - "name": "kimik2.5" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 64, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 384, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "mxfp4", - "expert_parallel_size": 1, - "routed_tensor_parallel_size": 4, - "shared_tensor_parallel_size": 4, - "n_shared_experts": 1 - }, - "name": "kimik2.5" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 128, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 384, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "mxfp4", - "expert_parallel_size": 1, - "routed_tensor_parallel_size": 8, - "shared_tensor_parallel_size": 8, - "n_shared_experts": 1 - }, - "name": "kimik2.5" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 128, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 384, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "mxfp4", - "expert_parallel_size": 1, - "routed_tensor_parallel_size": 4, - "shared_tensor_parallel_size": 4, - "n_shared_experts": 1 - }, - "name": "kimik2.5" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 2048, - "hidden": 3072, - "intermediate": 1536, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "expert_parallel_size": 2, - "routed_tensor_parallel_size": 1, - "shared_tensor_parallel_size": 1, - "n_shared_experts": 0 - }, - "name": "minimaxm2.5" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 2, - "hidden": 3072, - "intermediate": 1536, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "expert_parallel_size": 2, - "routed_tensor_parallel_size": 1, - "shared_tensor_parallel_size": 1, - "n_shared_experts": 0 - }, - "name": "minimaxm2.5" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 4096, - "hidden": 3072, - "intermediate": 1536, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "expert_parallel_size": 2, - "routed_tensor_parallel_size": 1, - "shared_tensor_parallel_size": 1, - "n_shared_experts": 0 - }, - "name": "minimaxm2.5" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 4, - "hidden": 3072, - "intermediate": 1536, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "expert_parallel_size": 2, - "routed_tensor_parallel_size": 1, - "shared_tensor_parallel_size": 1, - "n_shared_experts": 0 - }, - "name": "minimaxm2.5" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 8192, - "hidden": 3072, - "intermediate": 1536, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "expert_parallel_size": 2, - "routed_tensor_parallel_size": 1, - "shared_tensor_parallel_size": 1, - "n_shared_experts": 0 - }, - "name": "minimaxm2.5" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 8, - "hidden": 3072, - "intermediate": 1536, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "expert_parallel_size": 2, - "routed_tensor_parallel_size": 1, - "shared_tensor_parallel_size": 1, - "n_shared_experts": 0 - }, - "name": "minimaxm2.5" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 16384, - "hidden": 3072, - "intermediate": 1536, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "expert_parallel_size": 2, - "routed_tensor_parallel_size": 1, - "shared_tensor_parallel_size": 1, - "n_shared_experts": 0 - }, - "name": "minimaxm2.5" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 16, - "hidden": 3072, - "intermediate": 1536, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "expert_parallel_size": 2, - "routed_tensor_parallel_size": 1, - "shared_tensor_parallel_size": 1, - "n_shared_experts": 0 - }, - "name": "minimaxm2.5" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 32, - "hidden": 3072, - "intermediate": 1536, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "expert_parallel_size": 2, - "routed_tensor_parallel_size": 1, - "shared_tensor_parallel_size": 1, - "n_shared_experts": 0 - }, - "name": "minimaxm2.5" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 64, - "hidden": 3072, - "intermediate": 1536, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "expert_parallel_size": 2, - "routed_tensor_parallel_size": 1, - "shared_tensor_parallel_size": 1, - "n_shared_experts": 0 - }, - "name": "minimaxm2.5" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 128, - "hidden": 3072, - "intermediate": 1536, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "expert_parallel_size": 2, - "routed_tensor_parallel_size": 1, - "shared_tensor_parallel_size": 1, - "n_shared_experts": 0 - }, - "name": "minimaxm2.5" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 256, - "hidden": 3072, - "intermediate": 1536, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "expert_parallel_size": 2, - "routed_tensor_parallel_size": 1, - "shared_tensor_parallel_size": 1, - "n_shared_experts": 0 - }, - "name": "minimaxm2.5" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 128, - "hidden": 3072, - "intermediate": 1536, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "expert_parallel_size": 4, - "routed_tensor_parallel_size": 1, - "shared_tensor_parallel_size": 1, - "n_shared_experts": 0 - }, - "name": "minimaxm2.5" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 2048, - "hidden": 3072, - "intermediate": 1536, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "expert_parallel_size": 8, - "routed_tensor_parallel_size": 1, - "shared_tensor_parallel_size": 1, - "n_shared_experts": 0 - }, - "name": "minimaxm2.5" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 2, - "hidden": 3072, - "intermediate": 1536, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "expert_parallel_size": 8, - "routed_tensor_parallel_size": 1, - "shared_tensor_parallel_size": 1, - "n_shared_experts": 0 - }, - "name": "minimaxm2.5" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 16384, - "hidden": 3072, - "intermediate": 1536, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "expert_parallel_size": 8, - "routed_tensor_parallel_size": 1, - "shared_tensor_parallel_size": 1, - "n_shared_experts": 0 - }, - "name": "minimaxm2.5" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 4096, - "hidden": 3072, - "intermediate": 1536, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "expert_parallel_size": 1, - "routed_tensor_parallel_size": 2, - "shared_tensor_parallel_size": 2, - "n_shared_experts": 0 - }, - "name": "minimaxm2.5" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 8192, - "hidden": 3072, - "intermediate": 1536, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "expert_parallel_size": 1, - "routed_tensor_parallel_size": 2, - "shared_tensor_parallel_size": 2, - "n_shared_experts": 0 - }, - "name": "minimaxm2.5" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 16384, - "hidden": 3072, - "intermediate": 1536, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "expert_parallel_size": 1, - "routed_tensor_parallel_size": 2, - "shared_tensor_parallel_size": 2, - "n_shared_experts": 0 - }, - "name": "minimaxm2.5" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 4096, - "hidden": 3072, - "intermediate": 1536, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "mxfp4", - "expert_parallel_size": 1, - "routed_tensor_parallel_size": 1, - "shared_tensor_parallel_size": 1, - "n_shared_experts": 0 - }, - "name": "minimaxm2.5" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 4, - "hidden": 3072, - "intermediate": 1536, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "mxfp4", - "expert_parallel_size": 1, - "routed_tensor_parallel_size": 1, - "shared_tensor_parallel_size": 1, - "n_shared_experts": 0 - }, - "name": "minimaxm2.5" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 8192, - "hidden": 3072, - "intermediate": 1536, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "mxfp4", - "expert_parallel_size": 1, - "routed_tensor_parallel_size": 1, - "shared_tensor_parallel_size": 1, - "n_shared_experts": 0 - }, - "name": "minimaxm2.5" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 8, - "hidden": 3072, - "intermediate": 1536, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "mxfp4", - "expert_parallel_size": 1, - "routed_tensor_parallel_size": 1, - "shared_tensor_parallel_size": 1, - "n_shared_experts": 0 - }, - "name": "minimaxm2.5" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 16384, - "hidden": 3072, - "intermediate": 1536, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "mxfp4", - "expert_parallel_size": 1, - "routed_tensor_parallel_size": 1, - "shared_tensor_parallel_size": 1, - "n_shared_experts": 0 - }, - "name": "minimaxm2.5" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 16, - "hidden": 3072, - "intermediate": 1536, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "mxfp4", - "expert_parallel_size": 1, - "routed_tensor_parallel_size": 1, - "shared_tensor_parallel_size": 1, - "n_shared_experts": 0 - }, - "name": "minimaxm2.5" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 32768, - "hidden": 3072, - "intermediate": 1536, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "mxfp4", - "expert_parallel_size": 1, - "routed_tensor_parallel_size": 1, - "shared_tensor_parallel_size": 1, - "n_shared_experts": 0 - }, - "name": "minimaxm2.5" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 32, - "hidden": 3072, - "intermediate": 1536, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "mxfp4", - "expert_parallel_size": 1, - "routed_tensor_parallel_size": 1, - "shared_tensor_parallel_size": 1, - "n_shared_experts": 0 - }, - "name": "minimaxm2.5" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 64, - "hidden": 3072, - "intermediate": 1536, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "mxfp4", - "expert_parallel_size": 1, - "routed_tensor_parallel_size": 1, - "shared_tensor_parallel_size": 1, - "n_shared_experts": 0 - }, - "name": "minimaxm2.5" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 128, - "hidden": 3072, - "intermediate": 1536, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "mxfp4", - "expert_parallel_size": 1, - "routed_tensor_parallel_size": 1, - "shared_tensor_parallel_size": 1, - "n_shared_experts": 0 - }, - "name": "minimaxm2.5" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 256, - "hidden": 3072, - "intermediate": 1536, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "mxfp4", - "expert_parallel_size": 1, - "routed_tensor_parallel_size": 1, - "shared_tensor_parallel_size": 1, - "n_shared_experts": 0 - }, - "name": "minimaxm2.5" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 512, - "hidden": 3072, - "intermediate": 1536, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "mxfp4", - "expert_parallel_size": 1, - "routed_tensor_parallel_size": 1, - "shared_tensor_parallel_size": 1, - "n_shared_experts": 0 - }, - "name": "minimaxm2.5" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 1024, - "hidden": 3072, - "intermediate": 1536, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "mxfp4", - "expert_parallel_size": 1, - "routed_tensor_parallel_size": 1, - "shared_tensor_parallel_size": 1, - "n_shared_experts": 0 - }, - "name": "minimaxm2.5" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 4096, - "hidden": 3072, - "intermediate": 1536, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "mxfp4", - "expert_parallel_size": 1, - "routed_tensor_parallel_size": 2, - "shared_tensor_parallel_size": 2, - "n_shared_experts": 0 - }, - "name": "minimaxm2.5" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 4, - "hidden": 3072, - "intermediate": 1536, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "mxfp4", - "expert_parallel_size": 1, - "routed_tensor_parallel_size": 2, - "shared_tensor_parallel_size": 2, - "n_shared_experts": 0 - }, - "name": "minimaxm2.5" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 8192, - "hidden": 3072, - "intermediate": 1536, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "mxfp4", - "expert_parallel_size": 1, - "routed_tensor_parallel_size": 2, - "shared_tensor_parallel_size": 2, - "n_shared_experts": 0 - }, - "name": "minimaxm2.5" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 8, - "hidden": 3072, - "intermediate": 1536, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "mxfp4", - "expert_parallel_size": 1, - "routed_tensor_parallel_size": 2, - "shared_tensor_parallel_size": 2, - "n_shared_experts": 0 - }, - "name": "minimaxm2.5" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 16384, - "hidden": 3072, - "intermediate": 1536, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "mxfp4", - "expert_parallel_size": 1, - "routed_tensor_parallel_size": 2, - "shared_tensor_parallel_size": 2, - "n_shared_experts": 0 - }, - "name": "minimaxm2.5" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 16, - "hidden": 3072, - "intermediate": 1536, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "mxfp4", - "expert_parallel_size": 1, - "routed_tensor_parallel_size": 2, - "shared_tensor_parallel_size": 2, - "n_shared_experts": 0 - }, - "name": "minimaxm2.5" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 32768, - "hidden": 3072, - "intermediate": 1536, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "mxfp4", - "expert_parallel_size": 1, - "routed_tensor_parallel_size": 2, - "shared_tensor_parallel_size": 2, - "n_shared_experts": 0 - }, - "name": "minimaxm2.5" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 32, - "hidden": 3072, - "intermediate": 1536, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "mxfp4", - "expert_parallel_size": 1, - "routed_tensor_parallel_size": 2, - "shared_tensor_parallel_size": 2, - "n_shared_experts": 0 - }, - "name": "minimaxm2.5" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 64, - "hidden": 3072, - "intermediate": 1536, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "mxfp4", - "expert_parallel_size": 1, - "routed_tensor_parallel_size": 2, - "shared_tensor_parallel_size": 2, - "n_shared_experts": 0 - }, - "name": "minimaxm2.5" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 128, - "hidden": 3072, - "intermediate": 1536, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "mxfp4", - "expert_parallel_size": 1, - "routed_tensor_parallel_size": 2, - "shared_tensor_parallel_size": 2, - "n_shared_experts": 0 - }, - "name": "minimaxm2.5" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 256, - "hidden": 3072, - "intermediate": 1536, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "mxfp4", - "expert_parallel_size": 1, - "routed_tensor_parallel_size": 2, - "shared_tensor_parallel_size": 2, - "n_shared_experts": 0 - }, - "name": "minimaxm2.5" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 512, - "hidden": 3072, - "intermediate": 1536, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "mxfp4", - "expert_parallel_size": 1, - "routed_tensor_parallel_size": 2, - "shared_tensor_parallel_size": 2, - "n_shared_experts": 0 - }, - "name": "minimaxm2.5" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 1024, - "hidden": 3072, - "intermediate": 1536, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "mxfp4", - "expert_parallel_size": 1, - "routed_tensor_parallel_size": 2, - "shared_tensor_parallel_size": 2, - "n_shared_experts": 0 - }, - "name": "minimaxm2.5" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 4096, - "hidden": 3072, - "intermediate": 1536, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "mxfp4", - "expert_parallel_size": 1, - "routed_tensor_parallel_size": 4, - "shared_tensor_parallel_size": 4, - "n_shared_experts": 0 - }, - "name": "minimaxm2.5" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 4, - "hidden": 3072, - "intermediate": 1536, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "mxfp4", - "expert_parallel_size": 1, - "routed_tensor_parallel_size": 4, - "shared_tensor_parallel_size": 4, - "n_shared_experts": 0 - }, - "name": "minimaxm2.5" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 8192, - "hidden": 3072, - "intermediate": 1536, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "mxfp4", - "expert_parallel_size": 1, - "routed_tensor_parallel_size": 4, - "shared_tensor_parallel_size": 4, - "n_shared_experts": 0 - }, - "name": "minimaxm2.5" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 8, - "hidden": 3072, - "intermediate": 1536, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "mxfp4", - "expert_parallel_size": 1, - "routed_tensor_parallel_size": 4, - "shared_tensor_parallel_size": 4, - "n_shared_experts": 0 - }, - "name": "minimaxm2.5" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 16384, - "hidden": 3072, - "intermediate": 1536, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "mxfp4", - "expert_parallel_size": 1, - "routed_tensor_parallel_size": 4, - "shared_tensor_parallel_size": 4, - "n_shared_experts": 0 - }, - "name": "minimaxm2.5" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 16, - "hidden": 3072, - "intermediate": 1536, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "mxfp4", - "expert_parallel_size": 1, - "routed_tensor_parallel_size": 4, - "shared_tensor_parallel_size": 4, - "n_shared_experts": 0 - }, - "name": "minimaxm2.5" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 32768, - "hidden": 3072, - "intermediate": 1536, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "mxfp4", - "expert_parallel_size": 1, - "routed_tensor_parallel_size": 4, - "shared_tensor_parallel_size": 4, - "n_shared_experts": 0 - }, - "name": "minimaxm2.5" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 32, - "hidden": 3072, - "intermediate": 1536, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "mxfp4", - "expert_parallel_size": 1, - "routed_tensor_parallel_size": 4, - "shared_tensor_parallel_size": 4, - "n_shared_experts": 0 - }, - "name": "minimaxm2.5" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 64, - "hidden": 3072, - "intermediate": 1536, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "mxfp4", - "expert_parallel_size": 1, - "routed_tensor_parallel_size": 4, - "shared_tensor_parallel_size": 4, - "n_shared_experts": 0 - }, - "name": "minimaxm2.5" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 128, - "hidden": 3072, - "intermediate": 1536, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "mxfp4", - "expert_parallel_size": 1, - "routed_tensor_parallel_size": 4, - "shared_tensor_parallel_size": 4, - "n_shared_experts": 0 - }, - "name": "minimaxm2.5" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 4096, - "hidden": 3072, - "intermediate": 1536, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "mxfp4", - "expert_parallel_size": 1, - "routed_tensor_parallel_size": 8, - "shared_tensor_parallel_size": 8, - "n_shared_experts": 0 - }, - "name": "minimaxm2.5" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 4, - "hidden": 3072, - "intermediate": 1536, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "mxfp4", - "expert_parallel_size": 1, - "routed_tensor_parallel_size": 8, - "shared_tensor_parallel_size": 8, - "n_shared_experts": 0 - }, - "name": "minimaxm2.5" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 8192, - "hidden": 3072, - "intermediate": 1536, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "mxfp4", - "expert_parallel_size": 1, - "routed_tensor_parallel_size": 8, - "shared_tensor_parallel_size": 8, - "n_shared_experts": 0 - }, - "name": "minimaxm2.5" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 8, - "hidden": 3072, - "intermediate": 1536, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "mxfp4", - "expert_parallel_size": 1, - "routed_tensor_parallel_size": 8, - "shared_tensor_parallel_size": 8, - "n_shared_experts": 0 - }, - "name": "minimaxm2.5" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 16384, - "hidden": 3072, - "intermediate": 1536, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "mxfp4", - "expert_parallel_size": 1, - "routed_tensor_parallel_size": 8, - "shared_tensor_parallel_size": 8, - "n_shared_experts": 0 - }, - "name": "minimaxm2.5" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 16, - "hidden": 3072, - "intermediate": 1536, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "mxfp4", - "expert_parallel_size": 1, - "routed_tensor_parallel_size": 8, - "shared_tensor_parallel_size": 8, - "n_shared_experts": 0 - }, - "name": "minimaxm2.5" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 32768, - "hidden": 3072, - "intermediate": 1536, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "mxfp4", - "expert_parallel_size": 1, - "routed_tensor_parallel_size": 8, - "shared_tensor_parallel_size": 8, - "n_shared_experts": 0 - }, - "name": "minimaxm2.5" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 4096, - "hidden": 3072, - "intermediate": 1536, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "expert_parallel_size": 8, - "routed_tensor_parallel_size": 1, - "shared_tensor_parallel_size": 1, - "n_shared_experts": 0 - }, - "name": "minimaxm2.5" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 4, - "hidden": 3072, - "intermediate": 1536, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "expert_parallel_size": 8, - "routed_tensor_parallel_size": 1, - "shared_tensor_parallel_size": 1, - "n_shared_experts": 0 - }, - "name": "minimaxm2.5" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 8192, - "hidden": 3072, - "intermediate": 1536, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "expert_parallel_size": 8, - "routed_tensor_parallel_size": 1, - "shared_tensor_parallel_size": 1, - "n_shared_experts": 0 - }, - "name": "minimaxm2.5" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 8, - "hidden": 3072, - "intermediate": 1536, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "expert_parallel_size": 8, - "routed_tensor_parallel_size": 1, - "shared_tensor_parallel_size": 1, - "n_shared_experts": 0 - }, - "name": "minimaxm2.5" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 16, - "hidden": 3072, - "intermediate": 1536, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "expert_parallel_size": 8, - "routed_tensor_parallel_size": 1, - "shared_tensor_parallel_size": 1, - "n_shared_experts": 0 - }, - "name": "minimaxm2.5" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 32768, - "hidden": 3072, - "intermediate": 1536, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "expert_parallel_size": 8, - "routed_tensor_parallel_size": 1, - "shared_tensor_parallel_size": 1, - "n_shared_experts": 0 - }, - "name": "minimaxm2.5" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 32, - "hidden": 3072, - "intermediate": 1536, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "expert_parallel_size": 8, - "routed_tensor_parallel_size": 1, - "shared_tensor_parallel_size": 1, - "n_shared_experts": 0 - }, - "name": "minimaxm2.5" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 64, - "hidden": 3072, - "intermediate": 1536, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "expert_parallel_size": 8, - "routed_tensor_parallel_size": 1, - "shared_tensor_parallel_size": 1, - "n_shared_experts": 0 - }, - "name": "minimaxm2.5" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 128, - "hidden": 3072, - "intermediate": 1536, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "expert_parallel_size": 8, - "routed_tensor_parallel_size": 1, - "shared_tensor_parallel_size": 1, - "n_shared_experts": 0 - }, - "name": "minimaxm2.5" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 256, - "hidden": 3072, - "intermediate": 1536, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "expert_parallel_size": 8, - "routed_tensor_parallel_size": 1, - "shared_tensor_parallel_size": 1, - "n_shared_experts": 0 - }, - "name": "minimaxm2.5" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 512, - "hidden": 3072, - "intermediate": 1536, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "expert_parallel_size": 8, - "routed_tensor_parallel_size": 1, - "shared_tensor_parallel_size": 1, - "n_shared_experts": 0 - }, - "name": "minimaxm2.5" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 32768, - "hidden": 2880, - "intermediate": 2880, - "num_experts": 128, - "top_k": 4, - "dtype_act": "fp8", - "dtype_weight": "mxfp4", - "expert_parallel_size": 1, - "routed_tensor_parallel_size": 1, - "shared_tensor_parallel_size": 1, - "n_shared_experts": 0 - }, - "name": "gptoss" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 256, - "hidden": 2880, - "intermediate": 2880, - "num_experts": 128, - "top_k": 4, - "dtype_act": "fp8", - "dtype_weight": "mxfp4", - "expert_parallel_size": 1, - "routed_tensor_parallel_size": 1, - "shared_tensor_parallel_size": 1, - "n_shared_experts": 0 - }, - "name": "gptoss" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 1024, - "hidden": 2880, - "intermediate": 2880, - "num_experts": 128, - "top_k": 4, - "dtype_act": "fp8", - "dtype_weight": "mxfp4", - "expert_parallel_size": 1, - "routed_tensor_parallel_size": 8, - "shared_tensor_parallel_size": 8, - "n_shared_experts": 0 - }, - "name": "gptoss" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 1, - "hidden": 2880, - "intermediate": 2880, - "num_experts": 128, - "top_k": 4, - "dtype_act": "fp8", - "dtype_weight": "mxfp4", - "expert_parallel_size": 1, - "routed_tensor_parallel_size": 8, - "shared_tensor_parallel_size": 8, - "n_shared_experts": 0 - }, - "name": "gptoss" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 2048, - "hidden": 2880, - "intermediate": 2880, - "num_experts": 128, - "top_k": 4, - "dtype_act": "fp8", - "dtype_weight": "mxfp4", - "expert_parallel_size": 1, - "routed_tensor_parallel_size": 8, - "shared_tensor_parallel_size": 8, - "n_shared_experts": 0 - }, - "name": "gptoss" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 2, - "hidden": 2880, - "intermediate": 2880, - "num_experts": 128, - "top_k": 4, - "dtype_act": "fp8", - "dtype_weight": "mxfp4", - "expert_parallel_size": 1, - "routed_tensor_parallel_size": 8, - "shared_tensor_parallel_size": 8, - "n_shared_experts": 0 - }, - "name": "gptoss" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 8192, - "hidden": 2880, - "intermediate": 2880, - "num_experts": 128, - "top_k": 4, - "dtype_act": "fp8", - "dtype_weight": "mxfp4", - "expert_parallel_size": 1, - "routed_tensor_parallel_size": 8, - "shared_tensor_parallel_size": 8, - "n_shared_experts": 0 - }, - "name": "gptoss" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 16384, - "hidden": 2880, - "intermediate": 2880, - "num_experts": 128, - "top_k": 4, - "dtype_act": "fp8", - "dtype_weight": "mxfp4", - "expert_parallel_size": 1, - "routed_tensor_parallel_size": 8, - "shared_tensor_parallel_size": 8, - "n_shared_experts": 0 - }, - "name": "gptoss" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 4096, - "hidden": 2880, - "intermediate": 2880, - "num_experts": 128, - "top_k": 4, - "dtype_act": "fp8", - "dtype_weight": "mxfp4", - "expert_parallel_size": 1, - "routed_tensor_parallel_size": 1, - "shared_tensor_parallel_size": 1, - "n_shared_experts": 0 - }, - "name": "gptoss" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 4, - "hidden": 2880, - "intermediate": 2880, - "num_experts": 128, - "top_k": 4, - "dtype_act": "fp8", - "dtype_weight": "mxfp4", - "expert_parallel_size": 1, - "routed_tensor_parallel_size": 1, - "shared_tensor_parallel_size": 1, - "n_shared_experts": 0 - }, - "name": "gptoss" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 8192, - "hidden": 2880, - "intermediate": 2880, - "num_experts": 128, - "top_k": 4, - "dtype_act": "fp8", - "dtype_weight": "mxfp4", - "expert_parallel_size": 1, - "routed_tensor_parallel_size": 1, - "shared_tensor_parallel_size": 1, - "n_shared_experts": 0 - }, - "name": "gptoss" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 8, - "hidden": 2880, - "intermediate": 2880, - "num_experts": 128, - "top_k": 4, - "dtype_act": "fp8", - "dtype_weight": "mxfp4", - "expert_parallel_size": 1, - "routed_tensor_parallel_size": 1, - "shared_tensor_parallel_size": 1, - "n_shared_experts": 0 - }, - "name": "gptoss" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 16384, - "hidden": 2880, - "intermediate": 2880, - "num_experts": 128, - "top_k": 4, - "dtype_act": "fp8", - "dtype_weight": "mxfp4", - "expert_parallel_size": 1, - "routed_tensor_parallel_size": 1, - "shared_tensor_parallel_size": 1, - "n_shared_experts": 0 - }, - "name": "gptoss" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 16, - "hidden": 2880, - "intermediate": 2880, - "num_experts": 128, - "top_k": 4, - "dtype_act": "fp8", - "dtype_weight": "mxfp4", - "expert_parallel_size": 1, - "routed_tensor_parallel_size": 1, - "shared_tensor_parallel_size": 1, - "n_shared_experts": 0 - }, - "name": "gptoss" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 32, - "hidden": 2880, - "intermediate": 2880, - "num_experts": 128, - "top_k": 4, - "dtype_act": "fp8", - "dtype_weight": "mxfp4", - "expert_parallel_size": 1, - "routed_tensor_parallel_size": 1, - "shared_tensor_parallel_size": 1, - "n_shared_experts": 0 - }, - "name": "gptoss" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 64, - "hidden": 2880, - "intermediate": 2880, - "num_experts": 128, - "top_k": 4, - "dtype_act": "fp8", - "dtype_weight": "mxfp4", - "expert_parallel_size": 1, - "routed_tensor_parallel_size": 1, - "shared_tensor_parallel_size": 1, - "n_shared_experts": 0 - }, - "name": "gptoss" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 128, - "hidden": 2880, - "intermediate": 2880, - "num_experts": 128, - "top_k": 4, - "dtype_act": "fp8", - "dtype_weight": "mxfp4", - "expert_parallel_size": 1, - "routed_tensor_parallel_size": 1, - "shared_tensor_parallel_size": 1, - "n_shared_experts": 0 - }, - "name": "gptoss" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 4096, - "hidden": 2880, - "intermediate": 2880, - "num_experts": 128, - "top_k": 4, - "dtype_act": "fp8", - "dtype_weight": "mxfp4", - "expert_parallel_size": 1, - "routed_tensor_parallel_size": 4, - "shared_tensor_parallel_size": 4, - "n_shared_experts": 0 - }, - "name": "gptoss" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 4, - "hidden": 2880, - "intermediate": 2880, - "num_experts": 128, - "top_k": 4, - "dtype_act": "fp8", - "dtype_weight": "mxfp4", - "expert_parallel_size": 1, - "routed_tensor_parallel_size": 4, - "shared_tensor_parallel_size": 4, - "n_shared_experts": 0 - }, - "name": "gptoss" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 8192, - "hidden": 2880, - "intermediate": 2880, - "num_experts": 128, - "top_k": 4, - "dtype_act": "fp8", - "dtype_weight": "mxfp4", - "expert_parallel_size": 1, - "routed_tensor_parallel_size": 4, - "shared_tensor_parallel_size": 4, - "n_shared_experts": 0 - }, - "name": "gptoss" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 8, - "hidden": 2880, - "intermediate": 2880, - "num_experts": 128, - "top_k": 4, - "dtype_act": "fp8", - "dtype_weight": "mxfp4", - "expert_parallel_size": 1, - "routed_tensor_parallel_size": 4, - "shared_tensor_parallel_size": 4, - "n_shared_experts": 0 - }, - "name": "gptoss" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 4096, - "hidden": 2880, - "intermediate": 2880, - "num_experts": 128, - "top_k": 4, - "dtype_act": "fp8", - "dtype_weight": "mxfp4", - "expert_parallel_size": 1, - "routed_tensor_parallel_size": 8, - "shared_tensor_parallel_size": 8, - "n_shared_experts": 0 - }, - "name": "gptoss" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 4, - "hidden": 2880, - "intermediate": 2880, - "num_experts": 128, - "top_k": 4, - "dtype_act": "fp8", - "dtype_weight": "mxfp4", - "expert_parallel_size": 1, - "routed_tensor_parallel_size": 8, - "shared_tensor_parallel_size": 8, - "n_shared_experts": 0 - }, - "name": "gptoss" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 8, - "hidden": 2880, - "intermediate": 2880, - "num_experts": 128, - "top_k": 4, - "dtype_act": "fp8", - "dtype_weight": "mxfp4", - "expert_parallel_size": 1, - "routed_tensor_parallel_size": 8, - "shared_tensor_parallel_size": 8, - "n_shared_experts": 0 - }, - "name": "gptoss" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 16, - "hidden": 2880, - "intermediate": 2880, - "num_experts": 128, - "top_k": 4, - "dtype_act": "fp8", - "dtype_weight": "mxfp4", - "expert_parallel_size": 1, - "routed_tensor_parallel_size": 8, - "shared_tensor_parallel_size": 8, - "n_shared_experts": 0 - }, - "name": "gptoss" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 32768, - "hidden": 2880, - "intermediate": 2880, - "num_experts": 128, - "top_k": 4, - "dtype_act": "fp8", - "dtype_weight": "mxfp4", - "expert_parallel_size": 1, - "routed_tensor_parallel_size": 4, - "shared_tensor_parallel_size": 4, - "n_shared_experts": 0 - }, - "name": "gptoss" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 32768, - "hidden": 2880, - "intermediate": 2880, - "num_experts": 128, - "top_k": 4, - "dtype_act": "fp8", - "dtype_weight": "mxfp4", - "expert_parallel_size": 1, - "routed_tensor_parallel_size": 8, - "shared_tensor_parallel_size": 8, - "n_shared_experts": 0 - }, - "name": "gptoss" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 32, - "hidden": 2880, - "intermediate": 2880, - "num_experts": 128, - "top_k": 4, - "dtype_act": "fp8", - "dtype_weight": "mxfp4", - "expert_parallel_size": 1, - "routed_tensor_parallel_size": 8, - "shared_tensor_parallel_size": 8, - "n_shared_experts": 0 - }, - "name": "gptoss" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 768, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "expert_parallel_size": 8, - "routed_tensor_parallel_size": 1, - "shared_tensor_parallel_size": 1, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 2048, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "expert_parallel_size": 1, - "routed_tensor_parallel_size": 4, - "shared_tensor_parallel_size": 4, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 1024, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "expert_parallel_size": 1, - "routed_tensor_parallel_size": 4, - "shared_tensor_parallel_size": 4, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 1536, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "expert_parallel_size": 8, - "routed_tensor_parallel_size": 1, - "shared_tensor_parallel_size": 1, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 1024, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "mxfp4", - "expert_parallel_size": 1, - "routed_tensor_parallel_size": 8, - "shared_tensor_parallel_size": 8, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 2048, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "mxfp4", - "expert_parallel_size": 1, - "routed_tensor_parallel_size": 8, - "shared_tensor_parallel_size": 8, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 32768, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "mxfp4", - "expert_parallel_size": 4, - "routed_tensor_parallel_size": 1, - "shared_tensor_parallel_size": 1, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 1024, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "mxfp4", - "expert_parallel_size": 8, - "routed_tensor_parallel_size": 1, - "shared_tensor_parallel_size": 1, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 2048, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "mxfp4", - "expert_parallel_size": 8, - "routed_tensor_parallel_size": 1, - "shared_tensor_parallel_size": 1, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 4096, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "mxfp4", - "expert_parallel_size": 8, - "routed_tensor_parallel_size": 1, - "shared_tensor_parallel_size": 1, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 32768, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "mxfp4", - "expert_parallel_size": 8, - "routed_tensor_parallel_size": 1, - "shared_tensor_parallel_size": 1, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 8192, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "mxfp4", - "expert_parallel_size": 8, - "routed_tensor_parallel_size": 1, - "shared_tensor_parallel_size": 1, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 16384, - "hidden": 7168, - "intermediate": 3072, - "num_experts": 384, - "top_k": 6, - "dtype_act": "fp8", - "dtype_weight": "mxfp4", - "expert_parallel_size": 1, - "routed_tensor_parallel_size": 8, - "shared_tensor_parallel_size": 1, - "n_shared_experts": 1 - }, - "name": "dsv4" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 16, - "hidden": 7168, - "intermediate": 3072, - "num_experts": 384, - "top_k": 6, - "dtype_act": "fp8", - "dtype_weight": "mxfp4", - "expert_parallel_size": 1, - "routed_tensor_parallel_size": 8, - "shared_tensor_parallel_size": 1, - "n_shared_experts": 1 - }, - "name": "dsv4" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 32768, - "hidden": 7168, - "intermediate": 3072, - "num_experts": 384, - "top_k": 6, - "dtype_act": "fp8", - "dtype_weight": "mxfp4", - "expert_parallel_size": 1, - "routed_tensor_parallel_size": 8, - "shared_tensor_parallel_size": 1, - "n_shared_experts": 1 - }, - "name": "dsv4" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 32, - "hidden": 7168, - "intermediate": 3072, - "num_experts": 384, - "top_k": 6, - "dtype_act": "fp8", - "dtype_weight": "mxfp4", - "expert_parallel_size": 1, - "routed_tensor_parallel_size": 8, - "shared_tensor_parallel_size": 1, - "n_shared_experts": 1 - }, - "name": "dsv4" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 64, - "hidden": 7168, - "intermediate": 3072, - "num_experts": 384, - "top_k": 6, - "dtype_act": "fp8", - "dtype_weight": "mxfp4", - "expert_parallel_size": 1, - "routed_tensor_parallel_size": 8, - "shared_tensor_parallel_size": 1, - "n_shared_experts": 1 - }, - "name": "dsv4" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 128, - "hidden": 7168, - "intermediate": 3072, - "num_experts": 384, - "top_k": 6, - "dtype_act": "fp8", - "dtype_weight": "mxfp4", - "expert_parallel_size": 1, - "routed_tensor_parallel_size": 8, - "shared_tensor_parallel_size": 1, - "n_shared_experts": 1 - }, - "name": "dsv4" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 256, - "hidden": 7168, - "intermediate": 3072, - "num_experts": 384, - "top_k": 6, - "dtype_act": "fp8", - "dtype_weight": "mxfp4", - "expert_parallel_size": 1, - "routed_tensor_parallel_size": 8, - "shared_tensor_parallel_size": 1, - "n_shared_experts": 1 - }, - "name": "dsv4" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 1024, - "hidden": 7168, - "intermediate": 3072, - "num_experts": 384, - "top_k": 6, - "dtype_act": "fp8", - "dtype_weight": "mxfp4", - "expert_parallel_size": 1, - "routed_tensor_parallel_size": 8, - "shared_tensor_parallel_size": 1, - "n_shared_experts": 1 - }, - "name": "dsv4" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 1, - "hidden": 7168, - "intermediate": 3072, - "num_experts": 384, - "top_k": 6, - "dtype_act": "fp8", - "dtype_weight": "mxfp4", - "expert_parallel_size": 1, - "routed_tensor_parallel_size": 8, - "shared_tensor_parallel_size": 1, - "n_shared_experts": 1 - }, - "name": "dsv4" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 8192, - "hidden": 7168, - "intermediate": 3072, - "num_experts": 384, - "top_k": 6, - "dtype_act": "fp8", - "dtype_weight": "mxfp4", - "expert_parallel_size": 1, - "routed_tensor_parallel_size": 8, - "shared_tensor_parallel_size": 1, - "n_shared_experts": 1 - }, - "name": "dsv4" - } -] \ No newline at end of file diff --git a/operatorx/testlists/moe_gemm_subset.json b/operatorx/testlists/moe_gemm_subset.json deleted file mode 100644 index 68ac4b1ddf..0000000000 --- a/operatorx/testlists/moe_gemm_subset.json +++ /dev/null @@ -1,542 +0,0 @@ -[ - { - "type": "moe_gemm", - "args": { - "num_tokens": 1024, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "bf16", - "dtype_weight": "mxfp4", - "expert_parallel_size": 1, - "routed_tensor_parallel_size": 1, - "shared_tensor_parallel_size": 1, - "n_shared_experts": 0, - "expert_distribution": "uniform" - }, - "name": "dsv3" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 1024, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "bf16", - "dtype_weight": "mxfp4", - "expert_parallel_size": 1, - "routed_tensor_parallel_size": 1, - "shared_tensor_parallel_size": 1, - "n_shared_experts": 1, - "expert_distribution": "uniform" - }, - "name": "dsv3" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 1024, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "bf16", - "dtype_weight": "mxfp4", - "expert_parallel_size": 1, - "routed_tensor_parallel_size": 4, - "shared_tensor_parallel_size": 4, - "n_shared_experts": 0, - "expert_distribution": "uniform" - }, - "name": "dsv3" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 1024, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "bf16", - "dtype_weight": "mxfp4", - "expert_parallel_size": 1, - "routed_tensor_parallel_size": 4, - "shared_tensor_parallel_size": 4, - "n_shared_experts": 1, - "expert_distribution": "uniform" - }, - "name": "dsv3" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 1024, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "bf16", - "dtype_weight": "mxfp4", - "expert_parallel_size": 1, - "routed_tensor_parallel_size": 8, - "shared_tensor_parallel_size": 8, - "n_shared_experts": 0, - "expert_distribution": "uniform" - }, - "name": "dsv3" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 1024, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "bf16", - "dtype_weight": "mxfp4", - "expert_parallel_size": 1, - "routed_tensor_parallel_size": 8, - "shared_tensor_parallel_size": 8, - "n_shared_experts": 1, - "expert_distribution": "uniform" - }, - "name": "dsv3" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 1024, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "bf16", - "dtype_weight": "mxfp4", - "expert_parallel_size": 4, - "routed_tensor_parallel_size": 1, - "shared_tensor_parallel_size": 1, - "n_shared_experts": 0, - "expert_distribution": "uniform" - }, - "name": "dsv3" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 1024, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "bf16", - "dtype_weight": "mxfp4", - "expert_parallel_size": 4, - "routed_tensor_parallel_size": 1, - "shared_tensor_parallel_size": 1, - "n_shared_experts": 1, - "expert_distribution": "uniform" - }, - "name": "dsv3" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 1024, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "bf16", - "dtype_weight": "mxfp4", - "expert_parallel_size": 8, - "routed_tensor_parallel_size": 1, - "shared_tensor_parallel_size": 1, - "n_shared_experts": 0, - "expert_distribution": "uniform" - }, - "name": "dsv3" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 1024, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "bf16", - "dtype_weight": "mxfp4", - "expert_parallel_size": 8, - "routed_tensor_parallel_size": 1, - "shared_tensor_parallel_size": 1, - "n_shared_experts": 1, - "expert_distribution": "uniform" - }, - "name": "dsv3" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 8192, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "bf16", - "dtype_weight": "mxfp4", - "expert_parallel_size": 1, - "routed_tensor_parallel_size": 1, - "shared_tensor_parallel_size": 1, - "n_shared_experts": 0, - "expert_distribution": "uniform" - }, - "name": "dsv3" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 8192, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "bf16", - "dtype_weight": "mxfp4", - "expert_parallel_size": 1, - "routed_tensor_parallel_size": 1, - "shared_tensor_parallel_size": 1, - "n_shared_experts": 1, - "expert_distribution": "uniform" - }, - "name": "dsv3" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 8192, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "bf16", - "dtype_weight": "mxfp4", - "expert_parallel_size": 1, - "routed_tensor_parallel_size": 4, - "shared_tensor_parallel_size": 4, - "n_shared_experts": 0, - "expert_distribution": "uniform" - }, - "name": "dsv3" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 8192, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "bf16", - "dtype_weight": "mxfp4", - "expert_parallel_size": 1, - "routed_tensor_parallel_size": 4, - "shared_tensor_parallel_size": 4, - "n_shared_experts": 1, - "expert_distribution": "uniform" - }, - "name": "dsv3" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 8192, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "bf16", - "dtype_weight": "mxfp4", - "expert_parallel_size": 1, - "routed_tensor_parallel_size": 8, - "shared_tensor_parallel_size": 8, - "n_shared_experts": 0, - "expert_distribution": "uniform" - }, - "name": "dsv3" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 8192, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "bf16", - "dtype_weight": "mxfp4", - "expert_parallel_size": 1, - "routed_tensor_parallel_size": 8, - "shared_tensor_parallel_size": 8, - "n_shared_experts": 1, - "expert_distribution": "uniform" - }, - "name": "dsv3" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 8192, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "bf16", - "dtype_weight": "mxfp4", - "expert_parallel_size": 4, - "routed_tensor_parallel_size": 1, - "shared_tensor_parallel_size": 1, - "n_shared_experts": 0, - "expert_distribution": "uniform" - }, - "name": "dsv3" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 8192, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "bf16", - "dtype_weight": "mxfp4", - "expert_parallel_size": 4, - "routed_tensor_parallel_size": 1, - "shared_tensor_parallel_size": 1, - "n_shared_experts": 1, - "expert_distribution": "uniform" - }, - "name": "dsv3" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 8192, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "bf16", - "dtype_weight": "mxfp4", - "expert_parallel_size": 8, - "routed_tensor_parallel_size": 1, - "shared_tensor_parallel_size": 1, - "n_shared_experts": 0, - "expert_distribution": "uniform" - }, - "name": "dsv3" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 8192, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "bf16", - "dtype_weight": "mxfp4", - "expert_parallel_size": 8, - "routed_tensor_parallel_size": 1, - "shared_tensor_parallel_size": 1, - "n_shared_experts": 1, - "expert_distribution": "uniform" - }, - "name": "dsv3" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 65536, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "bf16", - "dtype_weight": "mxfp4", - "expert_parallel_size": 1, - "routed_tensor_parallel_size": 1, - "shared_tensor_parallel_size": 1, - "n_shared_experts": 0, - "expert_distribution": "uniform" - }, - "name": "dsv3" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 65536, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "bf16", - "dtype_weight": "mxfp4", - "expert_parallel_size": 1, - "routed_tensor_parallel_size": 1, - "shared_tensor_parallel_size": 1, - "n_shared_experts": 1, - "expert_distribution": "uniform" - }, - "name": "dsv3" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 65536, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "bf16", - "dtype_weight": "mxfp4", - "expert_parallel_size": 1, - "routed_tensor_parallel_size": 4, - "shared_tensor_parallel_size": 4, - "n_shared_experts": 0, - "expert_distribution": "uniform" - }, - "name": "dsv3" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 65536, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "bf16", - "dtype_weight": "mxfp4", - "expert_parallel_size": 1, - "routed_tensor_parallel_size": 4, - "shared_tensor_parallel_size": 4, - "n_shared_experts": 1, - "expert_distribution": "uniform" - }, - "name": "dsv3" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 65536, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "bf16", - "dtype_weight": "mxfp4", - "expert_parallel_size": 1, - "routed_tensor_parallel_size": 8, - "shared_tensor_parallel_size": 8, - "n_shared_experts": 0, - "expert_distribution": "uniform" - }, - "name": "dsv3" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 65536, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "bf16", - "dtype_weight": "mxfp4", - "expert_parallel_size": 1, - "routed_tensor_parallel_size": 8, - "shared_tensor_parallel_size": 8, - "n_shared_experts": 1, - "expert_distribution": "uniform" - }, - "name": "dsv3" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 65536, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "bf16", - "dtype_weight": "mxfp4", - "expert_parallel_size": 4, - "routed_tensor_parallel_size": 1, - "shared_tensor_parallel_size": 1, - "n_shared_experts": 0, - "expert_distribution": "uniform" - }, - "name": "dsv3" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 65536, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "bf16", - "dtype_weight": "mxfp4", - "expert_parallel_size": 4, - "routed_tensor_parallel_size": 1, - "shared_tensor_parallel_size": 1, - "n_shared_experts": 1, - "expert_distribution": "uniform" - }, - "name": "dsv3" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 65536, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "bf16", - "dtype_weight": "mxfp4", - "expert_parallel_size": 8, - "routed_tensor_parallel_size": 1, - "shared_tensor_parallel_size": 1, - "n_shared_experts": 0, - "expert_distribution": "uniform" - }, - "name": "dsv3" - }, - { - "type": "moe_gemm", - "args": { - "num_tokens": 65536, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "bf16", - "dtype_weight": "mxfp4", - "expert_parallel_size": 8, - "routed_tensor_parallel_size": 1, - "shared_tensor_parallel_size": 1, - "n_shared_experts": 1, - "expert_distribution": "uniform" - }, - "name": "dsv3" - } -] \ No newline at end of file diff --git a/operatorx/testlists/moe_kimi_k3.json b/operatorx/testlists/moe_kimi_k3.json new file mode 100644 index 0000000000..3891b42763 --- /dev/null +++ b/operatorx/testlists/moe_kimi_k3.json @@ -0,0 +1,2626 @@ +[ + { + "type": "moe", + "args": { + "tokens": 1, + "activation": { + "alpha": 4.0, + "beta": 25.0, + "kind": "situ" + }, + "experts": { + "a1": { + "dtype": "bf16" + }, + "a2": { + "dtype": "bf16" + }, + "inter": 3072, + "latent": 3584, + "latent_norm": true, + "num": 896, + "top_k": 16, + "w1": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + }, + "w2": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + } + }, + "hidden": 7168, + "router": { + "bias": true, + "gate": { + "dtype": "bf16", + "logits": "fp32" + }, + "renormalize": true, + "scale": 1.0, + "scoring": "sigmoid", + "select": { + "groups": 1, + "kind": "grouped_topk", + "topk_groups": 1 + } + }, + "routing": { + "distribution": "natural", + "seed": 0 + }, + "shared": { + "a1": { + "dtype": "bf16" + }, + "count": 2, + "inter": 3072, + "w1": { + "dtype": "bf16" + }, + "w2": { + "dtype": "bf16" + } + } + }, + "sources": [ + "moonshotai/Kimi-K3/block_sparse_moe" + ] + }, + { + "type": "moe", + "args": { + "tokens": 2, + "activation": { + "alpha": 4.0, + "beta": 25.0, + "kind": "situ" + }, + "experts": { + "a1": { + "dtype": "bf16" + }, + "a2": { + "dtype": "bf16" + }, + "inter": 3072, + "latent": 3584, + "latent_norm": true, + "num": 896, + "top_k": 16, + "w1": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + }, + "w2": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + } + }, + "hidden": 7168, + "router": { + "bias": true, + "gate": { + "dtype": "bf16", + "logits": "fp32" + }, + "renormalize": true, + "scale": 1.0, + "scoring": "sigmoid", + "select": { + "groups": 1, + "kind": "grouped_topk", + "topk_groups": 1 + } + }, + "routing": { + "distribution": "natural", + "seed": 0 + }, + "shared": { + "a1": { + "dtype": "bf16" + }, + "count": 2, + "inter": 3072, + "w1": { + "dtype": "bf16" + }, + "w2": { + "dtype": "bf16" + } + } + }, + "sources": [ + "moonshotai/Kimi-K3/block_sparse_moe" + ] + }, + { + "type": "moe", + "args": { + "tokens": 3, + "activation": { + "alpha": 4.0, + "beta": 25.0, + "kind": "situ" + }, + "experts": { + "a1": { + "dtype": "bf16" + }, + "a2": { + "dtype": "bf16" + }, + "inter": 3072, + "latent": 3584, + "latent_norm": true, + "num": 896, + "top_k": 16, + "w1": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + }, + "w2": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + } + }, + "hidden": 7168, + "router": { + "bias": true, + "gate": { + "dtype": "bf16", + "logits": "fp32" + }, + "renormalize": true, + "scale": 1.0, + "scoring": "sigmoid", + "select": { + "groups": 1, + "kind": "grouped_topk", + "topk_groups": 1 + } + }, + "routing": { + "distribution": "natural", + "seed": 0 + }, + "shared": { + "a1": { + "dtype": "bf16" + }, + "count": 2, + "inter": 3072, + "w1": { + "dtype": "bf16" + }, + "w2": { + "dtype": "bf16" + } + } + }, + "sources": [ + "moonshotai/Kimi-K3/block_sparse_moe" + ] + }, + { + "type": "moe", + "args": { + "tokens": 4, + "activation": { + "alpha": 4.0, + "beta": 25.0, + "kind": "situ" + }, + "experts": { + "a1": { + "dtype": "bf16" + }, + "a2": { + "dtype": "bf16" + }, + "inter": 3072, + "latent": 3584, + "latent_norm": true, + "num": 896, + "top_k": 16, + "w1": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + }, + "w2": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + } + }, + "hidden": 7168, + "router": { + "bias": true, + "gate": { + "dtype": "bf16", + "logits": "fp32" + }, + "renormalize": true, + "scale": 1.0, + "scoring": "sigmoid", + "select": { + "groups": 1, + "kind": "grouped_topk", + "topk_groups": 1 + } + }, + "routing": { + "distribution": "natural", + "seed": 0 + }, + "shared": { + "a1": { + "dtype": "bf16" + }, + "count": 2, + "inter": 3072, + "w1": { + "dtype": "bf16" + }, + "w2": { + "dtype": "bf16" + } + } + }, + "sources": [ + "moonshotai/Kimi-K3/block_sparse_moe" + ] + }, + { + "type": "moe", + "args": { + "tokens": 6, + "activation": { + "alpha": 4.0, + "beta": 25.0, + "kind": "situ" + }, + "experts": { + "a1": { + "dtype": "bf16" + }, + "a2": { + "dtype": "bf16" + }, + "inter": 3072, + "latent": 3584, + "latent_norm": true, + "num": 896, + "top_k": 16, + "w1": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + }, + "w2": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + } + }, + "hidden": 7168, + "router": { + "bias": true, + "gate": { + "dtype": "bf16", + "logits": "fp32" + }, + "renormalize": true, + "scale": 1.0, + "scoring": "sigmoid", + "select": { + "groups": 1, + "kind": "grouped_topk", + "topk_groups": 1 + } + }, + "routing": { + "distribution": "natural", + "seed": 0 + }, + "shared": { + "a1": { + "dtype": "bf16" + }, + "count": 2, + "inter": 3072, + "w1": { + "dtype": "bf16" + }, + "w2": { + "dtype": "bf16" + } + } + }, + "sources": [ + "moonshotai/Kimi-K3/block_sparse_moe" + ] + }, + { + "type": "moe", + "args": { + "tokens": 8, + "activation": { + "alpha": 4.0, + "beta": 25.0, + "kind": "situ" + }, + "experts": { + "a1": { + "dtype": "bf16" + }, + "a2": { + "dtype": "bf16" + }, + "inter": 3072, + "latent": 3584, + "latent_norm": true, + "num": 896, + "top_k": 16, + "w1": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + }, + "w2": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + } + }, + "hidden": 7168, + "router": { + "bias": true, + "gate": { + "dtype": "bf16", + "logits": "fp32" + }, + "renormalize": true, + "scale": 1.0, + "scoring": "sigmoid", + "select": { + "groups": 1, + "kind": "grouped_topk", + "topk_groups": 1 + } + }, + "routing": { + "distribution": "natural", + "seed": 0 + }, + "shared": { + "a1": { + "dtype": "bf16" + }, + "count": 2, + "inter": 3072, + "w1": { + "dtype": "bf16" + }, + "w2": { + "dtype": "bf16" + } + } + }, + "sources": [ + "moonshotai/Kimi-K3/block_sparse_moe" + ] + }, + { + "type": "moe", + "args": { + "tokens": 12, + "activation": { + "alpha": 4.0, + "beta": 25.0, + "kind": "situ" + }, + "experts": { + "a1": { + "dtype": "bf16" + }, + "a2": { + "dtype": "bf16" + }, + "inter": 3072, + "latent": 3584, + "latent_norm": true, + "num": 896, + "top_k": 16, + "w1": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + }, + "w2": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + } + }, + "hidden": 7168, + "router": { + "bias": true, + "gate": { + "dtype": "bf16", + "logits": "fp32" + }, + "renormalize": true, + "scale": 1.0, + "scoring": "sigmoid", + "select": { + "groups": 1, + "kind": "grouped_topk", + "topk_groups": 1 + } + }, + "routing": { + "distribution": "natural", + "seed": 0 + }, + "shared": { + "a1": { + "dtype": "bf16" + }, + "count": 2, + "inter": 3072, + "w1": { + "dtype": "bf16" + }, + "w2": { + "dtype": "bf16" + } + } + }, + "sources": [ + "moonshotai/Kimi-K3/block_sparse_moe" + ] + }, + { + "type": "moe", + "args": { + "tokens": 16, + "activation": { + "alpha": 4.0, + "beta": 25.0, + "kind": "situ" + }, + "experts": { + "a1": { + "dtype": "bf16" + }, + "a2": { + "dtype": "bf16" + }, + "inter": 3072, + "latent": 3584, + "latent_norm": true, + "num": 896, + "top_k": 16, + "w1": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + }, + "w2": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + } + }, + "hidden": 7168, + "router": { + "bias": true, + "gate": { + "dtype": "bf16", + "logits": "fp32" + }, + "renormalize": true, + "scale": 1.0, + "scoring": "sigmoid", + "select": { + "groups": 1, + "kind": "grouped_topk", + "topk_groups": 1 + } + }, + "routing": { + "distribution": "natural", + "seed": 0 + }, + "shared": { + "a1": { + "dtype": "bf16" + }, + "count": 2, + "inter": 3072, + "w1": { + "dtype": "bf16" + }, + "w2": { + "dtype": "bf16" + } + } + }, + "sources": [ + "moonshotai/Kimi-K3/block_sparse_moe" + ] + }, + { + "type": "moe", + "args": { + "tokens": 24, + "activation": { + "alpha": 4.0, + "beta": 25.0, + "kind": "situ" + }, + "experts": { + "a1": { + "dtype": "bf16" + }, + "a2": { + "dtype": "bf16" + }, + "inter": 3072, + "latent": 3584, + "latent_norm": true, + "num": 896, + "top_k": 16, + "w1": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + }, + "w2": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + } + }, + "hidden": 7168, + "router": { + "bias": true, + "gate": { + "dtype": "bf16", + "logits": "fp32" + }, + "renormalize": true, + "scale": 1.0, + "scoring": "sigmoid", + "select": { + "groups": 1, + "kind": "grouped_topk", + "topk_groups": 1 + } + }, + "routing": { + "distribution": "natural", + "seed": 0 + }, + "shared": { + "a1": { + "dtype": "bf16" + }, + "count": 2, + "inter": 3072, + "w1": { + "dtype": "bf16" + }, + "w2": { + "dtype": "bf16" + } + } + }, + "sources": [ + "moonshotai/Kimi-K3/block_sparse_moe" + ] + }, + { + "type": "moe", + "args": { + "tokens": 32, + "activation": { + "alpha": 4.0, + "beta": 25.0, + "kind": "situ" + }, + "experts": { + "a1": { + "dtype": "bf16" + }, + "a2": { + "dtype": "bf16" + }, + "inter": 3072, + "latent": 3584, + "latent_norm": true, + "num": 896, + "top_k": 16, + "w1": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + }, + "w2": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + } + }, + "hidden": 7168, + "router": { + "bias": true, + "gate": { + "dtype": "bf16", + "logits": "fp32" + }, + "renormalize": true, + "scale": 1.0, + "scoring": "sigmoid", + "select": { + "groups": 1, + "kind": "grouped_topk", + "topk_groups": 1 + } + }, + "routing": { + "distribution": "natural", + "seed": 0 + }, + "shared": { + "a1": { + "dtype": "bf16" + }, + "count": 2, + "inter": 3072, + "w1": { + "dtype": "bf16" + }, + "w2": { + "dtype": "bf16" + } + } + }, + "sources": [ + "moonshotai/Kimi-K3/block_sparse_moe" + ] + }, + { + "type": "moe", + "args": { + "tokens": 48, + "activation": { + "alpha": 4.0, + "beta": 25.0, + "kind": "situ" + }, + "experts": { + "a1": { + "dtype": "bf16" + }, + "a2": { + "dtype": "bf16" + }, + "inter": 3072, + "latent": 3584, + "latent_norm": true, + "num": 896, + "top_k": 16, + "w1": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + }, + "w2": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + } + }, + "hidden": 7168, + "router": { + "bias": true, + "gate": { + "dtype": "bf16", + "logits": "fp32" + }, + "renormalize": true, + "scale": 1.0, + "scoring": "sigmoid", + "select": { + "groups": 1, + "kind": "grouped_topk", + "topk_groups": 1 + } + }, + "routing": { + "distribution": "natural", + "seed": 0 + }, + "shared": { + "a1": { + "dtype": "bf16" + }, + "count": 2, + "inter": 3072, + "w1": { + "dtype": "bf16" + }, + "w2": { + "dtype": "bf16" + } + } + }, + "sources": [ + "moonshotai/Kimi-K3/block_sparse_moe" + ] + }, + { + "type": "moe", + "args": { + "tokens": 64, + "activation": { + "alpha": 4.0, + "beta": 25.0, + "kind": "situ" + }, + "experts": { + "a1": { + "dtype": "bf16" + }, + "a2": { + "dtype": "bf16" + }, + "inter": 3072, + "latent": 3584, + "latent_norm": true, + "num": 896, + "top_k": 16, + "w1": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + }, + "w2": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + } + }, + "hidden": 7168, + "router": { + "bias": true, + "gate": { + "dtype": "bf16", + "logits": "fp32" + }, + "renormalize": true, + "scale": 1.0, + "scoring": "sigmoid", + "select": { + "groups": 1, + "kind": "grouped_topk", + "topk_groups": 1 + } + }, + "routing": { + "distribution": "natural", + "seed": 0 + }, + "shared": { + "a1": { + "dtype": "bf16" + }, + "count": 2, + "inter": 3072, + "w1": { + "dtype": "bf16" + }, + "w2": { + "dtype": "bf16" + } + } + }, + "sources": [ + "moonshotai/Kimi-K3/block_sparse_moe" + ] + }, + { + "type": "moe", + "args": { + "tokens": 96, + "activation": { + "alpha": 4.0, + "beta": 25.0, + "kind": "situ" + }, + "experts": { + "a1": { + "dtype": "bf16" + }, + "a2": { + "dtype": "bf16" + }, + "inter": 3072, + "latent": 3584, + "latent_norm": true, + "num": 896, + "top_k": 16, + "w1": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + }, + "w2": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + } + }, + "hidden": 7168, + "router": { + "bias": true, + "gate": { + "dtype": "bf16", + "logits": "fp32" + }, + "renormalize": true, + "scale": 1.0, + "scoring": "sigmoid", + "select": { + "groups": 1, + "kind": "grouped_topk", + "topk_groups": 1 + } + }, + "routing": { + "distribution": "natural", + "seed": 0 + }, + "shared": { + "a1": { + "dtype": "bf16" + }, + "count": 2, + "inter": 3072, + "w1": { + "dtype": "bf16" + }, + "w2": { + "dtype": "bf16" + } + } + }, + "sources": [ + "moonshotai/Kimi-K3/block_sparse_moe" + ] + }, + { + "type": "moe", + "args": { + "tokens": 128, + "activation": { + "alpha": 4.0, + "beta": 25.0, + "kind": "situ" + }, + "experts": { + "a1": { + "dtype": "bf16" + }, + "a2": { + "dtype": "bf16" + }, + "inter": 3072, + "latent": 3584, + "latent_norm": true, + "num": 896, + "top_k": 16, + "w1": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + }, + "w2": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + } + }, + "hidden": 7168, + "router": { + "bias": true, + "gate": { + "dtype": "bf16", + "logits": "fp32" + }, + "renormalize": true, + "scale": 1.0, + "scoring": "sigmoid", + "select": { + "groups": 1, + "kind": "grouped_topk", + "topk_groups": 1 + } + }, + "routing": { + "distribution": "natural", + "seed": 0 + }, + "shared": { + "a1": { + "dtype": "bf16" + }, + "count": 2, + "inter": 3072, + "w1": { + "dtype": "bf16" + }, + "w2": { + "dtype": "bf16" + } + } + }, + "sources": [ + "moonshotai/Kimi-K3/block_sparse_moe" + ] + }, + { + "type": "moe", + "args": { + "tokens": 192, + "activation": { + "alpha": 4.0, + "beta": 25.0, + "kind": "situ" + }, + "experts": { + "a1": { + "dtype": "bf16" + }, + "a2": { + "dtype": "bf16" + }, + "inter": 3072, + "latent": 3584, + "latent_norm": true, + "num": 896, + "top_k": 16, + "w1": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + }, + "w2": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + } + }, + "hidden": 7168, + "router": { + "bias": true, + "gate": { + "dtype": "bf16", + "logits": "fp32" + }, + "renormalize": true, + "scale": 1.0, + "scoring": "sigmoid", + "select": { + "groups": 1, + "kind": "grouped_topk", + "topk_groups": 1 + } + }, + "routing": { + "distribution": "natural", + "seed": 0 + }, + "shared": { + "a1": { + "dtype": "bf16" + }, + "count": 2, + "inter": 3072, + "w1": { + "dtype": "bf16" + }, + "w2": { + "dtype": "bf16" + } + } + }, + "sources": [ + "moonshotai/Kimi-K3/block_sparse_moe" + ] + }, + { + "type": "moe", + "args": { + "tokens": 256, + "activation": { + "alpha": 4.0, + "beta": 25.0, + "kind": "situ" + }, + "experts": { + "a1": { + "dtype": "bf16" + }, + "a2": { + "dtype": "bf16" + }, + "inter": 3072, + "latent": 3584, + "latent_norm": true, + "num": 896, + "top_k": 16, + "w1": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + }, + "w2": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + } + }, + "hidden": 7168, + "router": { + "bias": true, + "gate": { + "dtype": "bf16", + "logits": "fp32" + }, + "renormalize": true, + "scale": 1.0, + "scoring": "sigmoid", + "select": { + "groups": 1, + "kind": "grouped_topk", + "topk_groups": 1 + } + }, + "routing": { + "distribution": "natural", + "seed": 0 + }, + "shared": { + "a1": { + "dtype": "bf16" + }, + "count": 2, + "inter": 3072, + "w1": { + "dtype": "bf16" + }, + "w2": { + "dtype": "bf16" + } + } + }, + "sources": [ + "moonshotai/Kimi-K3/block_sparse_moe" + ] + }, + { + "type": "moe", + "args": { + "tokens": 384, + "activation": { + "alpha": 4.0, + "beta": 25.0, + "kind": "situ" + }, + "experts": { + "a1": { + "dtype": "bf16" + }, + "a2": { + "dtype": "bf16" + }, + "inter": 3072, + "latent": 3584, + "latent_norm": true, + "num": 896, + "top_k": 16, + "w1": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + }, + "w2": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + } + }, + "hidden": 7168, + "router": { + "bias": true, + "gate": { + "dtype": "bf16", + "logits": "fp32" + }, + "renormalize": true, + "scale": 1.0, + "scoring": "sigmoid", + "select": { + "groups": 1, + "kind": "grouped_topk", + "topk_groups": 1 + } + }, + "routing": { + "distribution": "natural", + "seed": 0 + }, + "shared": { + "a1": { + "dtype": "bf16" + }, + "count": 2, + "inter": 3072, + "w1": { + "dtype": "bf16" + }, + "w2": { + "dtype": "bf16" + } + } + }, + "sources": [ + "moonshotai/Kimi-K3/block_sparse_moe" + ] + }, + { + "type": "moe", + "args": { + "tokens": 512, + "activation": { + "alpha": 4.0, + "beta": 25.0, + "kind": "situ" + }, + "experts": { + "a1": { + "dtype": "bf16" + }, + "a2": { + "dtype": "bf16" + }, + "inter": 3072, + "latent": 3584, + "latent_norm": true, + "num": 896, + "top_k": 16, + "w1": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + }, + "w2": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + } + }, + "hidden": 7168, + "router": { + "bias": true, + "gate": { + "dtype": "bf16", + "logits": "fp32" + }, + "renormalize": true, + "scale": 1.0, + "scoring": "sigmoid", + "select": { + "groups": 1, + "kind": "grouped_topk", + "topk_groups": 1 + } + }, + "routing": { + "distribution": "natural", + "seed": 0 + }, + "shared": { + "a1": { + "dtype": "bf16" + }, + "count": 2, + "inter": 3072, + "w1": { + "dtype": "bf16" + }, + "w2": { + "dtype": "bf16" + } + } + }, + "sources": [ + "moonshotai/Kimi-K3/block_sparse_moe" + ] + }, + { + "type": "moe", + "args": { + "tokens": 768, + "activation": { + "alpha": 4.0, + "beta": 25.0, + "kind": "situ" + }, + "experts": { + "a1": { + "dtype": "bf16" + }, + "a2": { + "dtype": "bf16" + }, + "inter": 3072, + "latent": 3584, + "latent_norm": true, + "num": 896, + "top_k": 16, + "w1": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + }, + "w2": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + } + }, + "hidden": 7168, + "router": { + "bias": true, + "gate": { + "dtype": "bf16", + "logits": "fp32" + }, + "renormalize": true, + "scale": 1.0, + "scoring": "sigmoid", + "select": { + "groups": 1, + "kind": "grouped_topk", + "topk_groups": 1 + } + }, + "routing": { + "distribution": "natural", + "seed": 0 + }, + "shared": { + "a1": { + "dtype": "bf16" + }, + "count": 2, + "inter": 3072, + "w1": { + "dtype": "bf16" + }, + "w2": { + "dtype": "bf16" + } + } + }, + "sources": [ + "moonshotai/Kimi-K3/block_sparse_moe" + ] + }, + { + "type": "moe", + "args": { + "tokens": 1024, + "activation": { + "alpha": 4.0, + "beta": 25.0, + "kind": "situ" + }, + "experts": { + "a1": { + "dtype": "bf16" + }, + "a2": { + "dtype": "bf16" + }, + "inter": 3072, + "latent": 3584, + "latent_norm": true, + "num": 896, + "top_k": 16, + "w1": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + }, + "w2": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + } + }, + "hidden": 7168, + "router": { + "bias": true, + "gate": { + "dtype": "bf16", + "logits": "fp32" + }, + "renormalize": true, + "scale": 1.0, + "scoring": "sigmoid", + "select": { + "groups": 1, + "kind": "grouped_topk", + "topk_groups": 1 + } + }, + "routing": { + "distribution": "natural", + "seed": 0 + }, + "shared": { + "a1": { + "dtype": "bf16" + }, + "count": 2, + "inter": 3072, + "w1": { + "dtype": "bf16" + }, + "w2": { + "dtype": "bf16" + } + } + }, + "sources": [ + "moonshotai/Kimi-K3/block_sparse_moe" + ] + }, + { + "type": "moe", + "args": { + "tokens": 1536, + "activation": { + "alpha": 4.0, + "beta": 25.0, + "kind": "situ" + }, + "experts": { + "a1": { + "dtype": "bf16" + }, + "a2": { + "dtype": "bf16" + }, + "inter": 3072, + "latent": 3584, + "latent_norm": true, + "num": 896, + "top_k": 16, + "w1": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + }, + "w2": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + } + }, + "hidden": 7168, + "router": { + "bias": true, + "gate": { + "dtype": "bf16", + "logits": "fp32" + }, + "renormalize": true, + "scale": 1.0, + "scoring": "sigmoid", + "select": { + "groups": 1, + "kind": "grouped_topk", + "topk_groups": 1 + } + }, + "routing": { + "distribution": "natural", + "seed": 0 + }, + "shared": { + "a1": { + "dtype": "bf16" + }, + "count": 2, + "inter": 3072, + "w1": { + "dtype": "bf16" + }, + "w2": { + "dtype": "bf16" + } + } + }, + "sources": [ + "moonshotai/Kimi-K3/block_sparse_moe" + ] + }, + { + "type": "moe", + "args": { + "tokens": 2048, + "activation": { + "alpha": 4.0, + "beta": 25.0, + "kind": "situ" + }, + "experts": { + "a1": { + "dtype": "bf16" + }, + "a2": { + "dtype": "bf16" + }, + "inter": 3072, + "latent": 3584, + "latent_norm": true, + "num": 896, + "top_k": 16, + "w1": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + }, + "w2": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + } + }, + "hidden": 7168, + "router": { + "bias": true, + "gate": { + "dtype": "bf16", + "logits": "fp32" + }, + "renormalize": true, + "scale": 1.0, + "scoring": "sigmoid", + "select": { + "groups": 1, + "kind": "grouped_topk", + "topk_groups": 1 + } + }, + "routing": { + "distribution": "natural", + "seed": 0 + }, + "shared": { + "a1": { + "dtype": "bf16" + }, + "count": 2, + "inter": 3072, + "w1": { + "dtype": "bf16" + }, + "w2": { + "dtype": "bf16" + } + } + }, + "sources": [ + "moonshotai/Kimi-K3/block_sparse_moe" + ] + }, + { + "type": "moe", + "args": { + "tokens": 3072, + "activation": { + "alpha": 4.0, + "beta": 25.0, + "kind": "situ" + }, + "experts": { + "a1": { + "dtype": "bf16" + }, + "a2": { + "dtype": "bf16" + }, + "inter": 3072, + "latent": 3584, + "latent_norm": true, + "num": 896, + "top_k": 16, + "w1": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + }, + "w2": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + } + }, + "hidden": 7168, + "router": { + "bias": true, + "gate": { + "dtype": "bf16", + "logits": "fp32" + }, + "renormalize": true, + "scale": 1.0, + "scoring": "sigmoid", + "select": { + "groups": 1, + "kind": "grouped_topk", + "topk_groups": 1 + } + }, + "routing": { + "distribution": "natural", + "seed": 0 + }, + "shared": { + "a1": { + "dtype": "bf16" + }, + "count": 2, + "inter": 3072, + "w1": { + "dtype": "bf16" + }, + "w2": { + "dtype": "bf16" + } + } + }, + "sources": [ + "moonshotai/Kimi-K3/block_sparse_moe" + ] + }, + { + "type": "moe", + "args": { + "tokens": 4096, + "activation": { + "alpha": 4.0, + "beta": 25.0, + "kind": "situ" + }, + "experts": { + "a1": { + "dtype": "bf16" + }, + "a2": { + "dtype": "bf16" + }, + "inter": 3072, + "latent": 3584, + "latent_norm": true, + "num": 896, + "top_k": 16, + "w1": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + }, + "w2": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + } + }, + "hidden": 7168, + "router": { + "bias": true, + "gate": { + "dtype": "bf16", + "logits": "fp32" + }, + "renormalize": true, + "scale": 1.0, + "scoring": "sigmoid", + "select": { + "groups": 1, + "kind": "grouped_topk", + "topk_groups": 1 + } + }, + "routing": { + "distribution": "natural", + "seed": 0 + }, + "shared": { + "a1": { + "dtype": "bf16" + }, + "count": 2, + "inter": 3072, + "w1": { + "dtype": "bf16" + }, + "w2": { + "dtype": "bf16" + } + } + }, + "sources": [ + "moonshotai/Kimi-K3/block_sparse_moe" + ] + }, + { + "type": "moe", + "args": { + "tokens": 6144, + "activation": { + "alpha": 4.0, + "beta": 25.0, + "kind": "situ" + }, + "experts": { + "a1": { + "dtype": "bf16" + }, + "a2": { + "dtype": "bf16" + }, + "inter": 3072, + "latent": 3584, + "latent_norm": true, + "num": 896, + "top_k": 16, + "w1": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + }, + "w2": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + } + }, + "hidden": 7168, + "router": { + "bias": true, + "gate": { + "dtype": "bf16", + "logits": "fp32" + }, + "renormalize": true, + "scale": 1.0, + "scoring": "sigmoid", + "select": { + "groups": 1, + "kind": "grouped_topk", + "topk_groups": 1 + } + }, + "routing": { + "distribution": "natural", + "seed": 0 + }, + "shared": { + "a1": { + "dtype": "bf16" + }, + "count": 2, + "inter": 3072, + "w1": { + "dtype": "bf16" + }, + "w2": { + "dtype": "bf16" + } + } + }, + "sources": [ + "moonshotai/Kimi-K3/block_sparse_moe" + ] + }, + { + "type": "moe", + "args": { + "tokens": 8192, + "activation": { + "alpha": 4.0, + "beta": 25.0, + "kind": "situ" + }, + "experts": { + "a1": { + "dtype": "bf16" + }, + "a2": { + "dtype": "bf16" + }, + "inter": 3072, + "latent": 3584, + "latent_norm": true, + "num": 896, + "top_k": 16, + "w1": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + }, + "w2": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + } + }, + "hidden": 7168, + "router": { + "bias": true, + "gate": { + "dtype": "bf16", + "logits": "fp32" + }, + "renormalize": true, + "scale": 1.0, + "scoring": "sigmoid", + "select": { + "groups": 1, + "kind": "grouped_topk", + "topk_groups": 1 + } + }, + "routing": { + "distribution": "natural", + "seed": 0 + }, + "shared": { + "a1": { + "dtype": "bf16" + }, + "count": 2, + "inter": 3072, + "w1": { + "dtype": "bf16" + }, + "w2": { + "dtype": "bf16" + } + } + }, + "sources": [ + "moonshotai/Kimi-K3/block_sparse_moe" + ] + }, + { + "type": "moe", + "args": { + "tokens": 12288, + "activation": { + "alpha": 4.0, + "beta": 25.0, + "kind": "situ" + }, + "experts": { + "a1": { + "dtype": "bf16" + }, + "a2": { + "dtype": "bf16" + }, + "inter": 3072, + "latent": 3584, + "latent_norm": true, + "num": 896, + "top_k": 16, + "w1": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + }, + "w2": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + } + }, + "hidden": 7168, + "router": { + "bias": true, + "gate": { + "dtype": "bf16", + "logits": "fp32" + }, + "renormalize": true, + "scale": 1.0, + "scoring": "sigmoid", + "select": { + "groups": 1, + "kind": "grouped_topk", + "topk_groups": 1 + } + }, + "routing": { + "distribution": "natural", + "seed": 0 + }, + "shared": { + "a1": { + "dtype": "bf16" + }, + "count": 2, + "inter": 3072, + "w1": { + "dtype": "bf16" + }, + "w2": { + "dtype": "bf16" + } + } + }, + "sources": [ + "moonshotai/Kimi-K3/block_sparse_moe" + ] + }, + { + "type": "moe", + "args": { + "tokens": 16384, + "activation": { + "alpha": 4.0, + "beta": 25.0, + "kind": "situ" + }, + "experts": { + "a1": { + "dtype": "bf16" + }, + "a2": { + "dtype": "bf16" + }, + "inter": 3072, + "latent": 3584, + "latent_norm": true, + "num": 896, + "top_k": 16, + "w1": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + }, + "w2": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + } + }, + "hidden": 7168, + "router": { + "bias": true, + "gate": { + "dtype": "bf16", + "logits": "fp32" + }, + "renormalize": true, + "scale": 1.0, + "scoring": "sigmoid", + "select": { + "groups": 1, + "kind": "grouped_topk", + "topk_groups": 1 + } + }, + "routing": { + "distribution": "natural", + "seed": 0 + }, + "shared": { + "a1": { + "dtype": "bf16" + }, + "count": 2, + "inter": 3072, + "w1": { + "dtype": "bf16" + }, + "w2": { + "dtype": "bf16" + } + } + }, + "sources": [ + "moonshotai/Kimi-K3/block_sparse_moe" + ] + }, + { + "type": "moe", + "args": { + "tokens": 24576, + "activation": { + "alpha": 4.0, + "beta": 25.0, + "kind": "situ" + }, + "experts": { + "a1": { + "dtype": "bf16" + }, + "a2": { + "dtype": "bf16" + }, + "inter": 3072, + "latent": 3584, + "latent_norm": true, + "num": 896, + "top_k": 16, + "w1": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + }, + "w2": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + } + }, + "hidden": 7168, + "router": { + "bias": true, + "gate": { + "dtype": "bf16", + "logits": "fp32" + }, + "renormalize": true, + "scale": 1.0, + "scoring": "sigmoid", + "select": { + "groups": 1, + "kind": "grouped_topk", + "topk_groups": 1 + } + }, + "routing": { + "distribution": "natural", + "seed": 0 + }, + "shared": { + "a1": { + "dtype": "bf16" + }, + "count": 2, + "inter": 3072, + "w1": { + "dtype": "bf16" + }, + "w2": { + "dtype": "bf16" + } + } + }, + "sources": [ + "moonshotai/Kimi-K3/block_sparse_moe" + ] + }, + { + "type": "moe", + "args": { + "tokens": 32768, + "activation": { + "alpha": 4.0, + "beta": 25.0, + "kind": "situ" + }, + "experts": { + "a1": { + "dtype": "bf16" + }, + "a2": { + "dtype": "bf16" + }, + "inter": 3072, + "latent": 3584, + "latent_norm": true, + "num": 896, + "top_k": 16, + "w1": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + }, + "w2": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + } + }, + "hidden": 7168, + "router": { + "bias": true, + "gate": { + "dtype": "bf16", + "logits": "fp32" + }, + "renormalize": true, + "scale": 1.0, + "scoring": "sigmoid", + "select": { + "groups": 1, + "kind": "grouped_topk", + "topk_groups": 1 + } + }, + "routing": { + "distribution": "natural", + "seed": 0 + }, + "shared": { + "a1": { + "dtype": "bf16" + }, + "count": 2, + "inter": 3072, + "w1": { + "dtype": "bf16" + }, + "w2": { + "dtype": "bf16" + } + } + }, + "sources": [ + "moonshotai/Kimi-K3/block_sparse_moe" + ] + }, + { + "type": "moe", + "args": { + "tokens": 49152, + "activation": { + "alpha": 4.0, + "beta": 25.0, + "kind": "situ" + }, + "experts": { + "a1": { + "dtype": "bf16" + }, + "a2": { + "dtype": "bf16" + }, + "inter": 3072, + "latent": 3584, + "latent_norm": true, + "num": 896, + "top_k": 16, + "w1": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + }, + "w2": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + } + }, + "hidden": 7168, + "router": { + "bias": true, + "gate": { + "dtype": "bf16", + "logits": "fp32" + }, + "renormalize": true, + "scale": 1.0, + "scoring": "sigmoid", + "select": { + "groups": 1, + "kind": "grouped_topk", + "topk_groups": 1 + } + }, + "routing": { + "distribution": "natural", + "seed": 0 + }, + "shared": { + "a1": { + "dtype": "bf16" + }, + "count": 2, + "inter": 3072, + "w1": { + "dtype": "bf16" + }, + "w2": { + "dtype": "bf16" + } + } + }, + "sources": [ + "moonshotai/Kimi-K3/block_sparse_moe" + ] + }, + { + "type": "moe", + "args": { + "tokens": 65536, + "activation": { + "alpha": 4.0, + "beta": 25.0, + "kind": "situ" + }, + "experts": { + "a1": { + "dtype": "bf16" + }, + "a2": { + "dtype": "bf16" + }, + "inter": 3072, + "latent": 3584, + "latent_norm": true, + "num": 896, + "top_k": 16, + "w1": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + }, + "w2": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + } + }, + "hidden": 7168, + "router": { + "bias": true, + "gate": { + "dtype": "bf16", + "logits": "fp32" + }, + "renormalize": true, + "scale": 1.0, + "scoring": "sigmoid", + "select": { + "groups": 1, + "kind": "grouped_topk", + "topk_groups": 1 + } + }, + "routing": { + "distribution": "natural", + "seed": 0 + }, + "shared": { + "a1": { + "dtype": "bf16" + }, + "count": 2, + "inter": 3072, + "w1": { + "dtype": "bf16" + }, + "w2": { + "dtype": "bf16" + } + } + }, + "sources": [ + "moonshotai/Kimi-K3/block_sparse_moe" + ] + } +] diff --git a/operatorx/testlists/moe_minimax_m3.json b/operatorx/testlists/moe_minimax_m3.json new file mode 100644 index 0000000000..1fe387f8c7 --- /dev/null +++ b/operatorx/testlists/moe_minimax_m3.json @@ -0,0 +1,12354 @@ +[ + { + "type": "moe", + "args": { + "tokens": 1, + "activation": { + "alpha": 1.702, + "beta": 1.0, + "kind": "swigluoai", + "limit": 7.0 + }, + "experts": { + "a1": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "a2": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "inter": 3072, + "num": 128, + "top_k": 4, + "w1": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + }, + "w2": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + } + }, + "hidden": 6144, + "router": { + "bias": true, + "gate": { + "dtype": "fp32" + }, + "renormalize": true, + "scale": 2.0, + "scoring": "sigmoid", + "select": { + "kind": "topk" + } + }, + "routing": { + "distribution": "natural", + "seed": 0 + }, + "shared": { + "a1": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "count": 1, + "inter": 3072, + "w1": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + }, + "w2": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + } + } + }, + "sources": [ + "MiniMaxAI/MiniMax-M3-MXFP8/block_sparse_moe" + ] + }, + { + "type": "moe", + "args": { + "tokens": 2, + "activation": { + "alpha": 1.702, + "beta": 1.0, + "kind": "swigluoai", + "limit": 7.0 + }, + "experts": { + "a1": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "a2": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "inter": 3072, + "num": 128, + "top_k": 4, + "w1": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + }, + "w2": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + } + }, + "hidden": 6144, + "router": { + "bias": true, + "gate": { + "dtype": "fp32" + }, + "renormalize": true, + "scale": 2.0, + "scoring": "sigmoid", + "select": { + "kind": "topk" + } + }, + "routing": { + "distribution": "natural", + "seed": 0 + }, + "shared": { + "a1": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "count": 1, + "inter": 3072, + "w1": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + }, + "w2": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + } + } + }, + "sources": [ + "MiniMaxAI/MiniMax-M3-MXFP8/block_sparse_moe" + ] + }, + { + "type": "moe", + "args": { + "tokens": 3, + "activation": { + "alpha": 1.702, + "beta": 1.0, + "kind": "swigluoai", + "limit": 7.0 + }, + "experts": { + "a1": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "a2": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "inter": 3072, + "num": 128, + "top_k": 4, + "w1": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + }, + "w2": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + } + }, + "hidden": 6144, + "router": { + "bias": true, + "gate": { + "dtype": "fp32" + }, + "renormalize": true, + "scale": 2.0, + "scoring": "sigmoid", + "select": { + "kind": "topk" + } + }, + "routing": { + "distribution": "natural", + "seed": 0 + }, + "shared": { + "a1": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "count": 1, + "inter": 3072, + "w1": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + }, + "w2": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + } + } + }, + "sources": [ + "MiniMaxAI/MiniMax-M3-MXFP8/block_sparse_moe" + ] + }, + { + "type": "moe", + "args": { + "tokens": 4, + "activation": { + "alpha": 1.702, + "beta": 1.0, + "kind": "swigluoai", + "limit": 7.0 + }, + "experts": { + "a1": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "a2": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "inter": 3072, + "num": 128, + "top_k": 4, + "w1": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + }, + "w2": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + } + }, + "hidden": 6144, + "router": { + "bias": true, + "gate": { + "dtype": "fp32" + }, + "renormalize": true, + "scale": 2.0, + "scoring": "sigmoid", + "select": { + "kind": "topk" + } + }, + "routing": { + "distribution": "natural", + "seed": 0 + }, + "shared": { + "a1": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "count": 1, + "inter": 3072, + "w1": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + }, + "w2": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + } + } + }, + "sources": [ + "MiniMaxAI/MiniMax-M3-MXFP8/block_sparse_moe" + ] + }, + { + "type": "moe", + "args": { + "tokens": 6, + "activation": { + "alpha": 1.702, + "beta": 1.0, + "kind": "swigluoai", + "limit": 7.0 + }, + "experts": { + "a1": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "a2": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "inter": 3072, + "num": 128, + "top_k": 4, + "w1": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + }, + "w2": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + } + }, + "hidden": 6144, + "router": { + "bias": true, + "gate": { + "dtype": "fp32" + }, + "renormalize": true, + "scale": 2.0, + "scoring": "sigmoid", + "select": { + "kind": "topk" + } + }, + "routing": { + "distribution": "natural", + "seed": 0 + }, + "shared": { + "a1": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "count": 1, + "inter": 3072, + "w1": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + }, + "w2": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + } + } + }, + "sources": [ + "MiniMaxAI/MiniMax-M3-MXFP8/block_sparse_moe" + ] + }, + { + "type": "moe", + "args": { + "tokens": 8, + "activation": { + "alpha": 1.702, + "beta": 1.0, + "kind": "swigluoai", + "limit": 7.0 + }, + "experts": { + "a1": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "a2": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "inter": 3072, + "num": 128, + "top_k": 4, + "w1": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + }, + "w2": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + } + }, + "hidden": 6144, + "router": { + "bias": true, + "gate": { + "dtype": "fp32" + }, + "renormalize": true, + "scale": 2.0, + "scoring": "sigmoid", + "select": { + "kind": "topk" + } + }, + "routing": { + "distribution": "natural", + "seed": 0 + }, + "shared": { + "a1": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "count": 1, + "inter": 3072, + "w1": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + }, + "w2": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + } + } + }, + "sources": [ + "MiniMaxAI/MiniMax-M3-MXFP8/block_sparse_moe" + ] + }, + { + "type": "moe", + "args": { + "tokens": 12, + "activation": { + "alpha": 1.702, + "beta": 1.0, + "kind": "swigluoai", + "limit": 7.0 + }, + "experts": { + "a1": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "a2": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "inter": 3072, + "num": 128, + "top_k": 4, + "w1": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + }, + "w2": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + } + }, + "hidden": 6144, + "router": { + "bias": true, + "gate": { + "dtype": "fp32" + }, + "renormalize": true, + "scale": 2.0, + "scoring": "sigmoid", + "select": { + "kind": "topk" + } + }, + "routing": { + "distribution": "natural", + "seed": 0 + }, + "shared": { + "a1": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "count": 1, + "inter": 3072, + "w1": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + }, + "w2": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + } + } + }, + "sources": [ + "MiniMaxAI/MiniMax-M3-MXFP8/block_sparse_moe" + ] + }, + { + "type": "moe", + "args": { + "tokens": 16, + "activation": { + "alpha": 1.702, + "beta": 1.0, + "kind": "swigluoai", + "limit": 7.0 + }, + "experts": { + "a1": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "a2": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "inter": 3072, + "num": 128, + "top_k": 4, + "w1": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + }, + "w2": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + } + }, + "hidden": 6144, + "router": { + "bias": true, + "gate": { + "dtype": "fp32" + }, + "renormalize": true, + "scale": 2.0, + "scoring": "sigmoid", + "select": { + "kind": "topk" + } + }, + "routing": { + "distribution": "natural", + "seed": 0 + }, + "shared": { + "a1": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "count": 1, + "inter": 3072, + "w1": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + }, + "w2": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + } + } + }, + "sources": [ + "MiniMaxAI/MiniMax-M3-MXFP8/block_sparse_moe" + ] + }, + { + "type": "moe", + "args": { + "tokens": 24, + "activation": { + "alpha": 1.702, + "beta": 1.0, + "kind": "swigluoai", + "limit": 7.0 + }, + "experts": { + "a1": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "a2": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "inter": 3072, + "num": 128, + "top_k": 4, + "w1": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + }, + "w2": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + } + }, + "hidden": 6144, + "router": { + "bias": true, + "gate": { + "dtype": "fp32" + }, + "renormalize": true, + "scale": 2.0, + "scoring": "sigmoid", + "select": { + "kind": "topk" + } + }, + "routing": { + "distribution": "natural", + "seed": 0 + }, + "shared": { + "a1": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "count": 1, + "inter": 3072, + "w1": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + }, + "w2": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + } + } + }, + "sources": [ + "MiniMaxAI/MiniMax-M3-MXFP8/block_sparse_moe" + ] + }, + { + "type": "moe", + "args": { + "tokens": 32, + "activation": { + "alpha": 1.702, + "beta": 1.0, + "kind": "swigluoai", + "limit": 7.0 + }, + "experts": { + "a1": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "a2": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "inter": 3072, + "num": 128, + "top_k": 4, + "w1": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + }, + "w2": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + } + }, + "hidden": 6144, + "router": { + "bias": true, + "gate": { + "dtype": "fp32" + }, + "renormalize": true, + "scale": 2.0, + "scoring": "sigmoid", + "select": { + "kind": "topk" + } + }, + "routing": { + "distribution": "natural", + "seed": 0 + }, + "shared": { + "a1": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "count": 1, + "inter": 3072, + "w1": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + }, + "w2": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + } + } + }, + "sources": [ + "MiniMaxAI/MiniMax-M3-MXFP8/block_sparse_moe" + ] + }, + { + "type": "moe", + "args": { + "tokens": 48, + "activation": { + "alpha": 1.702, + "beta": 1.0, + "kind": "swigluoai", + "limit": 7.0 + }, + "experts": { + "a1": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "a2": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "inter": 3072, + "num": 128, + "top_k": 4, + "w1": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + }, + "w2": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + } + }, + "hidden": 6144, + "router": { + "bias": true, + "gate": { + "dtype": "fp32" + }, + "renormalize": true, + "scale": 2.0, + "scoring": "sigmoid", + "select": { + "kind": "topk" + } + }, + "routing": { + "distribution": "natural", + "seed": 0 + }, + "shared": { + "a1": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "count": 1, + "inter": 3072, + "w1": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + }, + "w2": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + } + } + }, + "sources": [ + "MiniMaxAI/MiniMax-M3-MXFP8/block_sparse_moe" + ] + }, + { + "type": "moe", + "args": { + "tokens": 64, + "activation": { + "alpha": 1.702, + "beta": 1.0, + "kind": "swigluoai", + "limit": 7.0 + }, + "experts": { + "a1": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "a2": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "inter": 3072, + "num": 128, + "top_k": 4, + "w1": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + }, + "w2": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + } + }, + "hidden": 6144, + "router": { + "bias": true, + "gate": { + "dtype": "fp32" + }, + "renormalize": true, + "scale": 2.0, + "scoring": "sigmoid", + "select": { + "kind": "topk" + } + }, + "routing": { + "distribution": "natural", + "seed": 0 + }, + "shared": { + "a1": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "count": 1, + "inter": 3072, + "w1": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + }, + "w2": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + } + } + }, + "sources": [ + "MiniMaxAI/MiniMax-M3-MXFP8/block_sparse_moe" + ] + }, + { + "type": "moe", + "args": { + "tokens": 96, + "activation": { + "alpha": 1.702, + "beta": 1.0, + "kind": "swigluoai", + "limit": 7.0 + }, + "experts": { + "a1": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "a2": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "inter": 3072, + "num": 128, + "top_k": 4, + "w1": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + }, + "w2": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + } + }, + "hidden": 6144, + "router": { + "bias": true, + "gate": { + "dtype": "fp32" + }, + "renormalize": true, + "scale": 2.0, + "scoring": "sigmoid", + "select": { + "kind": "topk" + } + }, + "routing": { + "distribution": "natural", + "seed": 0 + }, + "shared": { + "a1": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "count": 1, + "inter": 3072, + "w1": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + }, + "w2": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + } + } + }, + "sources": [ + "MiniMaxAI/MiniMax-M3-MXFP8/block_sparse_moe" + ] + }, + { + "type": "moe", + "args": { + "tokens": 128, + "activation": { + "alpha": 1.702, + "beta": 1.0, + "kind": "swigluoai", + "limit": 7.0 + }, + "experts": { + "a1": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "a2": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "inter": 3072, + "num": 128, + "top_k": 4, + "w1": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + }, + "w2": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + } + }, + "hidden": 6144, + "router": { + "bias": true, + "gate": { + "dtype": "fp32" + }, + "renormalize": true, + "scale": 2.0, + "scoring": "sigmoid", + "select": { + "kind": "topk" + } + }, + "routing": { + "distribution": "natural", + "seed": 0 + }, + "shared": { + "a1": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "count": 1, + "inter": 3072, + "w1": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + }, + "w2": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + } + } + }, + "sources": [ + "MiniMaxAI/MiniMax-M3-MXFP8/block_sparse_moe" + ] + }, + { + "type": "moe", + "args": { + "tokens": 192, + "activation": { + "alpha": 1.702, + "beta": 1.0, + "kind": "swigluoai", + "limit": 7.0 + }, + "experts": { + "a1": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "a2": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "inter": 3072, + "num": 128, + "top_k": 4, + "w1": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + }, + "w2": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + } + }, + "hidden": 6144, + "router": { + "bias": true, + "gate": { + "dtype": "fp32" + }, + "renormalize": true, + "scale": 2.0, + "scoring": "sigmoid", + "select": { + "kind": "topk" + } + }, + "routing": { + "distribution": "natural", + "seed": 0 + }, + "shared": { + "a1": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "count": 1, + "inter": 3072, + "w1": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + }, + "w2": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + } + } + }, + "sources": [ + "MiniMaxAI/MiniMax-M3-MXFP8/block_sparse_moe" + ] + }, + { + "type": "moe", + "args": { + "tokens": 256, + "activation": { + "alpha": 1.702, + "beta": 1.0, + "kind": "swigluoai", + "limit": 7.0 + }, + "experts": { + "a1": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "a2": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "inter": 3072, + "num": 128, + "top_k": 4, + "w1": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + }, + "w2": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + } + }, + "hidden": 6144, + "router": { + "bias": true, + "gate": { + "dtype": "fp32" + }, + "renormalize": true, + "scale": 2.0, + "scoring": "sigmoid", + "select": { + "kind": "topk" + } + }, + "routing": { + "distribution": "natural", + "seed": 0 + }, + "shared": { + "a1": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "count": 1, + "inter": 3072, + "w1": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + }, + "w2": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + } + } + }, + "sources": [ + "MiniMaxAI/MiniMax-M3-MXFP8/block_sparse_moe" + ] + }, + { + "type": "moe", + "args": { + "tokens": 384, + "activation": { + "alpha": 1.702, + "beta": 1.0, + "kind": "swigluoai", + "limit": 7.0 + }, + "experts": { + "a1": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "a2": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "inter": 3072, + "num": 128, + "top_k": 4, + "w1": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + }, + "w2": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + } + }, + "hidden": 6144, + "router": { + "bias": true, + "gate": { + "dtype": "fp32" + }, + "renormalize": true, + "scale": 2.0, + "scoring": "sigmoid", + "select": { + "kind": "topk" + } + }, + "routing": { + "distribution": "natural", + "seed": 0 + }, + "shared": { + "a1": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "count": 1, + "inter": 3072, + "w1": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + }, + "w2": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + } + } + }, + "sources": [ + "MiniMaxAI/MiniMax-M3-MXFP8/block_sparse_moe" + ] + }, + { + "type": "moe", + "args": { + "tokens": 512, + "activation": { + "alpha": 1.702, + "beta": 1.0, + "kind": "swigluoai", + "limit": 7.0 + }, + "experts": { + "a1": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "a2": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "inter": 3072, + "num": 128, + "top_k": 4, + "w1": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + }, + "w2": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + } + }, + "hidden": 6144, + "router": { + "bias": true, + "gate": { + "dtype": "fp32" + }, + "renormalize": true, + "scale": 2.0, + "scoring": "sigmoid", + "select": { + "kind": "topk" + } + }, + "routing": { + "distribution": "natural", + "seed": 0 + }, + "shared": { + "a1": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "count": 1, + "inter": 3072, + "w1": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + }, + "w2": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + } + } + }, + "sources": [ + "MiniMaxAI/MiniMax-M3-MXFP8/block_sparse_moe" + ] + }, + { + "type": "moe", + "args": { + "tokens": 768, + "activation": { + "alpha": 1.702, + "beta": 1.0, + "kind": "swigluoai", + "limit": 7.0 + }, + "experts": { + "a1": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "a2": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "inter": 3072, + "num": 128, + "top_k": 4, + "w1": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + }, + "w2": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + } + }, + "hidden": 6144, + "router": { + "bias": true, + "gate": { + "dtype": "fp32" + }, + "renormalize": true, + "scale": 2.0, + "scoring": "sigmoid", + "select": { + "kind": "topk" + } + }, + "routing": { + "distribution": "natural", + "seed": 0 + }, + "shared": { + "a1": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "count": 1, + "inter": 3072, + "w1": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + }, + "w2": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + } + } + }, + "sources": [ + "MiniMaxAI/MiniMax-M3-MXFP8/block_sparse_moe" + ] + }, + { + "type": "moe", + "args": { + "tokens": 1024, + "activation": { + "alpha": 1.702, + "beta": 1.0, + "kind": "swigluoai", + "limit": 7.0 + }, + "experts": { + "a1": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "a2": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "inter": 3072, + "num": 128, + "top_k": 4, + "w1": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + }, + "w2": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + } + }, + "hidden": 6144, + "router": { + "bias": true, + "gate": { + "dtype": "fp32" + }, + "renormalize": true, + "scale": 2.0, + "scoring": "sigmoid", + "select": { + "kind": "topk" + } + }, + "routing": { + "distribution": "natural", + "seed": 0 + }, + "shared": { + "a1": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "count": 1, + "inter": 3072, + "w1": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + }, + "w2": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + } + } + }, + "sources": [ + "MiniMaxAI/MiniMax-M3-MXFP8/block_sparse_moe" + ] + }, + { + "type": "moe", + "args": { + "tokens": 1536, + "activation": { + "alpha": 1.702, + "beta": 1.0, + "kind": "swigluoai", + "limit": 7.0 + }, + "experts": { + "a1": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "a2": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "inter": 3072, + "num": 128, + "top_k": 4, + "w1": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + }, + "w2": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + } + }, + "hidden": 6144, + "router": { + "bias": true, + "gate": { + "dtype": "fp32" + }, + "renormalize": true, + "scale": 2.0, + "scoring": "sigmoid", + "select": { + "kind": "topk" + } + }, + "routing": { + "distribution": "natural", + "seed": 0 + }, + "shared": { + "a1": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "count": 1, + "inter": 3072, + "w1": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + }, + "w2": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + } + } + }, + "sources": [ + "MiniMaxAI/MiniMax-M3-MXFP8/block_sparse_moe" + ] + }, + { + "type": "moe", + "args": { + "tokens": 2048, + "activation": { + "alpha": 1.702, + "beta": 1.0, + "kind": "swigluoai", + "limit": 7.0 + }, + "experts": { + "a1": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "a2": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "inter": 3072, + "num": 128, + "top_k": 4, + "w1": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + }, + "w2": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + } + }, + "hidden": 6144, + "router": { + "bias": true, + "gate": { + "dtype": "fp32" + }, + "renormalize": true, + "scale": 2.0, + "scoring": "sigmoid", + "select": { + "kind": "topk" + } + }, + "routing": { + "distribution": "natural", + "seed": 0 + }, + "shared": { + "a1": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "count": 1, + "inter": 3072, + "w1": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + }, + "w2": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + } + } + }, + "sources": [ + "MiniMaxAI/MiniMax-M3-MXFP8/block_sparse_moe" + ] + }, + { + "type": "moe", + "args": { + "tokens": 3072, + "activation": { + "alpha": 1.702, + "beta": 1.0, + "kind": "swigluoai", + "limit": 7.0 + }, + "experts": { + "a1": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "a2": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "inter": 3072, + "num": 128, + "top_k": 4, + "w1": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + }, + "w2": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + } + }, + "hidden": 6144, + "router": { + "bias": true, + "gate": { + "dtype": "fp32" + }, + "renormalize": true, + "scale": 2.0, + "scoring": "sigmoid", + "select": { + "kind": "topk" + } + }, + "routing": { + "distribution": "natural", + "seed": 0 + }, + "shared": { + "a1": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "count": 1, + "inter": 3072, + "w1": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + }, + "w2": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + } + } + }, + "sources": [ + "MiniMaxAI/MiniMax-M3-MXFP8/block_sparse_moe" + ] + }, + { + "type": "moe", + "args": { + "tokens": 4096, + "activation": { + "alpha": 1.702, + "beta": 1.0, + "kind": "swigluoai", + "limit": 7.0 + }, + "experts": { + "a1": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "a2": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "inter": 3072, + "num": 128, + "top_k": 4, + "w1": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + }, + "w2": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + } + }, + "hidden": 6144, + "router": { + "bias": true, + "gate": { + "dtype": "fp32" + }, + "renormalize": true, + "scale": 2.0, + "scoring": "sigmoid", + "select": { + "kind": "topk" + } + }, + "routing": { + "distribution": "natural", + "seed": 0 + }, + "shared": { + "a1": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "count": 1, + "inter": 3072, + "w1": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + }, + "w2": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + } + } + }, + "sources": [ + "MiniMaxAI/MiniMax-M3-MXFP8/block_sparse_moe" + ] + }, + { + "type": "moe", + "args": { + "tokens": 6144, + "activation": { + "alpha": 1.702, + "beta": 1.0, + "kind": "swigluoai", + "limit": 7.0 + }, + "experts": { + "a1": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "a2": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "inter": 3072, + "num": 128, + "top_k": 4, + "w1": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + }, + "w2": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + } + }, + "hidden": 6144, + "router": { + "bias": true, + "gate": { + "dtype": "fp32" + }, + "renormalize": true, + "scale": 2.0, + "scoring": "sigmoid", + "select": { + "kind": "topk" + } + }, + "routing": { + "distribution": "natural", + "seed": 0 + }, + "shared": { + "a1": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "count": 1, + "inter": 3072, + "w1": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + }, + "w2": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + } + } + }, + "sources": [ + "MiniMaxAI/MiniMax-M3-MXFP8/block_sparse_moe" + ] + }, + { + "type": "moe", + "args": { + "tokens": 8192, + "activation": { + "alpha": 1.702, + "beta": 1.0, + "kind": "swigluoai", + "limit": 7.0 + }, + "experts": { + "a1": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "a2": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "inter": 3072, + "num": 128, + "top_k": 4, + "w1": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + }, + "w2": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + } + }, + "hidden": 6144, + "router": { + "bias": true, + "gate": { + "dtype": "fp32" + }, + "renormalize": true, + "scale": 2.0, + "scoring": "sigmoid", + "select": { + "kind": "topk" + } + }, + "routing": { + "distribution": "natural", + "seed": 0 + }, + "shared": { + "a1": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "count": 1, + "inter": 3072, + "w1": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + }, + "w2": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + } + } + }, + "sources": [ + "MiniMaxAI/MiniMax-M3-MXFP8/block_sparse_moe" + ] + }, + { + "type": "moe", + "args": { + "tokens": 12288, + "activation": { + "alpha": 1.702, + "beta": 1.0, + "kind": "swigluoai", + "limit": 7.0 + }, + "experts": { + "a1": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "a2": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "inter": 3072, + "num": 128, + "top_k": 4, + "w1": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + }, + "w2": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + } + }, + "hidden": 6144, + "router": { + "bias": true, + "gate": { + "dtype": "fp32" + }, + "renormalize": true, + "scale": 2.0, + "scoring": "sigmoid", + "select": { + "kind": "topk" + } + }, + "routing": { + "distribution": "natural", + "seed": 0 + }, + "shared": { + "a1": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "count": 1, + "inter": 3072, + "w1": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + }, + "w2": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + } + } + }, + "sources": [ + "MiniMaxAI/MiniMax-M3-MXFP8/block_sparse_moe" + ] + }, + { + "type": "moe", + "args": { + "tokens": 16384, + "activation": { + "alpha": 1.702, + "beta": 1.0, + "kind": "swigluoai", + "limit": 7.0 + }, + "experts": { + "a1": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "a2": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "inter": 3072, + "num": 128, + "top_k": 4, + "w1": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + }, + "w2": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + } + }, + "hidden": 6144, + "router": { + "bias": true, + "gate": { + "dtype": "fp32" + }, + "renormalize": true, + "scale": 2.0, + "scoring": "sigmoid", + "select": { + "kind": "topk" + } + }, + "routing": { + "distribution": "natural", + "seed": 0 + }, + "shared": { + "a1": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "count": 1, + "inter": 3072, + "w1": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + }, + "w2": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + } + } + }, + "sources": [ + "MiniMaxAI/MiniMax-M3-MXFP8/block_sparse_moe" + ] + }, + { + "type": "moe", + "args": { + "tokens": 24576, + "activation": { + "alpha": 1.702, + "beta": 1.0, + "kind": "swigluoai", + "limit": 7.0 + }, + "experts": { + "a1": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "a2": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "inter": 3072, + "num": 128, + "top_k": 4, + "w1": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + }, + "w2": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + } + }, + "hidden": 6144, + "router": { + "bias": true, + "gate": { + "dtype": "fp32" + }, + "renormalize": true, + "scale": 2.0, + "scoring": "sigmoid", + "select": { + "kind": "topk" + } + }, + "routing": { + "distribution": "natural", + "seed": 0 + }, + "shared": { + "a1": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "count": 1, + "inter": 3072, + "w1": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + }, + "w2": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + } + } + }, + "sources": [ + "MiniMaxAI/MiniMax-M3-MXFP8/block_sparse_moe" + ] + }, + { + "type": "moe", + "args": { + "tokens": 32768, + "activation": { + "alpha": 1.702, + "beta": 1.0, + "kind": "swigluoai", + "limit": 7.0 + }, + "experts": { + "a1": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "a2": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "inter": 3072, + "num": 128, + "top_k": 4, + "w1": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + }, + "w2": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + } + }, + "hidden": 6144, + "router": { + "bias": true, + "gate": { + "dtype": "fp32" + }, + "renormalize": true, + "scale": 2.0, + "scoring": "sigmoid", + "select": { + "kind": "topk" + } + }, + "routing": { + "distribution": "natural", + "seed": 0 + }, + "shared": { + "a1": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "count": 1, + "inter": 3072, + "w1": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + }, + "w2": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + } + } + }, + "sources": [ + "MiniMaxAI/MiniMax-M3-MXFP8/block_sparse_moe" + ] + }, + { + "type": "moe", + "args": { + "tokens": 49152, + "activation": { + "alpha": 1.702, + "beta": 1.0, + "kind": "swigluoai", + "limit": 7.0 + }, + "experts": { + "a1": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "a2": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "inter": 3072, + "num": 128, + "top_k": 4, + "w1": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + }, + "w2": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + } + }, + "hidden": 6144, + "router": { + "bias": true, + "gate": { + "dtype": "fp32" + }, + "renormalize": true, + "scale": 2.0, + "scoring": "sigmoid", + "select": { + "kind": "topk" + } + }, + "routing": { + "distribution": "natural", + "seed": 0 + }, + "shared": { + "a1": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "count": 1, + "inter": 3072, + "w1": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + }, + "w2": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + } + } + }, + "sources": [ + "MiniMaxAI/MiniMax-M3-MXFP8/block_sparse_moe" + ] + }, + { + "type": "moe", + "args": { + "tokens": 65536, + "activation": { + "alpha": 1.702, + "beta": 1.0, + "kind": "swigluoai", + "limit": 7.0 + }, + "experts": { + "a1": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "a2": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "inter": 3072, + "num": 128, + "top_k": 4, + "w1": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + }, + "w2": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + } + }, + "hidden": 6144, + "router": { + "bias": true, + "gate": { + "dtype": "fp32" + }, + "renormalize": true, + "scale": 2.0, + "scoring": "sigmoid", + "select": { + "kind": "topk" + } + }, + "routing": { + "distribution": "natural", + "seed": 0 + }, + "shared": { + "a1": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "count": 1, + "inter": 3072, + "w1": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + }, + "w2": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + } + } + }, + "sources": [ + "MiniMaxAI/MiniMax-M3-MXFP8/block_sparse_moe" + ] + }, + { + "type": "moe", + "args": { + "tokens": 1, + "activation": { + "alpha": 1.702, + "beta": 1.0, + "kind": "swigluoai", + "limit": 7.0 + }, + "experts": { + "a1": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "group": [ + 1, + 16 + ], + "static": false + }, + "scale2": { + "dtype": "fp32", + "group": [ + -1, + -1 + ], + "static": true + } + }, + "a2": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "group": [ + 1, + 16 + ], + "static": false + }, + "scale2": { + "dtype": "fp32", + "group": [ + -1, + -1 + ], + "static": true + } + }, + "inter": 3072, + "num": 128, + "top_k": 4, + "w1": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "group": [ + 1, + 16 + ], + "static": true + }, + "scale2": { + "dtype": "fp32", + "group": [ + -1, + -1 + ], + "static": true + } + }, + "w2": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "group": [ + 1, + 16 + ], + "static": true + }, + "scale2": { + "dtype": "fp32", + "group": [ + -1, + -1 + ], + "static": true + } + } + }, + "hidden": 6144, + "router": { + "bias": true, + "gate": { + "dtype": "fp32" + }, + "renormalize": true, + "scale": 2.0, + "scoring": "sigmoid", + "select": { + "kind": "topk" + } + }, + "routing": { + "distribution": "natural", + "seed": 0 + }, + "shared": { + "a1": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "count": 1, + "inter": 3072, + "w1": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + }, + "w2": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + } + } + }, + "sources": [ + "nvidia/MiniMax-M3-NVFP4/block_sparse_moe" + ] + }, + { + "type": "moe", + "args": { + "tokens": 2, + "activation": { + "alpha": 1.702, + "beta": 1.0, + "kind": "swigluoai", + "limit": 7.0 + }, + "experts": { + "a1": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "group": [ + 1, + 16 + ], + "static": false + }, + "scale2": { + "dtype": "fp32", + "group": [ + -1, + -1 + ], + "static": true + } + }, + "a2": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "group": [ + 1, + 16 + ], + "static": false + }, + "scale2": { + "dtype": "fp32", + "group": [ + -1, + -1 + ], + "static": true + } + }, + "inter": 3072, + "num": 128, + "top_k": 4, + "w1": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "group": [ + 1, + 16 + ], + "static": true + }, + "scale2": { + "dtype": "fp32", + "group": [ + -1, + -1 + ], + "static": true + } + }, + "w2": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "group": [ + 1, + 16 + ], + "static": true + }, + "scale2": { + "dtype": "fp32", + "group": [ + -1, + -1 + ], + "static": true + } + } + }, + "hidden": 6144, + "router": { + "bias": true, + "gate": { + "dtype": "fp32" + }, + "renormalize": true, + "scale": 2.0, + "scoring": "sigmoid", + "select": { + "kind": "topk" + } + }, + "routing": { + "distribution": "natural", + "seed": 0 + }, + "shared": { + "a1": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "count": 1, + "inter": 3072, + "w1": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + }, + "w2": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + } + } + }, + "sources": [ + "nvidia/MiniMax-M3-NVFP4/block_sparse_moe" + ] + }, + { + "type": "moe", + "args": { + "tokens": 3, + "activation": { + "alpha": 1.702, + "beta": 1.0, + "kind": "swigluoai", + "limit": 7.0 + }, + "experts": { + "a1": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "group": [ + 1, + 16 + ], + "static": false + }, + "scale2": { + "dtype": "fp32", + "group": [ + -1, + -1 + ], + "static": true + } + }, + "a2": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "group": [ + 1, + 16 + ], + "static": false + }, + "scale2": { + "dtype": "fp32", + "group": [ + -1, + -1 + ], + "static": true + } + }, + "inter": 3072, + "num": 128, + "top_k": 4, + "w1": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "group": [ + 1, + 16 + ], + "static": true + }, + "scale2": { + "dtype": "fp32", + "group": [ + -1, + -1 + ], + "static": true + } + }, + "w2": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "group": [ + 1, + 16 + ], + "static": true + }, + "scale2": { + "dtype": "fp32", + "group": [ + -1, + -1 + ], + "static": true + } + } + }, + "hidden": 6144, + "router": { + "bias": true, + "gate": { + "dtype": "fp32" + }, + "renormalize": true, + "scale": 2.0, + "scoring": "sigmoid", + "select": { + "kind": "topk" + } + }, + "routing": { + "distribution": "natural", + "seed": 0 + }, + "shared": { + "a1": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "count": 1, + "inter": 3072, + "w1": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + }, + "w2": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + } + } + }, + "sources": [ + "nvidia/MiniMax-M3-NVFP4/block_sparse_moe" + ] + }, + { + "type": "moe", + "args": { + "tokens": 4, + "activation": { + "alpha": 1.702, + "beta": 1.0, + "kind": "swigluoai", + "limit": 7.0 + }, + "experts": { + "a1": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "group": [ + 1, + 16 + ], + "static": false + }, + "scale2": { + "dtype": "fp32", + "group": [ + -1, + -1 + ], + "static": true + } + }, + "a2": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "group": [ + 1, + 16 + ], + "static": false + }, + "scale2": { + "dtype": "fp32", + "group": [ + -1, + -1 + ], + "static": true + } + }, + "inter": 3072, + "num": 128, + "top_k": 4, + "w1": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "group": [ + 1, + 16 + ], + "static": true + }, + "scale2": { + "dtype": "fp32", + "group": [ + -1, + -1 + ], + "static": true + } + }, + "w2": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "group": [ + 1, + 16 + ], + "static": true + }, + "scale2": { + "dtype": "fp32", + "group": [ + -1, + -1 + ], + "static": true + } + } + }, + "hidden": 6144, + "router": { + "bias": true, + "gate": { + "dtype": "fp32" + }, + "renormalize": true, + "scale": 2.0, + "scoring": "sigmoid", + "select": { + "kind": "topk" + } + }, + "routing": { + "distribution": "natural", + "seed": 0 + }, + "shared": { + "a1": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "count": 1, + "inter": 3072, + "w1": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + }, + "w2": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + } + } + }, + "sources": [ + "nvidia/MiniMax-M3-NVFP4/block_sparse_moe" + ] + }, + { + "type": "moe", + "args": { + "tokens": 6, + "activation": { + "alpha": 1.702, + "beta": 1.0, + "kind": "swigluoai", + "limit": 7.0 + }, + "experts": { + "a1": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "group": [ + 1, + 16 + ], + "static": false + }, + "scale2": { + "dtype": "fp32", + "group": [ + -1, + -1 + ], + "static": true + } + }, + "a2": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "group": [ + 1, + 16 + ], + "static": false + }, + "scale2": { + "dtype": "fp32", + "group": [ + -1, + -1 + ], + "static": true + } + }, + "inter": 3072, + "num": 128, + "top_k": 4, + "w1": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "group": [ + 1, + 16 + ], + "static": true + }, + "scale2": { + "dtype": "fp32", + "group": [ + -1, + -1 + ], + "static": true + } + }, + "w2": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "group": [ + 1, + 16 + ], + "static": true + }, + "scale2": { + "dtype": "fp32", + "group": [ + -1, + -1 + ], + "static": true + } + } + }, + "hidden": 6144, + "router": { + "bias": true, + "gate": { + "dtype": "fp32" + }, + "renormalize": true, + "scale": 2.0, + "scoring": "sigmoid", + "select": { + "kind": "topk" + } + }, + "routing": { + "distribution": "natural", + "seed": 0 + }, + "shared": { + "a1": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "count": 1, + "inter": 3072, + "w1": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + }, + "w2": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + } + } + }, + "sources": [ + "nvidia/MiniMax-M3-NVFP4/block_sparse_moe" + ] + }, + { + "type": "moe", + "args": { + "tokens": 8, + "activation": { + "alpha": 1.702, + "beta": 1.0, + "kind": "swigluoai", + "limit": 7.0 + }, + "experts": { + "a1": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "group": [ + 1, + 16 + ], + "static": false + }, + "scale2": { + "dtype": "fp32", + "group": [ + -1, + -1 + ], + "static": true + } + }, + "a2": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "group": [ + 1, + 16 + ], + "static": false + }, + "scale2": { + "dtype": "fp32", + "group": [ + -1, + -1 + ], + "static": true + } + }, + "inter": 3072, + "num": 128, + "top_k": 4, + "w1": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "group": [ + 1, + 16 + ], + "static": true + }, + "scale2": { + "dtype": "fp32", + "group": [ + -1, + -1 + ], + "static": true + } + }, + "w2": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "group": [ + 1, + 16 + ], + "static": true + }, + "scale2": { + "dtype": "fp32", + "group": [ + -1, + -1 + ], + "static": true + } + } + }, + "hidden": 6144, + "router": { + "bias": true, + "gate": { + "dtype": "fp32" + }, + "renormalize": true, + "scale": 2.0, + "scoring": "sigmoid", + "select": { + "kind": "topk" + } + }, + "routing": { + "distribution": "natural", + "seed": 0 + }, + "shared": { + "a1": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "count": 1, + "inter": 3072, + "w1": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + }, + "w2": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + } + } + }, + "sources": [ + "nvidia/MiniMax-M3-NVFP4/block_sparse_moe" + ] + }, + { + "type": "moe", + "args": { + "tokens": 12, + "activation": { + "alpha": 1.702, + "beta": 1.0, + "kind": "swigluoai", + "limit": 7.0 + }, + "experts": { + "a1": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "group": [ + 1, + 16 + ], + "static": false + }, + "scale2": { + "dtype": "fp32", + "group": [ + -1, + -1 + ], + "static": true + } + }, + "a2": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "group": [ + 1, + 16 + ], + "static": false + }, + "scale2": { + "dtype": "fp32", + "group": [ + -1, + -1 + ], + "static": true + } + }, + "inter": 3072, + "num": 128, + "top_k": 4, + "w1": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "group": [ + 1, + 16 + ], + "static": true + }, + "scale2": { + "dtype": "fp32", + "group": [ + -1, + -1 + ], + "static": true + } + }, + "w2": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "group": [ + 1, + 16 + ], + "static": true + }, + "scale2": { + "dtype": "fp32", + "group": [ + -1, + -1 + ], + "static": true + } + } + }, + "hidden": 6144, + "router": { + "bias": true, + "gate": { + "dtype": "fp32" + }, + "renormalize": true, + "scale": 2.0, + "scoring": "sigmoid", + "select": { + "kind": "topk" + } + }, + "routing": { + "distribution": "natural", + "seed": 0 + }, + "shared": { + "a1": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "count": 1, + "inter": 3072, + "w1": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + }, + "w2": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + } + } + }, + "sources": [ + "nvidia/MiniMax-M3-NVFP4/block_sparse_moe" + ] + }, + { + "type": "moe", + "args": { + "tokens": 16, + "activation": { + "alpha": 1.702, + "beta": 1.0, + "kind": "swigluoai", + "limit": 7.0 + }, + "experts": { + "a1": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "group": [ + 1, + 16 + ], + "static": false + }, + "scale2": { + "dtype": "fp32", + "group": [ + -1, + -1 + ], + "static": true + } + }, + "a2": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "group": [ + 1, + 16 + ], + "static": false + }, + "scale2": { + "dtype": "fp32", + "group": [ + -1, + -1 + ], + "static": true + } + }, + "inter": 3072, + "num": 128, + "top_k": 4, + "w1": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "group": [ + 1, + 16 + ], + "static": true + }, + "scale2": { + "dtype": "fp32", + "group": [ + -1, + -1 + ], + "static": true + } + }, + "w2": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "group": [ + 1, + 16 + ], + "static": true + }, + "scale2": { + "dtype": "fp32", + "group": [ + -1, + -1 + ], + "static": true + } + } + }, + "hidden": 6144, + "router": { + "bias": true, + "gate": { + "dtype": "fp32" + }, + "renormalize": true, + "scale": 2.0, + "scoring": "sigmoid", + "select": { + "kind": "topk" + } + }, + "routing": { + "distribution": "natural", + "seed": 0 + }, + "shared": { + "a1": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "count": 1, + "inter": 3072, + "w1": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + }, + "w2": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + } + } + }, + "sources": [ + "nvidia/MiniMax-M3-NVFP4/block_sparse_moe" + ] + }, + { + "type": "moe", + "args": { + "tokens": 24, + "activation": { + "alpha": 1.702, + "beta": 1.0, + "kind": "swigluoai", + "limit": 7.0 + }, + "experts": { + "a1": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "group": [ + 1, + 16 + ], + "static": false + }, + "scale2": { + "dtype": "fp32", + "group": [ + -1, + -1 + ], + "static": true + } + }, + "a2": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "group": [ + 1, + 16 + ], + "static": false + }, + "scale2": { + "dtype": "fp32", + "group": [ + -1, + -1 + ], + "static": true + } + }, + "inter": 3072, + "num": 128, + "top_k": 4, + "w1": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "group": [ + 1, + 16 + ], + "static": true + }, + "scale2": { + "dtype": "fp32", + "group": [ + -1, + -1 + ], + "static": true + } + }, + "w2": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "group": [ + 1, + 16 + ], + "static": true + }, + "scale2": { + "dtype": "fp32", + "group": [ + -1, + -1 + ], + "static": true + } + } + }, + "hidden": 6144, + "router": { + "bias": true, + "gate": { + "dtype": "fp32" + }, + "renormalize": true, + "scale": 2.0, + "scoring": "sigmoid", + "select": { + "kind": "topk" + } + }, + "routing": { + "distribution": "natural", + "seed": 0 + }, + "shared": { + "a1": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "count": 1, + "inter": 3072, + "w1": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + }, + "w2": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + } + } + }, + "sources": [ + "nvidia/MiniMax-M3-NVFP4/block_sparse_moe" + ] + }, + { + "type": "moe", + "args": { + "tokens": 32, + "activation": { + "alpha": 1.702, + "beta": 1.0, + "kind": "swigluoai", + "limit": 7.0 + }, + "experts": { + "a1": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "group": [ + 1, + 16 + ], + "static": false + }, + "scale2": { + "dtype": "fp32", + "group": [ + -1, + -1 + ], + "static": true + } + }, + "a2": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "group": [ + 1, + 16 + ], + "static": false + }, + "scale2": { + "dtype": "fp32", + "group": [ + -1, + -1 + ], + "static": true + } + }, + "inter": 3072, + "num": 128, + "top_k": 4, + "w1": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "group": [ + 1, + 16 + ], + "static": true + }, + "scale2": { + "dtype": "fp32", + "group": [ + -1, + -1 + ], + "static": true + } + }, + "w2": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "group": [ + 1, + 16 + ], + "static": true + }, + "scale2": { + "dtype": "fp32", + "group": [ + -1, + -1 + ], + "static": true + } + } + }, + "hidden": 6144, + "router": { + "bias": true, + "gate": { + "dtype": "fp32" + }, + "renormalize": true, + "scale": 2.0, + "scoring": "sigmoid", + "select": { + "kind": "topk" + } + }, + "routing": { + "distribution": "natural", + "seed": 0 + }, + "shared": { + "a1": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "count": 1, + "inter": 3072, + "w1": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + }, + "w2": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + } + } + }, + "sources": [ + "nvidia/MiniMax-M3-NVFP4/block_sparse_moe" + ] + }, + { + "type": "moe", + "args": { + "tokens": 48, + "activation": { + "alpha": 1.702, + "beta": 1.0, + "kind": "swigluoai", + "limit": 7.0 + }, + "experts": { + "a1": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "group": [ + 1, + 16 + ], + "static": false + }, + "scale2": { + "dtype": "fp32", + "group": [ + -1, + -1 + ], + "static": true + } + }, + "a2": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "group": [ + 1, + 16 + ], + "static": false + }, + "scale2": { + "dtype": "fp32", + "group": [ + -1, + -1 + ], + "static": true + } + }, + "inter": 3072, + "num": 128, + "top_k": 4, + "w1": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "group": [ + 1, + 16 + ], + "static": true + }, + "scale2": { + "dtype": "fp32", + "group": [ + -1, + -1 + ], + "static": true + } + }, + "w2": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "group": [ + 1, + 16 + ], + "static": true + }, + "scale2": { + "dtype": "fp32", + "group": [ + -1, + -1 + ], + "static": true + } + } + }, + "hidden": 6144, + "router": { + "bias": true, + "gate": { + "dtype": "fp32" + }, + "renormalize": true, + "scale": 2.0, + "scoring": "sigmoid", + "select": { + "kind": "topk" + } + }, + "routing": { + "distribution": "natural", + "seed": 0 + }, + "shared": { + "a1": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "count": 1, + "inter": 3072, + "w1": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + }, + "w2": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + } + } + }, + "sources": [ + "nvidia/MiniMax-M3-NVFP4/block_sparse_moe" + ] + }, + { + "type": "moe", + "args": { + "tokens": 64, + "activation": { + "alpha": 1.702, + "beta": 1.0, + "kind": "swigluoai", + "limit": 7.0 + }, + "experts": { + "a1": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "group": [ + 1, + 16 + ], + "static": false + }, + "scale2": { + "dtype": "fp32", + "group": [ + -1, + -1 + ], + "static": true + } + }, + "a2": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "group": [ + 1, + 16 + ], + "static": false + }, + "scale2": { + "dtype": "fp32", + "group": [ + -1, + -1 + ], + "static": true + } + }, + "inter": 3072, + "num": 128, + "top_k": 4, + "w1": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "group": [ + 1, + 16 + ], + "static": true + }, + "scale2": { + "dtype": "fp32", + "group": [ + -1, + -1 + ], + "static": true + } + }, + "w2": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "group": [ + 1, + 16 + ], + "static": true + }, + "scale2": { + "dtype": "fp32", + "group": [ + -1, + -1 + ], + "static": true + } + } + }, + "hidden": 6144, + "router": { + "bias": true, + "gate": { + "dtype": "fp32" + }, + "renormalize": true, + "scale": 2.0, + "scoring": "sigmoid", + "select": { + "kind": "topk" + } + }, + "routing": { + "distribution": "natural", + "seed": 0 + }, + "shared": { + "a1": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "count": 1, + "inter": 3072, + "w1": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + }, + "w2": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + } + } + }, + "sources": [ + "nvidia/MiniMax-M3-NVFP4/block_sparse_moe" + ] + }, + { + "type": "moe", + "args": { + "tokens": 96, + "activation": { + "alpha": 1.702, + "beta": 1.0, + "kind": "swigluoai", + "limit": 7.0 + }, + "experts": { + "a1": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "group": [ + 1, + 16 + ], + "static": false + }, + "scale2": { + "dtype": "fp32", + "group": [ + -1, + -1 + ], + "static": true + } + }, + "a2": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "group": [ + 1, + 16 + ], + "static": false + }, + "scale2": { + "dtype": "fp32", + "group": [ + -1, + -1 + ], + "static": true + } + }, + "inter": 3072, + "num": 128, + "top_k": 4, + "w1": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "group": [ + 1, + 16 + ], + "static": true + }, + "scale2": { + "dtype": "fp32", + "group": [ + -1, + -1 + ], + "static": true + } + }, + "w2": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "group": [ + 1, + 16 + ], + "static": true + }, + "scale2": { + "dtype": "fp32", + "group": [ + -1, + -1 + ], + "static": true + } + } + }, + "hidden": 6144, + "router": { + "bias": true, + "gate": { + "dtype": "fp32" + }, + "renormalize": true, + "scale": 2.0, + "scoring": "sigmoid", + "select": { + "kind": "topk" + } + }, + "routing": { + "distribution": "natural", + "seed": 0 + }, + "shared": { + "a1": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "count": 1, + "inter": 3072, + "w1": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + }, + "w2": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + } + } + }, + "sources": [ + "nvidia/MiniMax-M3-NVFP4/block_sparse_moe" + ] + }, + { + "type": "moe", + "args": { + "tokens": 128, + "activation": { + "alpha": 1.702, + "beta": 1.0, + "kind": "swigluoai", + "limit": 7.0 + }, + "experts": { + "a1": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "group": [ + 1, + 16 + ], + "static": false + }, + "scale2": { + "dtype": "fp32", + "group": [ + -1, + -1 + ], + "static": true + } + }, + "a2": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "group": [ + 1, + 16 + ], + "static": false + }, + "scale2": { + "dtype": "fp32", + "group": [ + -1, + -1 + ], + "static": true + } + }, + "inter": 3072, + "num": 128, + "top_k": 4, + "w1": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "group": [ + 1, + 16 + ], + "static": true + }, + "scale2": { + "dtype": "fp32", + "group": [ + -1, + -1 + ], + "static": true + } + }, + "w2": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "group": [ + 1, + 16 + ], + "static": true + }, + "scale2": { + "dtype": "fp32", + "group": [ + -1, + -1 + ], + "static": true + } + } + }, + "hidden": 6144, + "router": { + "bias": true, + "gate": { + "dtype": "fp32" + }, + "renormalize": true, + "scale": 2.0, + "scoring": "sigmoid", + "select": { + "kind": "topk" + } + }, + "routing": { + "distribution": "natural", + "seed": 0 + }, + "shared": { + "a1": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "count": 1, + "inter": 3072, + "w1": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + }, + "w2": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + } + } + }, + "sources": [ + "nvidia/MiniMax-M3-NVFP4/block_sparse_moe" + ] + }, + { + "type": "moe", + "args": { + "tokens": 192, + "activation": { + "alpha": 1.702, + "beta": 1.0, + "kind": "swigluoai", + "limit": 7.0 + }, + "experts": { + "a1": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "group": [ + 1, + 16 + ], + "static": false + }, + "scale2": { + "dtype": "fp32", + "group": [ + -1, + -1 + ], + "static": true + } + }, + "a2": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "group": [ + 1, + 16 + ], + "static": false + }, + "scale2": { + "dtype": "fp32", + "group": [ + -1, + -1 + ], + "static": true + } + }, + "inter": 3072, + "num": 128, + "top_k": 4, + "w1": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "group": [ + 1, + 16 + ], + "static": true + }, + "scale2": { + "dtype": "fp32", + "group": [ + -1, + -1 + ], + "static": true + } + }, + "w2": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "group": [ + 1, + 16 + ], + "static": true + }, + "scale2": { + "dtype": "fp32", + "group": [ + -1, + -1 + ], + "static": true + } + } + }, + "hidden": 6144, + "router": { + "bias": true, + "gate": { + "dtype": "fp32" + }, + "renormalize": true, + "scale": 2.0, + "scoring": "sigmoid", + "select": { + "kind": "topk" + } + }, + "routing": { + "distribution": "natural", + "seed": 0 + }, + "shared": { + "a1": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "count": 1, + "inter": 3072, + "w1": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + }, + "w2": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + } + } + }, + "sources": [ + "nvidia/MiniMax-M3-NVFP4/block_sparse_moe" + ] + }, + { + "type": "moe", + "args": { + "tokens": 256, + "activation": { + "alpha": 1.702, + "beta": 1.0, + "kind": "swigluoai", + "limit": 7.0 + }, + "experts": { + "a1": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "group": [ + 1, + 16 + ], + "static": false + }, + "scale2": { + "dtype": "fp32", + "group": [ + -1, + -1 + ], + "static": true + } + }, + "a2": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "group": [ + 1, + 16 + ], + "static": false + }, + "scale2": { + "dtype": "fp32", + "group": [ + -1, + -1 + ], + "static": true + } + }, + "inter": 3072, + "num": 128, + "top_k": 4, + "w1": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "group": [ + 1, + 16 + ], + "static": true + }, + "scale2": { + "dtype": "fp32", + "group": [ + -1, + -1 + ], + "static": true + } + }, + "w2": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "group": [ + 1, + 16 + ], + "static": true + }, + "scale2": { + "dtype": "fp32", + "group": [ + -1, + -1 + ], + "static": true + } + } + }, + "hidden": 6144, + "router": { + "bias": true, + "gate": { + "dtype": "fp32" + }, + "renormalize": true, + "scale": 2.0, + "scoring": "sigmoid", + "select": { + "kind": "topk" + } + }, + "routing": { + "distribution": "natural", + "seed": 0 + }, + "shared": { + "a1": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "count": 1, + "inter": 3072, + "w1": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + }, + "w2": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + } + } + }, + "sources": [ + "nvidia/MiniMax-M3-NVFP4/block_sparse_moe" + ] + }, + { + "type": "moe", + "args": { + "tokens": 384, + "activation": { + "alpha": 1.702, + "beta": 1.0, + "kind": "swigluoai", + "limit": 7.0 + }, + "experts": { + "a1": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "group": [ + 1, + 16 + ], + "static": false + }, + "scale2": { + "dtype": "fp32", + "group": [ + -1, + -1 + ], + "static": true + } + }, + "a2": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "group": [ + 1, + 16 + ], + "static": false + }, + "scale2": { + "dtype": "fp32", + "group": [ + -1, + -1 + ], + "static": true + } + }, + "inter": 3072, + "num": 128, + "top_k": 4, + "w1": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "group": [ + 1, + 16 + ], + "static": true + }, + "scale2": { + "dtype": "fp32", + "group": [ + -1, + -1 + ], + "static": true + } + }, + "w2": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "group": [ + 1, + 16 + ], + "static": true + }, + "scale2": { + "dtype": "fp32", + "group": [ + -1, + -1 + ], + "static": true + } + } + }, + "hidden": 6144, + "router": { + "bias": true, + "gate": { + "dtype": "fp32" + }, + "renormalize": true, + "scale": 2.0, + "scoring": "sigmoid", + "select": { + "kind": "topk" + } + }, + "routing": { + "distribution": "natural", + "seed": 0 + }, + "shared": { + "a1": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "count": 1, + "inter": 3072, + "w1": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + }, + "w2": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + } + } + }, + "sources": [ + "nvidia/MiniMax-M3-NVFP4/block_sparse_moe" + ] + }, + { + "type": "moe", + "args": { + "tokens": 512, + "activation": { + "alpha": 1.702, + "beta": 1.0, + "kind": "swigluoai", + "limit": 7.0 + }, + "experts": { + "a1": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "group": [ + 1, + 16 + ], + "static": false + }, + "scale2": { + "dtype": "fp32", + "group": [ + -1, + -1 + ], + "static": true + } + }, + "a2": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "group": [ + 1, + 16 + ], + "static": false + }, + "scale2": { + "dtype": "fp32", + "group": [ + -1, + -1 + ], + "static": true + } + }, + "inter": 3072, + "num": 128, + "top_k": 4, + "w1": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "group": [ + 1, + 16 + ], + "static": true + }, + "scale2": { + "dtype": "fp32", + "group": [ + -1, + -1 + ], + "static": true + } + }, + "w2": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "group": [ + 1, + 16 + ], + "static": true + }, + "scale2": { + "dtype": "fp32", + "group": [ + -1, + -1 + ], + "static": true + } + } + }, + "hidden": 6144, + "router": { + "bias": true, + "gate": { + "dtype": "fp32" + }, + "renormalize": true, + "scale": 2.0, + "scoring": "sigmoid", + "select": { + "kind": "topk" + } + }, + "routing": { + "distribution": "natural", + "seed": 0 + }, + "shared": { + "a1": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "count": 1, + "inter": 3072, + "w1": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + }, + "w2": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + } + } + }, + "sources": [ + "nvidia/MiniMax-M3-NVFP4/block_sparse_moe" + ] + }, + { + "type": "moe", + "args": { + "tokens": 768, + "activation": { + "alpha": 1.702, + "beta": 1.0, + "kind": "swigluoai", + "limit": 7.0 + }, + "experts": { + "a1": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "group": [ + 1, + 16 + ], + "static": false + }, + "scale2": { + "dtype": "fp32", + "group": [ + -1, + -1 + ], + "static": true + } + }, + "a2": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "group": [ + 1, + 16 + ], + "static": false + }, + "scale2": { + "dtype": "fp32", + "group": [ + -1, + -1 + ], + "static": true + } + }, + "inter": 3072, + "num": 128, + "top_k": 4, + "w1": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "group": [ + 1, + 16 + ], + "static": true + }, + "scale2": { + "dtype": "fp32", + "group": [ + -1, + -1 + ], + "static": true + } + }, + "w2": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "group": [ + 1, + 16 + ], + "static": true + }, + "scale2": { + "dtype": "fp32", + "group": [ + -1, + -1 + ], + "static": true + } + } + }, + "hidden": 6144, + "router": { + "bias": true, + "gate": { + "dtype": "fp32" + }, + "renormalize": true, + "scale": 2.0, + "scoring": "sigmoid", + "select": { + "kind": "topk" + } + }, + "routing": { + "distribution": "natural", + "seed": 0 + }, + "shared": { + "a1": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "count": 1, + "inter": 3072, + "w1": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + }, + "w2": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + } + } + }, + "sources": [ + "nvidia/MiniMax-M3-NVFP4/block_sparse_moe" + ] + }, + { + "type": "moe", + "args": { + "tokens": 1024, + "activation": { + "alpha": 1.702, + "beta": 1.0, + "kind": "swigluoai", + "limit": 7.0 + }, + "experts": { + "a1": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "group": [ + 1, + 16 + ], + "static": false + }, + "scale2": { + "dtype": "fp32", + "group": [ + -1, + -1 + ], + "static": true + } + }, + "a2": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "group": [ + 1, + 16 + ], + "static": false + }, + "scale2": { + "dtype": "fp32", + "group": [ + -1, + -1 + ], + "static": true + } + }, + "inter": 3072, + "num": 128, + "top_k": 4, + "w1": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "group": [ + 1, + 16 + ], + "static": true + }, + "scale2": { + "dtype": "fp32", + "group": [ + -1, + -1 + ], + "static": true + } + }, + "w2": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "group": [ + 1, + 16 + ], + "static": true + }, + "scale2": { + "dtype": "fp32", + "group": [ + -1, + -1 + ], + "static": true + } + } + }, + "hidden": 6144, + "router": { + "bias": true, + "gate": { + "dtype": "fp32" + }, + "renormalize": true, + "scale": 2.0, + "scoring": "sigmoid", + "select": { + "kind": "topk" + } + }, + "routing": { + "distribution": "natural", + "seed": 0 + }, + "shared": { + "a1": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "count": 1, + "inter": 3072, + "w1": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + }, + "w2": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + } + } + }, + "sources": [ + "nvidia/MiniMax-M3-NVFP4/block_sparse_moe" + ] + }, + { + "type": "moe", + "args": { + "tokens": 1536, + "activation": { + "alpha": 1.702, + "beta": 1.0, + "kind": "swigluoai", + "limit": 7.0 + }, + "experts": { + "a1": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "group": [ + 1, + 16 + ], + "static": false + }, + "scale2": { + "dtype": "fp32", + "group": [ + -1, + -1 + ], + "static": true + } + }, + "a2": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "group": [ + 1, + 16 + ], + "static": false + }, + "scale2": { + "dtype": "fp32", + "group": [ + -1, + -1 + ], + "static": true + } + }, + "inter": 3072, + "num": 128, + "top_k": 4, + "w1": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "group": [ + 1, + 16 + ], + "static": true + }, + "scale2": { + "dtype": "fp32", + "group": [ + -1, + -1 + ], + "static": true + } + }, + "w2": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "group": [ + 1, + 16 + ], + "static": true + }, + "scale2": { + "dtype": "fp32", + "group": [ + -1, + -1 + ], + "static": true + } + } + }, + "hidden": 6144, + "router": { + "bias": true, + "gate": { + "dtype": "fp32" + }, + "renormalize": true, + "scale": 2.0, + "scoring": "sigmoid", + "select": { + "kind": "topk" + } + }, + "routing": { + "distribution": "natural", + "seed": 0 + }, + "shared": { + "a1": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "count": 1, + "inter": 3072, + "w1": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + }, + "w2": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + } + } + }, + "sources": [ + "nvidia/MiniMax-M3-NVFP4/block_sparse_moe" + ] + }, + { + "type": "moe", + "args": { + "tokens": 2048, + "activation": { + "alpha": 1.702, + "beta": 1.0, + "kind": "swigluoai", + "limit": 7.0 + }, + "experts": { + "a1": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "group": [ + 1, + 16 + ], + "static": false + }, + "scale2": { + "dtype": "fp32", + "group": [ + -1, + -1 + ], + "static": true + } + }, + "a2": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "group": [ + 1, + 16 + ], + "static": false + }, + "scale2": { + "dtype": "fp32", + "group": [ + -1, + -1 + ], + "static": true + } + }, + "inter": 3072, + "num": 128, + "top_k": 4, + "w1": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "group": [ + 1, + 16 + ], + "static": true + }, + "scale2": { + "dtype": "fp32", + "group": [ + -1, + -1 + ], + "static": true + } + }, + "w2": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "group": [ + 1, + 16 + ], + "static": true + }, + "scale2": { + "dtype": "fp32", + "group": [ + -1, + -1 + ], + "static": true + } + } + }, + "hidden": 6144, + "router": { + "bias": true, + "gate": { + "dtype": "fp32" + }, + "renormalize": true, + "scale": 2.0, + "scoring": "sigmoid", + "select": { + "kind": "topk" + } + }, + "routing": { + "distribution": "natural", + "seed": 0 + }, + "shared": { + "a1": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "count": 1, + "inter": 3072, + "w1": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + }, + "w2": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + } + } + }, + "sources": [ + "nvidia/MiniMax-M3-NVFP4/block_sparse_moe" + ] + }, + { + "type": "moe", + "args": { + "tokens": 3072, + "activation": { + "alpha": 1.702, + "beta": 1.0, + "kind": "swigluoai", + "limit": 7.0 + }, + "experts": { + "a1": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "group": [ + 1, + 16 + ], + "static": false + }, + "scale2": { + "dtype": "fp32", + "group": [ + -1, + -1 + ], + "static": true + } + }, + "a2": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "group": [ + 1, + 16 + ], + "static": false + }, + "scale2": { + "dtype": "fp32", + "group": [ + -1, + -1 + ], + "static": true + } + }, + "inter": 3072, + "num": 128, + "top_k": 4, + "w1": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "group": [ + 1, + 16 + ], + "static": true + }, + "scale2": { + "dtype": "fp32", + "group": [ + -1, + -1 + ], + "static": true + } + }, + "w2": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "group": [ + 1, + 16 + ], + "static": true + }, + "scale2": { + "dtype": "fp32", + "group": [ + -1, + -1 + ], + "static": true + } + } + }, + "hidden": 6144, + "router": { + "bias": true, + "gate": { + "dtype": "fp32" + }, + "renormalize": true, + "scale": 2.0, + "scoring": "sigmoid", + "select": { + "kind": "topk" + } + }, + "routing": { + "distribution": "natural", + "seed": 0 + }, + "shared": { + "a1": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "count": 1, + "inter": 3072, + "w1": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + }, + "w2": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + } + } + }, + "sources": [ + "nvidia/MiniMax-M3-NVFP4/block_sparse_moe" + ] + }, + { + "type": "moe", + "args": { + "tokens": 4096, + "activation": { + "alpha": 1.702, + "beta": 1.0, + "kind": "swigluoai", + "limit": 7.0 + }, + "experts": { + "a1": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "group": [ + 1, + 16 + ], + "static": false + }, + "scale2": { + "dtype": "fp32", + "group": [ + -1, + -1 + ], + "static": true + } + }, + "a2": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "group": [ + 1, + 16 + ], + "static": false + }, + "scale2": { + "dtype": "fp32", + "group": [ + -1, + -1 + ], + "static": true + } + }, + "inter": 3072, + "num": 128, + "top_k": 4, + "w1": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "group": [ + 1, + 16 + ], + "static": true + }, + "scale2": { + "dtype": "fp32", + "group": [ + -1, + -1 + ], + "static": true + } + }, + "w2": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "group": [ + 1, + 16 + ], + "static": true + }, + "scale2": { + "dtype": "fp32", + "group": [ + -1, + -1 + ], + "static": true + } + } + }, + "hidden": 6144, + "router": { + "bias": true, + "gate": { + "dtype": "fp32" + }, + "renormalize": true, + "scale": 2.0, + "scoring": "sigmoid", + "select": { + "kind": "topk" + } + }, + "routing": { + "distribution": "natural", + "seed": 0 + }, + "shared": { + "a1": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "count": 1, + "inter": 3072, + "w1": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + }, + "w2": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + } + } + }, + "sources": [ + "nvidia/MiniMax-M3-NVFP4/block_sparse_moe" + ] + }, + { + "type": "moe", + "args": { + "tokens": 6144, + "activation": { + "alpha": 1.702, + "beta": 1.0, + "kind": "swigluoai", + "limit": 7.0 + }, + "experts": { + "a1": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "group": [ + 1, + 16 + ], + "static": false + }, + "scale2": { + "dtype": "fp32", + "group": [ + -1, + -1 + ], + "static": true + } + }, + "a2": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "group": [ + 1, + 16 + ], + "static": false + }, + "scale2": { + "dtype": "fp32", + "group": [ + -1, + -1 + ], + "static": true + } + }, + "inter": 3072, + "num": 128, + "top_k": 4, + "w1": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "group": [ + 1, + 16 + ], + "static": true + }, + "scale2": { + "dtype": "fp32", + "group": [ + -1, + -1 + ], + "static": true + } + }, + "w2": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "group": [ + 1, + 16 + ], + "static": true + }, + "scale2": { + "dtype": "fp32", + "group": [ + -1, + -1 + ], + "static": true + } + } + }, + "hidden": 6144, + "router": { + "bias": true, + "gate": { + "dtype": "fp32" + }, + "renormalize": true, + "scale": 2.0, + "scoring": "sigmoid", + "select": { + "kind": "topk" + } + }, + "routing": { + "distribution": "natural", + "seed": 0 + }, + "shared": { + "a1": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "count": 1, + "inter": 3072, + "w1": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + }, + "w2": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + } + } + }, + "sources": [ + "nvidia/MiniMax-M3-NVFP4/block_sparse_moe" + ] + }, + { + "type": "moe", + "args": { + "tokens": 8192, + "activation": { + "alpha": 1.702, + "beta": 1.0, + "kind": "swigluoai", + "limit": 7.0 + }, + "experts": { + "a1": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "group": [ + 1, + 16 + ], + "static": false + }, + "scale2": { + "dtype": "fp32", + "group": [ + -1, + -1 + ], + "static": true + } + }, + "a2": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "group": [ + 1, + 16 + ], + "static": false + }, + "scale2": { + "dtype": "fp32", + "group": [ + -1, + -1 + ], + "static": true + } + }, + "inter": 3072, + "num": 128, + "top_k": 4, + "w1": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "group": [ + 1, + 16 + ], + "static": true + }, + "scale2": { + "dtype": "fp32", + "group": [ + -1, + -1 + ], + "static": true + } + }, + "w2": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "group": [ + 1, + 16 + ], + "static": true + }, + "scale2": { + "dtype": "fp32", + "group": [ + -1, + -1 + ], + "static": true + } + } + }, + "hidden": 6144, + "router": { + "bias": true, + "gate": { + "dtype": "fp32" + }, + "renormalize": true, + "scale": 2.0, + "scoring": "sigmoid", + "select": { + "kind": "topk" + } + }, + "routing": { + "distribution": "natural", + "seed": 0 + }, + "shared": { + "a1": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "count": 1, + "inter": 3072, + "w1": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + }, + "w2": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + } + } + }, + "sources": [ + "nvidia/MiniMax-M3-NVFP4/block_sparse_moe" + ] + }, + { + "type": "moe", + "args": { + "tokens": 12288, + "activation": { + "alpha": 1.702, + "beta": 1.0, + "kind": "swigluoai", + "limit": 7.0 + }, + "experts": { + "a1": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "group": [ + 1, + 16 + ], + "static": false + }, + "scale2": { + "dtype": "fp32", + "group": [ + -1, + -1 + ], + "static": true + } + }, + "a2": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "group": [ + 1, + 16 + ], + "static": false + }, + "scale2": { + "dtype": "fp32", + "group": [ + -1, + -1 + ], + "static": true + } + }, + "inter": 3072, + "num": 128, + "top_k": 4, + "w1": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "group": [ + 1, + 16 + ], + "static": true + }, + "scale2": { + "dtype": "fp32", + "group": [ + -1, + -1 + ], + "static": true + } + }, + "w2": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "group": [ + 1, + 16 + ], + "static": true + }, + "scale2": { + "dtype": "fp32", + "group": [ + -1, + -1 + ], + "static": true + } + } + }, + "hidden": 6144, + "router": { + "bias": true, + "gate": { + "dtype": "fp32" + }, + "renormalize": true, + "scale": 2.0, + "scoring": "sigmoid", + "select": { + "kind": "topk" + } + }, + "routing": { + "distribution": "natural", + "seed": 0 + }, + "shared": { + "a1": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "count": 1, + "inter": 3072, + "w1": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + }, + "w2": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + } + } + }, + "sources": [ + "nvidia/MiniMax-M3-NVFP4/block_sparse_moe" + ] + }, + { + "type": "moe", + "args": { + "tokens": 16384, + "activation": { + "alpha": 1.702, + "beta": 1.0, + "kind": "swigluoai", + "limit": 7.0 + }, + "experts": { + "a1": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "group": [ + 1, + 16 + ], + "static": false + }, + "scale2": { + "dtype": "fp32", + "group": [ + -1, + -1 + ], + "static": true + } + }, + "a2": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "group": [ + 1, + 16 + ], + "static": false + }, + "scale2": { + "dtype": "fp32", + "group": [ + -1, + -1 + ], + "static": true + } + }, + "inter": 3072, + "num": 128, + "top_k": 4, + "w1": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "group": [ + 1, + 16 + ], + "static": true + }, + "scale2": { + "dtype": "fp32", + "group": [ + -1, + -1 + ], + "static": true + } + }, + "w2": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "group": [ + 1, + 16 + ], + "static": true + }, + "scale2": { + "dtype": "fp32", + "group": [ + -1, + -1 + ], + "static": true + } + } + }, + "hidden": 6144, + "router": { + "bias": true, + "gate": { + "dtype": "fp32" + }, + "renormalize": true, + "scale": 2.0, + "scoring": "sigmoid", + "select": { + "kind": "topk" + } + }, + "routing": { + "distribution": "natural", + "seed": 0 + }, + "shared": { + "a1": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "count": 1, + "inter": 3072, + "w1": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + }, + "w2": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + } + } + }, + "sources": [ + "nvidia/MiniMax-M3-NVFP4/block_sparse_moe" + ] + }, + { + "type": "moe", + "args": { + "tokens": 24576, + "activation": { + "alpha": 1.702, + "beta": 1.0, + "kind": "swigluoai", + "limit": 7.0 + }, + "experts": { + "a1": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "group": [ + 1, + 16 + ], + "static": false + }, + "scale2": { + "dtype": "fp32", + "group": [ + -1, + -1 + ], + "static": true + } + }, + "a2": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "group": [ + 1, + 16 + ], + "static": false + }, + "scale2": { + "dtype": "fp32", + "group": [ + -1, + -1 + ], + "static": true + } + }, + "inter": 3072, + "num": 128, + "top_k": 4, + "w1": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "group": [ + 1, + 16 + ], + "static": true + }, + "scale2": { + "dtype": "fp32", + "group": [ + -1, + -1 + ], + "static": true + } + }, + "w2": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "group": [ + 1, + 16 + ], + "static": true + }, + "scale2": { + "dtype": "fp32", + "group": [ + -1, + -1 + ], + "static": true + } + } + }, + "hidden": 6144, + "router": { + "bias": true, + "gate": { + "dtype": "fp32" + }, + "renormalize": true, + "scale": 2.0, + "scoring": "sigmoid", + "select": { + "kind": "topk" + } + }, + "routing": { + "distribution": "natural", + "seed": 0 + }, + "shared": { + "a1": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "count": 1, + "inter": 3072, + "w1": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + }, + "w2": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + } + } + }, + "sources": [ + "nvidia/MiniMax-M3-NVFP4/block_sparse_moe" + ] + }, + { + "type": "moe", + "args": { + "tokens": 32768, + "activation": { + "alpha": 1.702, + "beta": 1.0, + "kind": "swigluoai", + "limit": 7.0 + }, + "experts": { + "a1": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "group": [ + 1, + 16 + ], + "static": false + }, + "scale2": { + "dtype": "fp32", + "group": [ + -1, + -1 + ], + "static": true + } + }, + "a2": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "group": [ + 1, + 16 + ], + "static": false + }, + "scale2": { + "dtype": "fp32", + "group": [ + -1, + -1 + ], + "static": true + } + }, + "inter": 3072, + "num": 128, + "top_k": 4, + "w1": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "group": [ + 1, + 16 + ], + "static": true + }, + "scale2": { + "dtype": "fp32", + "group": [ + -1, + -1 + ], + "static": true + } + }, + "w2": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "group": [ + 1, + 16 + ], + "static": true + }, + "scale2": { + "dtype": "fp32", + "group": [ + -1, + -1 + ], + "static": true + } + } + }, + "hidden": 6144, + "router": { + "bias": true, + "gate": { + "dtype": "fp32" + }, + "renormalize": true, + "scale": 2.0, + "scoring": "sigmoid", + "select": { + "kind": "topk" + } + }, + "routing": { + "distribution": "natural", + "seed": 0 + }, + "shared": { + "a1": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "count": 1, + "inter": 3072, + "w1": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + }, + "w2": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + } + } + }, + "sources": [ + "nvidia/MiniMax-M3-NVFP4/block_sparse_moe" + ] + }, + { + "type": "moe", + "args": { + "tokens": 49152, + "activation": { + "alpha": 1.702, + "beta": 1.0, + "kind": "swigluoai", + "limit": 7.0 + }, + "experts": { + "a1": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "group": [ + 1, + 16 + ], + "static": false + }, + "scale2": { + "dtype": "fp32", + "group": [ + -1, + -1 + ], + "static": true + } + }, + "a2": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "group": [ + 1, + 16 + ], + "static": false + }, + "scale2": { + "dtype": "fp32", + "group": [ + -1, + -1 + ], + "static": true + } + }, + "inter": 3072, + "num": 128, + "top_k": 4, + "w1": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "group": [ + 1, + 16 + ], + "static": true + }, + "scale2": { + "dtype": "fp32", + "group": [ + -1, + -1 + ], + "static": true + } + }, + "w2": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "group": [ + 1, + 16 + ], + "static": true + }, + "scale2": { + "dtype": "fp32", + "group": [ + -1, + -1 + ], + "static": true + } + } + }, + "hidden": 6144, + "router": { + "bias": true, + "gate": { + "dtype": "fp32" + }, + "renormalize": true, + "scale": 2.0, + "scoring": "sigmoid", + "select": { + "kind": "topk" + } + }, + "routing": { + "distribution": "natural", + "seed": 0 + }, + "shared": { + "a1": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "count": 1, + "inter": 3072, + "w1": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + }, + "w2": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + } + } + }, + "sources": [ + "nvidia/MiniMax-M3-NVFP4/block_sparse_moe" + ] + }, + { + "type": "moe", + "args": { + "tokens": 65536, + "activation": { + "alpha": 1.702, + "beta": 1.0, + "kind": "swigluoai", + "limit": 7.0 + }, + "experts": { + "a1": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "group": [ + 1, + 16 + ], + "static": false + }, + "scale2": { + "dtype": "fp32", + "group": [ + -1, + -1 + ], + "static": true + } + }, + "a2": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "group": [ + 1, + 16 + ], + "static": false + }, + "scale2": { + "dtype": "fp32", + "group": [ + -1, + -1 + ], + "static": true + } + }, + "inter": 3072, + "num": 128, + "top_k": 4, + "w1": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "group": [ + 1, + 16 + ], + "static": true + }, + "scale2": { + "dtype": "fp32", + "group": [ + -1, + -1 + ], + "static": true + } + }, + "w2": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "group": [ + 1, + 16 + ], + "static": true + }, + "scale2": { + "dtype": "fp32", + "group": [ + -1, + -1 + ], + "static": true + } + } + }, + "hidden": 6144, + "router": { + "bias": true, + "gate": { + "dtype": "fp32" + }, + "renormalize": true, + "scale": 2.0, + "scoring": "sigmoid", + "select": { + "kind": "topk" + } + }, + "routing": { + "distribution": "natural", + "seed": 0 + }, + "shared": { + "a1": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "count": 1, + "inter": 3072, + "w1": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + }, + "w2": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + } + } + }, + "sources": [ + "nvidia/MiniMax-M3-NVFP4/block_sparse_moe" + ] + }, + { + "type": "moe", + "args": { + "tokens": 1, + "activation": { + "alpha": 1.702, + "beta": 1.0, + "kind": "swigluoai", + "limit": 7.0 + }, + "experts": { + "a1": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "a2": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "inter": 3072, + "num": 128, + "top_k": 4, + "w1": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + }, + "w2": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + } + }, + "hidden": 6144, + "router": { + "bias": true, + "gate": { + "dtype": "fp32" + }, + "renormalize": true, + "scale": 2.0, + "scoring": "sigmoid", + "select": { + "kind": "topk" + } + }, + "routing": { + "distribution": "natural", + "seed": 0 + }, + "shared": { + "a1": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "count": 1, + "inter": 3072, + "w1": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + }, + "w2": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + } + } + }, + "sources": [ + "amd/MiniMax-M3-MXFP4/block_sparse_moe" + ] + }, + { + "type": "moe", + "args": { + "tokens": 2, + "activation": { + "alpha": 1.702, + "beta": 1.0, + "kind": "swigluoai", + "limit": 7.0 + }, + "experts": { + "a1": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "a2": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "inter": 3072, + "num": 128, + "top_k": 4, + "w1": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + }, + "w2": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + } + }, + "hidden": 6144, + "router": { + "bias": true, + "gate": { + "dtype": "fp32" + }, + "renormalize": true, + "scale": 2.0, + "scoring": "sigmoid", + "select": { + "kind": "topk" + } + }, + "routing": { + "distribution": "natural", + "seed": 0 + }, + "shared": { + "a1": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "count": 1, + "inter": 3072, + "w1": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + }, + "w2": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + } + } + }, + "sources": [ + "amd/MiniMax-M3-MXFP4/block_sparse_moe" + ] + }, + { + "type": "moe", + "args": { + "tokens": 3, + "activation": { + "alpha": 1.702, + "beta": 1.0, + "kind": "swigluoai", + "limit": 7.0 + }, + "experts": { + "a1": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "a2": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "inter": 3072, + "num": 128, + "top_k": 4, + "w1": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + }, + "w2": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + } + }, + "hidden": 6144, + "router": { + "bias": true, + "gate": { + "dtype": "fp32" + }, + "renormalize": true, + "scale": 2.0, + "scoring": "sigmoid", + "select": { + "kind": "topk" + } + }, + "routing": { + "distribution": "natural", + "seed": 0 + }, + "shared": { + "a1": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "count": 1, + "inter": 3072, + "w1": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + }, + "w2": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + } + } + }, + "sources": [ + "amd/MiniMax-M3-MXFP4/block_sparse_moe" + ] + }, + { + "type": "moe", + "args": { + "tokens": 4, + "activation": { + "alpha": 1.702, + "beta": 1.0, + "kind": "swigluoai", + "limit": 7.0 + }, + "experts": { + "a1": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "a2": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "inter": 3072, + "num": 128, + "top_k": 4, + "w1": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + }, + "w2": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + } + }, + "hidden": 6144, + "router": { + "bias": true, + "gate": { + "dtype": "fp32" + }, + "renormalize": true, + "scale": 2.0, + "scoring": "sigmoid", + "select": { + "kind": "topk" + } + }, + "routing": { + "distribution": "natural", + "seed": 0 + }, + "shared": { + "a1": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "count": 1, + "inter": 3072, + "w1": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + }, + "w2": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + } + } + }, + "sources": [ + "amd/MiniMax-M3-MXFP4/block_sparse_moe" + ] + }, + { + "type": "moe", + "args": { + "tokens": 6, + "activation": { + "alpha": 1.702, + "beta": 1.0, + "kind": "swigluoai", + "limit": 7.0 + }, + "experts": { + "a1": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "a2": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "inter": 3072, + "num": 128, + "top_k": 4, + "w1": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + }, + "w2": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + } + }, + "hidden": 6144, + "router": { + "bias": true, + "gate": { + "dtype": "fp32" + }, + "renormalize": true, + "scale": 2.0, + "scoring": "sigmoid", + "select": { + "kind": "topk" + } + }, + "routing": { + "distribution": "natural", + "seed": 0 + }, + "shared": { + "a1": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "count": 1, + "inter": 3072, + "w1": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + }, + "w2": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + } + } + }, + "sources": [ + "amd/MiniMax-M3-MXFP4/block_sparse_moe" + ] + }, + { + "type": "moe", + "args": { + "tokens": 8, + "activation": { + "alpha": 1.702, + "beta": 1.0, + "kind": "swigluoai", + "limit": 7.0 + }, + "experts": { + "a1": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "a2": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "inter": 3072, + "num": 128, + "top_k": 4, + "w1": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + }, + "w2": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + } + }, + "hidden": 6144, + "router": { + "bias": true, + "gate": { + "dtype": "fp32" + }, + "renormalize": true, + "scale": 2.0, + "scoring": "sigmoid", + "select": { + "kind": "topk" + } + }, + "routing": { + "distribution": "natural", + "seed": 0 + }, + "shared": { + "a1": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "count": 1, + "inter": 3072, + "w1": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + }, + "w2": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + } + } + }, + "sources": [ + "amd/MiniMax-M3-MXFP4/block_sparse_moe" + ] + }, + { + "type": "moe", + "args": { + "tokens": 12, + "activation": { + "alpha": 1.702, + "beta": 1.0, + "kind": "swigluoai", + "limit": 7.0 + }, + "experts": { + "a1": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "a2": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "inter": 3072, + "num": 128, + "top_k": 4, + "w1": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + }, + "w2": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + } + }, + "hidden": 6144, + "router": { + "bias": true, + "gate": { + "dtype": "fp32" + }, + "renormalize": true, + "scale": 2.0, + "scoring": "sigmoid", + "select": { + "kind": "topk" + } + }, + "routing": { + "distribution": "natural", + "seed": 0 + }, + "shared": { + "a1": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "count": 1, + "inter": 3072, + "w1": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + }, + "w2": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + } + } + }, + "sources": [ + "amd/MiniMax-M3-MXFP4/block_sparse_moe" + ] + }, + { + "type": "moe", + "args": { + "tokens": 16, + "activation": { + "alpha": 1.702, + "beta": 1.0, + "kind": "swigluoai", + "limit": 7.0 + }, + "experts": { + "a1": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "a2": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "inter": 3072, + "num": 128, + "top_k": 4, + "w1": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + }, + "w2": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + } + }, + "hidden": 6144, + "router": { + "bias": true, + "gate": { + "dtype": "fp32" + }, + "renormalize": true, + "scale": 2.0, + "scoring": "sigmoid", + "select": { + "kind": "topk" + } + }, + "routing": { + "distribution": "natural", + "seed": 0 + }, + "shared": { + "a1": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "count": 1, + "inter": 3072, + "w1": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + }, + "w2": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + } + } + }, + "sources": [ + "amd/MiniMax-M3-MXFP4/block_sparse_moe" + ] + }, + { + "type": "moe", + "args": { + "tokens": 24, + "activation": { + "alpha": 1.702, + "beta": 1.0, + "kind": "swigluoai", + "limit": 7.0 + }, + "experts": { + "a1": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "a2": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "inter": 3072, + "num": 128, + "top_k": 4, + "w1": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + }, + "w2": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + } + }, + "hidden": 6144, + "router": { + "bias": true, + "gate": { + "dtype": "fp32" + }, + "renormalize": true, + "scale": 2.0, + "scoring": "sigmoid", + "select": { + "kind": "topk" + } + }, + "routing": { + "distribution": "natural", + "seed": 0 + }, + "shared": { + "a1": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "count": 1, + "inter": 3072, + "w1": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + }, + "w2": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + } + } + }, + "sources": [ + "amd/MiniMax-M3-MXFP4/block_sparse_moe" + ] + }, + { + "type": "moe", + "args": { + "tokens": 32, + "activation": { + "alpha": 1.702, + "beta": 1.0, + "kind": "swigluoai", + "limit": 7.0 + }, + "experts": { + "a1": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "a2": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "inter": 3072, + "num": 128, + "top_k": 4, + "w1": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + }, + "w2": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + } + }, + "hidden": 6144, + "router": { + "bias": true, + "gate": { + "dtype": "fp32" + }, + "renormalize": true, + "scale": 2.0, + "scoring": "sigmoid", + "select": { + "kind": "topk" + } + }, + "routing": { + "distribution": "natural", + "seed": 0 + }, + "shared": { + "a1": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "count": 1, + "inter": 3072, + "w1": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + }, + "w2": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + } + } + }, + "sources": [ + "amd/MiniMax-M3-MXFP4/block_sparse_moe" + ] + }, + { + "type": "moe", + "args": { + "tokens": 48, + "activation": { + "alpha": 1.702, + "beta": 1.0, + "kind": "swigluoai", + "limit": 7.0 + }, + "experts": { + "a1": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "a2": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "inter": 3072, + "num": 128, + "top_k": 4, + "w1": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + }, + "w2": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + } + }, + "hidden": 6144, + "router": { + "bias": true, + "gate": { + "dtype": "fp32" + }, + "renormalize": true, + "scale": 2.0, + "scoring": "sigmoid", + "select": { + "kind": "topk" + } + }, + "routing": { + "distribution": "natural", + "seed": 0 + }, + "shared": { + "a1": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "count": 1, + "inter": 3072, + "w1": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + }, + "w2": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + } + } + }, + "sources": [ + "amd/MiniMax-M3-MXFP4/block_sparse_moe" + ] + }, + { + "type": "moe", + "args": { + "tokens": 64, + "activation": { + "alpha": 1.702, + "beta": 1.0, + "kind": "swigluoai", + "limit": 7.0 + }, + "experts": { + "a1": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "a2": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "inter": 3072, + "num": 128, + "top_k": 4, + "w1": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + }, + "w2": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + } + }, + "hidden": 6144, + "router": { + "bias": true, + "gate": { + "dtype": "fp32" + }, + "renormalize": true, + "scale": 2.0, + "scoring": "sigmoid", + "select": { + "kind": "topk" + } + }, + "routing": { + "distribution": "natural", + "seed": 0 + }, + "shared": { + "a1": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "count": 1, + "inter": 3072, + "w1": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + }, + "w2": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + } + } + }, + "sources": [ + "amd/MiniMax-M3-MXFP4/block_sparse_moe" + ] + }, + { + "type": "moe", + "args": { + "tokens": 96, + "activation": { + "alpha": 1.702, + "beta": 1.0, + "kind": "swigluoai", + "limit": 7.0 + }, + "experts": { + "a1": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "a2": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "inter": 3072, + "num": 128, + "top_k": 4, + "w1": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + }, + "w2": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + } + }, + "hidden": 6144, + "router": { + "bias": true, + "gate": { + "dtype": "fp32" + }, + "renormalize": true, + "scale": 2.0, + "scoring": "sigmoid", + "select": { + "kind": "topk" + } + }, + "routing": { + "distribution": "natural", + "seed": 0 + }, + "shared": { + "a1": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "count": 1, + "inter": 3072, + "w1": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + }, + "w2": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + } + } + }, + "sources": [ + "amd/MiniMax-M3-MXFP4/block_sparse_moe" + ] + }, + { + "type": "moe", + "args": { + "tokens": 128, + "activation": { + "alpha": 1.702, + "beta": 1.0, + "kind": "swigluoai", + "limit": 7.0 + }, + "experts": { + "a1": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "a2": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "inter": 3072, + "num": 128, + "top_k": 4, + "w1": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + }, + "w2": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + } + }, + "hidden": 6144, + "router": { + "bias": true, + "gate": { + "dtype": "fp32" + }, + "renormalize": true, + "scale": 2.0, + "scoring": "sigmoid", + "select": { + "kind": "topk" + } + }, + "routing": { + "distribution": "natural", + "seed": 0 + }, + "shared": { + "a1": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "count": 1, + "inter": 3072, + "w1": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + }, + "w2": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + } + } + }, + "sources": [ + "amd/MiniMax-M3-MXFP4/block_sparse_moe" + ] + }, + { + "type": "moe", + "args": { + "tokens": 192, + "activation": { + "alpha": 1.702, + "beta": 1.0, + "kind": "swigluoai", + "limit": 7.0 + }, + "experts": { + "a1": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "a2": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "inter": 3072, + "num": 128, + "top_k": 4, + "w1": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + }, + "w2": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + } + }, + "hidden": 6144, + "router": { + "bias": true, + "gate": { + "dtype": "fp32" + }, + "renormalize": true, + "scale": 2.0, + "scoring": "sigmoid", + "select": { + "kind": "topk" + } + }, + "routing": { + "distribution": "natural", + "seed": 0 + }, + "shared": { + "a1": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "count": 1, + "inter": 3072, + "w1": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + }, + "w2": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + } + } + }, + "sources": [ + "amd/MiniMax-M3-MXFP4/block_sparse_moe" + ] + }, + { + "type": "moe", + "args": { + "tokens": 256, + "activation": { + "alpha": 1.702, + "beta": 1.0, + "kind": "swigluoai", + "limit": 7.0 + }, + "experts": { + "a1": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "a2": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "inter": 3072, + "num": 128, + "top_k": 4, + "w1": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + }, + "w2": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + } + }, + "hidden": 6144, + "router": { + "bias": true, + "gate": { + "dtype": "fp32" + }, + "renormalize": true, + "scale": 2.0, + "scoring": "sigmoid", + "select": { + "kind": "topk" + } + }, + "routing": { + "distribution": "natural", + "seed": 0 + }, + "shared": { + "a1": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "count": 1, + "inter": 3072, + "w1": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + }, + "w2": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + } + } + }, + "sources": [ + "amd/MiniMax-M3-MXFP4/block_sparse_moe" + ] + }, + { + "type": "moe", + "args": { + "tokens": 384, + "activation": { + "alpha": 1.702, + "beta": 1.0, + "kind": "swigluoai", + "limit": 7.0 + }, + "experts": { + "a1": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "a2": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "inter": 3072, + "num": 128, + "top_k": 4, + "w1": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + }, + "w2": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + } + }, + "hidden": 6144, + "router": { + "bias": true, + "gate": { + "dtype": "fp32" + }, + "renormalize": true, + "scale": 2.0, + "scoring": "sigmoid", + "select": { + "kind": "topk" + } + }, + "routing": { + "distribution": "natural", + "seed": 0 + }, + "shared": { + "a1": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "count": 1, + "inter": 3072, + "w1": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + }, + "w2": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + } + } + }, + "sources": [ + "amd/MiniMax-M3-MXFP4/block_sparse_moe" + ] + }, + { + "type": "moe", + "args": { + "tokens": 512, + "activation": { + "alpha": 1.702, + "beta": 1.0, + "kind": "swigluoai", + "limit": 7.0 + }, + "experts": { + "a1": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "a2": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "inter": 3072, + "num": 128, + "top_k": 4, + "w1": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + }, + "w2": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + } + }, + "hidden": 6144, + "router": { + "bias": true, + "gate": { + "dtype": "fp32" + }, + "renormalize": true, + "scale": 2.0, + "scoring": "sigmoid", + "select": { + "kind": "topk" + } + }, + "routing": { + "distribution": "natural", + "seed": 0 + }, + "shared": { + "a1": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "count": 1, + "inter": 3072, + "w1": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + }, + "w2": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + } + } + }, + "sources": [ + "amd/MiniMax-M3-MXFP4/block_sparse_moe" + ] + }, + { + "type": "moe", + "args": { + "tokens": 768, + "activation": { + "alpha": 1.702, + "beta": 1.0, + "kind": "swigluoai", + "limit": 7.0 + }, + "experts": { + "a1": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "a2": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "inter": 3072, + "num": 128, + "top_k": 4, + "w1": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + }, + "w2": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + } + }, + "hidden": 6144, + "router": { + "bias": true, + "gate": { + "dtype": "fp32" + }, + "renormalize": true, + "scale": 2.0, + "scoring": "sigmoid", + "select": { + "kind": "topk" + } + }, + "routing": { + "distribution": "natural", + "seed": 0 + }, + "shared": { + "a1": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "count": 1, + "inter": 3072, + "w1": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + }, + "w2": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + } + } + }, + "sources": [ + "amd/MiniMax-M3-MXFP4/block_sparse_moe" + ] + }, + { + "type": "moe", + "args": { + "tokens": 1024, + "activation": { + "alpha": 1.702, + "beta": 1.0, + "kind": "swigluoai", + "limit": 7.0 + }, + "experts": { + "a1": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "a2": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "inter": 3072, + "num": 128, + "top_k": 4, + "w1": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + }, + "w2": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + } + }, + "hidden": 6144, + "router": { + "bias": true, + "gate": { + "dtype": "fp32" + }, + "renormalize": true, + "scale": 2.0, + "scoring": "sigmoid", + "select": { + "kind": "topk" + } + }, + "routing": { + "distribution": "natural", + "seed": 0 + }, + "shared": { + "a1": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "count": 1, + "inter": 3072, + "w1": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + }, + "w2": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + } + } + }, + "sources": [ + "amd/MiniMax-M3-MXFP4/block_sparse_moe" + ] + }, + { + "type": "moe", + "args": { + "tokens": 1536, + "activation": { + "alpha": 1.702, + "beta": 1.0, + "kind": "swigluoai", + "limit": 7.0 + }, + "experts": { + "a1": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "a2": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "inter": 3072, + "num": 128, + "top_k": 4, + "w1": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + }, + "w2": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + } + }, + "hidden": 6144, + "router": { + "bias": true, + "gate": { + "dtype": "fp32" + }, + "renormalize": true, + "scale": 2.0, + "scoring": "sigmoid", + "select": { + "kind": "topk" + } + }, + "routing": { + "distribution": "natural", + "seed": 0 + }, + "shared": { + "a1": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "count": 1, + "inter": 3072, + "w1": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + }, + "w2": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + } + } + }, + "sources": [ + "amd/MiniMax-M3-MXFP4/block_sparse_moe" + ] + }, + { + "type": "moe", + "args": { + "tokens": 2048, + "activation": { + "alpha": 1.702, + "beta": 1.0, + "kind": "swigluoai", + "limit": 7.0 + }, + "experts": { + "a1": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "a2": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "inter": 3072, + "num": 128, + "top_k": 4, + "w1": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + }, + "w2": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + } + }, + "hidden": 6144, + "router": { + "bias": true, + "gate": { + "dtype": "fp32" + }, + "renormalize": true, + "scale": 2.0, + "scoring": "sigmoid", + "select": { + "kind": "topk" + } + }, + "routing": { + "distribution": "natural", + "seed": 0 + }, + "shared": { + "a1": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "count": 1, + "inter": 3072, + "w1": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + }, + "w2": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + } + } + }, + "sources": [ + "amd/MiniMax-M3-MXFP4/block_sparse_moe" + ] + }, + { + "type": "moe", + "args": { + "tokens": 3072, + "activation": { + "alpha": 1.702, + "beta": 1.0, + "kind": "swigluoai", + "limit": 7.0 + }, + "experts": { + "a1": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "a2": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "inter": 3072, + "num": 128, + "top_k": 4, + "w1": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + }, + "w2": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + } + }, + "hidden": 6144, + "router": { + "bias": true, + "gate": { + "dtype": "fp32" + }, + "renormalize": true, + "scale": 2.0, + "scoring": "sigmoid", + "select": { + "kind": "topk" + } + }, + "routing": { + "distribution": "natural", + "seed": 0 + }, + "shared": { + "a1": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "count": 1, + "inter": 3072, + "w1": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + }, + "w2": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + } + } + }, + "sources": [ + "amd/MiniMax-M3-MXFP4/block_sparse_moe" + ] + }, + { + "type": "moe", + "args": { + "tokens": 4096, + "activation": { + "alpha": 1.702, + "beta": 1.0, + "kind": "swigluoai", + "limit": 7.0 + }, + "experts": { + "a1": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "a2": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "inter": 3072, + "num": 128, + "top_k": 4, + "w1": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + }, + "w2": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + } + }, + "hidden": 6144, + "router": { + "bias": true, + "gate": { + "dtype": "fp32" + }, + "renormalize": true, + "scale": 2.0, + "scoring": "sigmoid", + "select": { + "kind": "topk" + } + }, + "routing": { + "distribution": "natural", + "seed": 0 + }, + "shared": { + "a1": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "count": 1, + "inter": 3072, + "w1": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + }, + "w2": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + } + } + }, + "sources": [ + "amd/MiniMax-M3-MXFP4/block_sparse_moe" + ] + }, + { + "type": "moe", + "args": { + "tokens": 6144, + "activation": { + "alpha": 1.702, + "beta": 1.0, + "kind": "swigluoai", + "limit": 7.0 + }, + "experts": { + "a1": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "a2": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "inter": 3072, + "num": 128, + "top_k": 4, + "w1": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + }, + "w2": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + } + }, + "hidden": 6144, + "router": { + "bias": true, + "gate": { + "dtype": "fp32" + }, + "renormalize": true, + "scale": 2.0, + "scoring": "sigmoid", + "select": { + "kind": "topk" + } + }, + "routing": { + "distribution": "natural", + "seed": 0 + }, + "shared": { + "a1": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "count": 1, + "inter": 3072, + "w1": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + }, + "w2": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + } + } + }, + "sources": [ + "amd/MiniMax-M3-MXFP4/block_sparse_moe" + ] + }, + { + "type": "moe", + "args": { + "tokens": 8192, + "activation": { + "alpha": 1.702, + "beta": 1.0, + "kind": "swigluoai", + "limit": 7.0 + }, + "experts": { + "a1": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "a2": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "inter": 3072, + "num": 128, + "top_k": 4, + "w1": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + }, + "w2": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + } + }, + "hidden": 6144, + "router": { + "bias": true, + "gate": { + "dtype": "fp32" + }, + "renormalize": true, + "scale": 2.0, + "scoring": "sigmoid", + "select": { + "kind": "topk" + } + }, + "routing": { + "distribution": "natural", + "seed": 0 + }, + "shared": { + "a1": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "count": 1, + "inter": 3072, + "w1": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + }, + "w2": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + } + } + }, + "sources": [ + "amd/MiniMax-M3-MXFP4/block_sparse_moe" + ] + }, + { + "type": "moe", + "args": { + "tokens": 12288, + "activation": { + "alpha": 1.702, + "beta": 1.0, + "kind": "swigluoai", + "limit": 7.0 + }, + "experts": { + "a1": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "a2": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "inter": 3072, + "num": 128, + "top_k": 4, + "w1": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + }, + "w2": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + } + }, + "hidden": 6144, + "router": { + "bias": true, + "gate": { + "dtype": "fp32" + }, + "renormalize": true, + "scale": 2.0, + "scoring": "sigmoid", + "select": { + "kind": "topk" + } + }, + "routing": { + "distribution": "natural", + "seed": 0 + }, + "shared": { + "a1": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "count": 1, + "inter": 3072, + "w1": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + }, + "w2": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + } + } + }, + "sources": [ + "amd/MiniMax-M3-MXFP4/block_sparse_moe" + ] + }, + { + "type": "moe", + "args": { + "tokens": 16384, + "activation": { + "alpha": 1.702, + "beta": 1.0, + "kind": "swigluoai", + "limit": 7.0 + }, + "experts": { + "a1": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "a2": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "inter": 3072, + "num": 128, + "top_k": 4, + "w1": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + }, + "w2": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + } + }, + "hidden": 6144, + "router": { + "bias": true, + "gate": { + "dtype": "fp32" + }, + "renormalize": true, + "scale": 2.0, + "scoring": "sigmoid", + "select": { + "kind": "topk" + } + }, + "routing": { + "distribution": "natural", + "seed": 0 + }, + "shared": { + "a1": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "count": 1, + "inter": 3072, + "w1": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + }, + "w2": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + } + } + }, + "sources": [ + "amd/MiniMax-M3-MXFP4/block_sparse_moe" + ] + }, + { + "type": "moe", + "args": { + "tokens": 24576, + "activation": { + "alpha": 1.702, + "beta": 1.0, + "kind": "swigluoai", + "limit": 7.0 + }, + "experts": { + "a1": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "a2": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "inter": 3072, + "num": 128, + "top_k": 4, + "w1": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + }, + "w2": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + } + }, + "hidden": 6144, + "router": { + "bias": true, + "gate": { + "dtype": "fp32" + }, + "renormalize": true, + "scale": 2.0, + "scoring": "sigmoid", + "select": { + "kind": "topk" + } + }, + "routing": { + "distribution": "natural", + "seed": 0 + }, + "shared": { + "a1": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "count": 1, + "inter": 3072, + "w1": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + }, + "w2": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + } + } + }, + "sources": [ + "amd/MiniMax-M3-MXFP4/block_sparse_moe" + ] + }, + { + "type": "moe", + "args": { + "tokens": 32768, + "activation": { + "alpha": 1.702, + "beta": 1.0, + "kind": "swigluoai", + "limit": 7.0 + }, + "experts": { + "a1": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "a2": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "inter": 3072, + "num": 128, + "top_k": 4, + "w1": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + }, + "w2": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + } + }, + "hidden": 6144, + "router": { + "bias": true, + "gate": { + "dtype": "fp32" + }, + "renormalize": true, + "scale": 2.0, + "scoring": "sigmoid", + "select": { + "kind": "topk" + } + }, + "routing": { + "distribution": "natural", + "seed": 0 + }, + "shared": { + "a1": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "count": 1, + "inter": 3072, + "w1": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + }, + "w2": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + } + } + }, + "sources": [ + "amd/MiniMax-M3-MXFP4/block_sparse_moe" + ] + }, + { + "type": "moe", + "args": { + "tokens": 49152, + "activation": { + "alpha": 1.702, + "beta": 1.0, + "kind": "swigluoai", + "limit": 7.0 + }, + "experts": { + "a1": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "a2": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "inter": 3072, + "num": 128, + "top_k": 4, + "w1": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + }, + "w2": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + } + }, + "hidden": 6144, + "router": { + "bias": true, + "gate": { + "dtype": "fp32" + }, + "renormalize": true, + "scale": 2.0, + "scoring": "sigmoid", + "select": { + "kind": "topk" + } + }, + "routing": { + "distribution": "natural", + "seed": 0 + }, + "shared": { + "a1": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "count": 1, + "inter": 3072, + "w1": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + }, + "w2": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + } + } + }, + "sources": [ + "amd/MiniMax-M3-MXFP4/block_sparse_moe" + ] + }, + { + "type": "moe", + "args": { + "tokens": 65536, + "activation": { + "alpha": 1.702, + "beta": 1.0, + "kind": "swigluoai", + "limit": 7.0 + }, + "experts": { + "a1": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "a2": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "inter": 3072, + "num": 128, + "top_k": 4, + "w1": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + }, + "w2": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + } + }, + "hidden": 6144, + "router": { + "bias": true, + "gate": { + "dtype": "fp32" + }, + "renormalize": true, + "scale": 2.0, + "scoring": "sigmoid", + "select": { + "kind": "topk" + } + }, + "routing": { + "distribution": "natural", + "seed": 0 + }, + "shared": { + "a1": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "count": 1, + "inter": 3072, + "w1": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + }, + "w2": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + } + } + }, + "sources": [ + "amd/MiniMax-M3-MXFP4/block_sparse_moe" + ] + } +] diff --git a/operatorx/testlists/moe_parallel.json b/operatorx/testlists/moe_parallel.json new file mode 100644 index 0000000000..85fdf0da6d --- /dev/null +++ b/operatorx/testlists/moe_parallel.json @@ -0,0 +1,5680 @@ +[ + { + "type": "moe", + "args": { + "tokens": 8, + "activation": { + "alpha": 1.702, + "beta": 1.0, + "kind": "swigluoai", + "limit": 7.0 + }, + "experts": { + "a1": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "a2": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "inter": 3072, + "num": 128, + "top_k": 4, + "w1": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + }, + "w2": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + } + }, + "hidden": 6144, + "router": { + "bias": true, + "gate": { + "dtype": "fp32" + }, + "renormalize": true, + "scale": 2.0, + "scoring": "sigmoid", + "select": { + "kind": "topk" + } + }, + "routing": { + "distribution": "natural", + "seed": 0 + }, + "shared": { + "a1": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "count": 1, + "inter": 3072, + "w1": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + }, + "w2": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + } + }, + "parallel": { + "tp": 8 + } + }, + "sources": [ + "MiniMaxAI/MiniMax-M3-MXFP8/block_sparse_moe" + ] + }, + { + "type": "moe", + "args": { + "tokens": 256, + "activation": { + "alpha": 1.702, + "beta": 1.0, + "kind": "swigluoai", + "limit": 7.0 + }, + "experts": { + "a1": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "a2": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "inter": 3072, + "num": 128, + "top_k": 4, + "w1": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + }, + "w2": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + } + }, + "hidden": 6144, + "router": { + "bias": true, + "gate": { + "dtype": "fp32" + }, + "renormalize": true, + "scale": 2.0, + "scoring": "sigmoid", + "select": { + "kind": "topk" + } + }, + "routing": { + "distribution": "natural", + "seed": 0 + }, + "shared": { + "a1": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "count": 1, + "inter": 3072, + "w1": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + }, + "w2": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + } + }, + "parallel": { + "tp": 8 + } + }, + "sources": [ + "MiniMaxAI/MiniMax-M3-MXFP8/block_sparse_moe" + ] + }, + { + "type": "moe", + "args": { + "tokens": 8, + "activation": { + "alpha": 1.702, + "beta": 1.0, + "kind": "swigluoai", + "limit": 7.0 + }, + "experts": { + "a1": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "group": [ + 1, + 16 + ], + "static": false + }, + "scale2": { + "dtype": "fp32", + "group": [ + -1, + -1 + ], + "static": true + } + }, + "a2": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "group": [ + 1, + 16 + ], + "static": false + }, + "scale2": { + "dtype": "fp32", + "group": [ + -1, + -1 + ], + "static": true + } + }, + "inter": 3072, + "num": 128, + "top_k": 4, + "w1": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "group": [ + 1, + 16 + ], + "static": true + }, + "scale2": { + "dtype": "fp32", + "group": [ + -1, + -1 + ], + "static": true + } + }, + "w2": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "group": [ + 1, + 16 + ], + "static": true + }, + "scale2": { + "dtype": "fp32", + "group": [ + -1, + -1 + ], + "static": true + } + } + }, + "hidden": 6144, + "router": { + "bias": true, + "gate": { + "dtype": "fp32" + }, + "renormalize": true, + "scale": 2.0, + "scoring": "sigmoid", + "select": { + "kind": "topk" + } + }, + "routing": { + "distribution": "natural", + "seed": 0 + }, + "shared": { + "a1": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "count": 1, + "inter": 3072, + "w1": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + }, + "w2": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + } + }, + "parallel": { + "tp": 2 + } + }, + "sources": [ + "nvidia/MiniMax-M3-NVFP4/block_sparse_moe" + ] + }, + { + "type": "moe", + "args": { + "tokens": 8, + "activation": { + "alpha": 1.702, + "beta": 1.0, + "kind": "swigluoai", + "limit": 7.0 + }, + "experts": { + "a1": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "group": [ + 1, + 16 + ], + "static": false + }, + "scale2": { + "dtype": "fp32", + "group": [ + -1, + -1 + ], + "static": true + } + }, + "a2": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "group": [ + 1, + 16 + ], + "static": false + }, + "scale2": { + "dtype": "fp32", + "group": [ + -1, + -1 + ], + "static": true + } + }, + "inter": 3072, + "num": 128, + "top_k": 4, + "w1": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "group": [ + 1, + 16 + ], + "static": true + }, + "scale2": { + "dtype": "fp32", + "group": [ + -1, + -1 + ], + "static": true + } + }, + "w2": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "group": [ + 1, + 16 + ], + "static": true + }, + "scale2": { + "dtype": "fp32", + "group": [ + -1, + -1 + ], + "static": true + } + } + }, + "hidden": 6144, + "router": { + "bias": true, + "gate": { + "dtype": "fp32" + }, + "renormalize": true, + "scale": 2.0, + "scoring": "sigmoid", + "select": { + "kind": "topk" + } + }, + "routing": { + "distribution": "natural", + "seed": 0 + }, + "shared": { + "a1": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "count": 1, + "inter": 3072, + "w1": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + }, + "w2": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + } + }, + "parallel": { + "tp": 4 + } + }, + "sources": [ + "nvidia/MiniMax-M3-NVFP4/block_sparse_moe" + ] + }, + { + "type": "moe", + "args": { + "tokens": 8, + "activation": { + "alpha": 1.702, + "beta": 1.0, + "kind": "swigluoai", + "limit": 7.0 + }, + "experts": { + "a1": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "group": [ + 1, + 16 + ], + "static": false + }, + "scale2": { + "dtype": "fp32", + "group": [ + -1, + -1 + ], + "static": true + } + }, + "a2": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "group": [ + 1, + 16 + ], + "static": false + }, + "scale2": { + "dtype": "fp32", + "group": [ + -1, + -1 + ], + "static": true + } + }, + "inter": 3072, + "num": 128, + "top_k": 4, + "w1": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "group": [ + 1, + 16 + ], + "static": true + }, + "scale2": { + "dtype": "fp32", + "group": [ + -1, + -1 + ], + "static": true + } + }, + "w2": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "group": [ + 1, + 16 + ], + "static": true + }, + "scale2": { + "dtype": "fp32", + "group": [ + -1, + -1 + ], + "static": true + } + } + }, + "hidden": 6144, + "router": { + "bias": true, + "gate": { + "dtype": "fp32" + }, + "renormalize": true, + "scale": 2.0, + "scoring": "sigmoid", + "select": { + "kind": "topk" + } + }, + "routing": { + "distribution": "natural", + "seed": 0 + }, + "shared": { + "a1": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "count": 1, + "inter": 3072, + "w1": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + }, + "w2": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + } + }, + "parallel": { + "tp": 8 + } + }, + "sources": [ + "nvidia/MiniMax-M3-NVFP4/block_sparse_moe" + ] + }, + { + "type": "moe", + "args": { + "tokens": 256, + "activation": { + "alpha": 1.702, + "beta": 1.0, + "kind": "swigluoai", + "limit": 7.0 + }, + "experts": { + "a1": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "group": [ + 1, + 16 + ], + "static": false + }, + "scale2": { + "dtype": "fp32", + "group": [ + -1, + -1 + ], + "static": true + } + }, + "a2": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "group": [ + 1, + 16 + ], + "static": false + }, + "scale2": { + "dtype": "fp32", + "group": [ + -1, + -1 + ], + "static": true + } + }, + "inter": 3072, + "num": 128, + "top_k": 4, + "w1": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "group": [ + 1, + 16 + ], + "static": true + }, + "scale2": { + "dtype": "fp32", + "group": [ + -1, + -1 + ], + "static": true + } + }, + "w2": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "group": [ + 1, + 16 + ], + "static": true + }, + "scale2": { + "dtype": "fp32", + "group": [ + -1, + -1 + ], + "static": true + } + } + }, + "hidden": 6144, + "router": { + "bias": true, + "gate": { + "dtype": "fp32" + }, + "renormalize": true, + "scale": 2.0, + "scoring": "sigmoid", + "select": { + "kind": "topk" + } + }, + "routing": { + "distribution": "natural", + "seed": 0 + }, + "shared": { + "a1": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "count": 1, + "inter": 3072, + "w1": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + }, + "w2": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + } + }, + "parallel": { + "tp": 2 + } + }, + "sources": [ + "nvidia/MiniMax-M3-NVFP4/block_sparse_moe" + ] + }, + { + "type": "moe", + "args": { + "tokens": 256, + "activation": { + "alpha": 1.702, + "beta": 1.0, + "kind": "swigluoai", + "limit": 7.0 + }, + "experts": { + "a1": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "group": [ + 1, + 16 + ], + "static": false + }, + "scale2": { + "dtype": "fp32", + "group": [ + -1, + -1 + ], + "static": true + } + }, + "a2": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "group": [ + 1, + 16 + ], + "static": false + }, + "scale2": { + "dtype": "fp32", + "group": [ + -1, + -1 + ], + "static": true + } + }, + "inter": 3072, + "num": 128, + "top_k": 4, + "w1": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "group": [ + 1, + 16 + ], + "static": true + }, + "scale2": { + "dtype": "fp32", + "group": [ + -1, + -1 + ], + "static": true + } + }, + "w2": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "group": [ + 1, + 16 + ], + "static": true + }, + "scale2": { + "dtype": "fp32", + "group": [ + -1, + -1 + ], + "static": true + } + } + }, + "hidden": 6144, + "router": { + "bias": true, + "gate": { + "dtype": "fp32" + }, + "renormalize": true, + "scale": 2.0, + "scoring": "sigmoid", + "select": { + "kind": "topk" + } + }, + "routing": { + "distribution": "natural", + "seed": 0 + }, + "shared": { + "a1": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "count": 1, + "inter": 3072, + "w1": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + }, + "w2": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + } + }, + "parallel": { + "tp": 4 + } + }, + "sources": [ + "nvidia/MiniMax-M3-NVFP4/block_sparse_moe" + ] + }, + { + "type": "moe", + "args": { + "tokens": 256, + "activation": { + "alpha": 1.702, + "beta": 1.0, + "kind": "swigluoai", + "limit": 7.0 + }, + "experts": { + "a1": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "group": [ + 1, + 16 + ], + "static": false + }, + "scale2": { + "dtype": "fp32", + "group": [ + -1, + -1 + ], + "static": true + } + }, + "a2": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "group": [ + 1, + 16 + ], + "static": false + }, + "scale2": { + "dtype": "fp32", + "group": [ + -1, + -1 + ], + "static": true + } + }, + "inter": 3072, + "num": 128, + "top_k": 4, + "w1": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "group": [ + 1, + 16 + ], + "static": true + }, + "scale2": { + "dtype": "fp32", + "group": [ + -1, + -1 + ], + "static": true + } + }, + "w2": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "group": [ + 1, + 16 + ], + "static": true + }, + "scale2": { + "dtype": "fp32", + "group": [ + -1, + -1 + ], + "static": true + } + } + }, + "hidden": 6144, + "router": { + "bias": true, + "gate": { + "dtype": "fp32" + }, + "renormalize": true, + "scale": 2.0, + "scoring": "sigmoid", + "select": { + "kind": "topk" + } + }, + "routing": { + "distribution": "natural", + "seed": 0 + }, + "shared": { + "a1": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "count": 1, + "inter": 3072, + "w1": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + }, + "w2": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + } + }, + "parallel": { + "tp": 8 + } + }, + "sources": [ + "nvidia/MiniMax-M3-NVFP4/block_sparse_moe" + ] + }, + { + "type": "moe", + "args": { + "tokens": 8, + "activation": { + "alpha": 1.702, + "beta": 1.0, + "kind": "swigluoai", + "limit": 7.0 + }, + "experts": { + "a1": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "a2": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "inter": 3072, + "num": 128, + "top_k": 4, + "w1": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + }, + "w2": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + } + }, + "hidden": 6144, + "router": { + "bias": true, + "gate": { + "dtype": "fp32" + }, + "renormalize": true, + "scale": 2.0, + "scoring": "sigmoid", + "select": { + "kind": "topk" + } + }, + "routing": { + "distribution": "natural", + "seed": 0 + }, + "shared": { + "a1": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "count": 1, + "inter": 3072, + "w1": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + }, + "w2": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + } + }, + "parallel": { + "tp": 8 + } + }, + "sources": [ + "amd/MiniMax-M3-MXFP4/block_sparse_moe" + ] + }, + { + "type": "moe", + "args": { + "tokens": 256, + "activation": { + "alpha": 1.702, + "beta": 1.0, + "kind": "swigluoai", + "limit": 7.0 + }, + "experts": { + "a1": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "a2": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "inter": 3072, + "num": 128, + "top_k": 4, + "w1": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + }, + "w2": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + } + }, + "hidden": 6144, + "router": { + "bias": true, + "gate": { + "dtype": "fp32" + }, + "renormalize": true, + "scale": 2.0, + "scoring": "sigmoid", + "select": { + "kind": "topk" + } + }, + "routing": { + "distribution": "natural", + "seed": 0 + }, + "shared": { + "a1": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "count": 1, + "inter": 3072, + "w1": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + }, + "w2": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + } + }, + "parallel": { + "tp": 8 + } + }, + "sources": [ + "amd/MiniMax-M3-MXFP4/block_sparse_moe" + ] + }, + { + "type": "moe", + "args": { + "tokens": 8, + "activation": { + "kind": "silu", + "limit": 10.0 + }, + "experts": { + "a1": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "a2": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "inter": 3072, + "num": 384, + "top_k": 6, + "w1": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + }, + "w2": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + } + }, + "hidden": 7168, + "router": { + "bias": true, + "gate": { + "dtype": "bf16", + "logits": "fp32" + }, + "renormalize": true, + "scale": 2.5, + "scoring": "sqrtsoftplus", + "select": { + "kind": "topk" + } + }, + "routing": { + "distribution": "natural", + "seed": 0 + }, + "shared": { + "a1": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 128 + ], + "static": false + } + }, + "count": 1, + "inter": 3072, + "w1": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + }, + "w2": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + } + }, + "parallel": { + "tp": 4 + } + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro/ffn" + ] + }, + { + "type": "moe", + "args": { + "tokens": 8, + "activation": { + "kind": "silu", + "limit": 10.0 + }, + "experts": { + "a1": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "a2": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "inter": 3072, + "num": 384, + "top_k": 6, + "w1": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + }, + "w2": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + } + }, + "hidden": 7168, + "router": { + "bias": true, + "gate": { + "dtype": "bf16", + "logits": "fp32" + }, + "renormalize": true, + "scale": 2.5, + "scoring": "sqrtsoftplus", + "select": { + "kind": "topk" + } + }, + "routing": { + "distribution": "natural", + "seed": 0 + }, + "shared": { + "a1": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 128 + ], + "static": false + } + }, + "count": 1, + "inter": 3072, + "w1": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + }, + "w2": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + } + }, + "parallel": { + "tp": 8 + } + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro/ffn" + ] + }, + { + "type": "moe", + "args": { + "tokens": 8, + "activation": { + "kind": "silu", + "limit": 10.0 + }, + "experts": { + "a1": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "a2": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "inter": 3072, + "num": 384, + "top_k": 6, + "w1": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + }, + "w2": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + } + }, + "hidden": 7168, + "router": { + "bias": true, + "gate": { + "dtype": "bf16", + "logits": "fp32" + }, + "renormalize": true, + "scale": 2.5, + "scoring": "sqrtsoftplus", + "select": { + "kind": "topk" + } + }, + "routing": { + "distribution": "natural", + "seed": 0 + }, + "shared": { + "a1": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 128 + ], + "static": false + } + }, + "count": 1, + "inter": 3072, + "w1": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + }, + "w2": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + } + }, + "parallel": { + "dp": 4, + "ep": 4 + } + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro/ffn" + ] + }, + { + "type": "moe", + "args": { + "tokens": 8, + "activation": { + "kind": "silu", + "limit": 10.0 + }, + "experts": { + "a1": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "a2": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "inter": 3072, + "num": 384, + "top_k": 6, + "w1": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + }, + "w2": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + } + }, + "hidden": 7168, + "router": { + "bias": true, + "gate": { + "dtype": "bf16", + "logits": "fp32" + }, + "renormalize": true, + "scale": 2.5, + "scoring": "sqrtsoftplus", + "select": { + "kind": "topk" + } + }, + "routing": { + "distribution": "natural", + "seed": 0 + }, + "shared": { + "a1": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 128 + ], + "static": false + } + }, + "count": 1, + "inter": 3072, + "w1": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + }, + "w2": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + } + }, + "parallel": { + "dp": 8, + "ep": 8 + } + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro/ffn" + ] + }, + { + "type": "moe", + "args": { + "tokens": 8, + "activation": { + "kind": "silu", + "limit": 10.0 + }, + "experts": { + "a1": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "a2": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "inter": 3072, + "num": 384, + "top_k": 6, + "w1": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + }, + "w2": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + } + }, + "hidden": 7168, + "router": { + "bias": true, + "gate": { + "dtype": "bf16", + "logits": "fp32" + }, + "renormalize": true, + "scale": 2.5, + "scoring": "sqrtsoftplus", + "select": { + "kind": "topk" + } + }, + "routing": { + "distribution": "natural", + "seed": 0 + }, + "shared": { + "a1": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 128 + ], + "static": false + } + }, + "count": 1, + "inter": 3072, + "w1": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + }, + "w2": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + } + }, + "parallel": { + "dp": 8 + } + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn" + ] + }, + { + "type": "moe", + "args": { + "tokens": 256, + "activation": { + "kind": "silu", + "limit": 10.0 + }, + "experts": { + "a1": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "a2": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "inter": 3072, + "num": 384, + "top_k": 6, + "w1": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + }, + "w2": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + } + }, + "hidden": 7168, + "router": { + "bias": true, + "gate": { + "dtype": "bf16", + "logits": "fp32" + }, + "renormalize": true, + "scale": 2.5, + "scoring": "sqrtsoftplus", + "select": { + "kind": "topk" + } + }, + "routing": { + "distribution": "natural", + "seed": 0 + }, + "shared": { + "a1": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 128 + ], + "static": false + } + }, + "count": 1, + "inter": 3072, + "w1": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + }, + "w2": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + } + }, + "parallel": { + "tp": 4 + } + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro/ffn" + ] + }, + { + "type": "moe", + "args": { + "tokens": 256, + "activation": { + "kind": "silu", + "limit": 10.0 + }, + "experts": { + "a1": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "a2": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "inter": 3072, + "num": 384, + "top_k": 6, + "w1": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + }, + "w2": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + } + }, + "hidden": 7168, + "router": { + "bias": true, + "gate": { + "dtype": "bf16", + "logits": "fp32" + }, + "renormalize": true, + "scale": 2.5, + "scoring": "sqrtsoftplus", + "select": { + "kind": "topk" + } + }, + "routing": { + "distribution": "natural", + "seed": 0 + }, + "shared": { + "a1": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 128 + ], + "static": false + } + }, + "count": 1, + "inter": 3072, + "w1": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + }, + "w2": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + } + }, + "parallel": { + "tp": 8 + } + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro/ffn" + ] + }, + { + "type": "moe", + "args": { + "tokens": 256, + "activation": { + "kind": "silu", + "limit": 10.0 + }, + "experts": { + "a1": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "a2": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "inter": 3072, + "num": 384, + "top_k": 6, + "w1": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + }, + "w2": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + } + }, + "hidden": 7168, + "router": { + "bias": true, + "gate": { + "dtype": "bf16", + "logits": "fp32" + }, + "renormalize": true, + "scale": 2.5, + "scoring": "sqrtsoftplus", + "select": { + "kind": "topk" + } + }, + "routing": { + "distribution": "natural", + "seed": 0 + }, + "shared": { + "a1": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 128 + ], + "static": false + } + }, + "count": 1, + "inter": 3072, + "w1": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + }, + "w2": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + } + }, + "parallel": { + "dp": 4, + "ep": 4 + } + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro/ffn" + ] + }, + { + "type": "moe", + "args": { + "tokens": 256, + "activation": { + "kind": "silu", + "limit": 10.0 + }, + "experts": { + "a1": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "a2": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "inter": 3072, + "num": 384, + "top_k": 6, + "w1": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + }, + "w2": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + } + }, + "hidden": 7168, + "router": { + "bias": true, + "gate": { + "dtype": "bf16", + "logits": "fp32" + }, + "renormalize": true, + "scale": 2.5, + "scoring": "sqrtsoftplus", + "select": { + "kind": "topk" + } + }, + "routing": { + "distribution": "natural", + "seed": 0 + }, + "shared": { + "a1": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 128 + ], + "static": false + } + }, + "count": 1, + "inter": 3072, + "w1": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + }, + "w2": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + } + }, + "parallel": { + "dp": 8, + "ep": 8 + } + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro/ffn" + ] + }, + { + "type": "moe", + "args": { + "tokens": 256, + "activation": { + "kind": "silu", + "limit": 10.0 + }, + "experts": { + "a1": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "a2": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "inter": 3072, + "num": 384, + "top_k": 6, + "w1": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + }, + "w2": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + } + }, + "hidden": 7168, + "router": { + "bias": true, + "gate": { + "dtype": "bf16", + "logits": "fp32" + }, + "renormalize": true, + "scale": 2.5, + "scoring": "sqrtsoftplus", + "select": { + "kind": "topk" + } + }, + "routing": { + "distribution": "natural", + "seed": 0 + }, + "shared": { + "a1": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 128 + ], + "static": false + } + }, + "count": 1, + "inter": 3072, + "w1": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + }, + "w2": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + } + }, + "parallel": { + "dp": 8 + } + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn" + ] + }, + { + "type": "moe", + "args": { + "tokens": 8, + "activation": { + "kind": "silu", + "limit": 10.0 + }, + "experts": { + "a1": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "a2": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "inter": 3072, + "num": 384, + "top_k": 6, + "w1": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + }, + "w2": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + } + }, + "hidden": 7168, + "router": { + "bias": false, + "gate": { + "dtype": "bf16", + "logits": "fp32" + }, + "renormalize": true, + "scale": 2.5, + "scoring": "sqrtsoftplus", + "select": { + "kind": "hash", + "vocab": 129280 + } + }, + "routing": { + "distribution": "natural", + "seed": 0 + }, + "shared": { + "a1": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 128 + ], + "static": false + } + }, + "count": 1, + "inter": 3072, + "w1": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + }, + "w2": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + } + }, + "parallel": { + "tp": 4 + } + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro/ffn" + ] + }, + { + "type": "moe", + "args": { + "tokens": 8, + "activation": { + "kind": "silu", + "limit": 10.0 + }, + "experts": { + "a1": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "a2": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "inter": 3072, + "num": 384, + "top_k": 6, + "w1": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + }, + "w2": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + } + }, + "hidden": 7168, + "router": { + "bias": false, + "gate": { + "dtype": "bf16", + "logits": "fp32" + }, + "renormalize": true, + "scale": 2.5, + "scoring": "sqrtsoftplus", + "select": { + "kind": "hash", + "vocab": 129280 + } + }, + "routing": { + "distribution": "natural", + "seed": 0 + }, + "shared": { + "a1": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 128 + ], + "static": false + } + }, + "count": 1, + "inter": 3072, + "w1": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + }, + "w2": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + } + }, + "parallel": { + "tp": 8 + } + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro/ffn" + ] + }, + { + "type": "moe", + "args": { + "tokens": 8, + "activation": { + "kind": "silu", + "limit": 10.0 + }, + "experts": { + "a1": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "a2": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "inter": 3072, + "num": 384, + "top_k": 6, + "w1": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + }, + "w2": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + } + }, + "hidden": 7168, + "router": { + "bias": false, + "gate": { + "dtype": "bf16", + "logits": "fp32" + }, + "renormalize": true, + "scale": 2.5, + "scoring": "sqrtsoftplus", + "select": { + "kind": "hash", + "vocab": 129280 + } + }, + "routing": { + "distribution": "natural", + "seed": 0 + }, + "shared": { + "a1": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 128 + ], + "static": false + } + }, + "count": 1, + "inter": 3072, + "w1": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + }, + "w2": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + } + }, + "parallel": { + "dp": 4, + "ep": 4 + } + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro/ffn" + ] + }, + { + "type": "moe", + "args": { + "tokens": 8, + "activation": { + "kind": "silu", + "limit": 10.0 + }, + "experts": { + "a1": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "a2": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "inter": 3072, + "num": 384, + "top_k": 6, + "w1": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + }, + "w2": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + } + }, + "hidden": 7168, + "router": { + "bias": false, + "gate": { + "dtype": "bf16", + "logits": "fp32" + }, + "renormalize": true, + "scale": 2.5, + "scoring": "sqrtsoftplus", + "select": { + "kind": "hash", + "vocab": 129280 + } + }, + "routing": { + "distribution": "natural", + "seed": 0 + }, + "shared": { + "a1": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 128 + ], + "static": false + } + }, + "count": 1, + "inter": 3072, + "w1": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + }, + "w2": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + } + }, + "parallel": { + "dp": 8, + "ep": 8 + } + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro/ffn" + ] + }, + { + "type": "moe", + "args": { + "tokens": 8, + "activation": { + "kind": "silu", + "limit": 10.0 + }, + "experts": { + "a1": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "a2": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "inter": 3072, + "num": 384, + "top_k": 6, + "w1": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + }, + "w2": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + } + }, + "hidden": 7168, + "router": { + "bias": false, + "gate": { + "dtype": "bf16", + "logits": "fp32" + }, + "renormalize": true, + "scale": 2.5, + "scoring": "sqrtsoftplus", + "select": { + "kind": "hash", + "vocab": 129280 + } + }, + "routing": { + "distribution": "natural", + "seed": 0 + }, + "shared": { + "a1": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 128 + ], + "static": false + } + }, + "count": 1, + "inter": 3072, + "w1": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + }, + "w2": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + } + }, + "parallel": { + "dp": 8 + } + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn" + ] + }, + { + "type": "moe", + "args": { + "tokens": 256, + "activation": { + "kind": "silu", + "limit": 10.0 + }, + "experts": { + "a1": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "a2": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "inter": 3072, + "num": 384, + "top_k": 6, + "w1": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + }, + "w2": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + } + }, + "hidden": 7168, + "router": { + "bias": false, + "gate": { + "dtype": "bf16", + "logits": "fp32" + }, + "renormalize": true, + "scale": 2.5, + "scoring": "sqrtsoftplus", + "select": { + "kind": "hash", + "vocab": 129280 + } + }, + "routing": { + "distribution": "natural", + "seed": 0 + }, + "shared": { + "a1": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 128 + ], + "static": false + } + }, + "count": 1, + "inter": 3072, + "w1": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + }, + "w2": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + } + }, + "parallel": { + "tp": 4 + } + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro/ffn" + ] + }, + { + "type": "moe", + "args": { + "tokens": 256, + "activation": { + "kind": "silu", + "limit": 10.0 + }, + "experts": { + "a1": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "a2": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "inter": 3072, + "num": 384, + "top_k": 6, + "w1": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + }, + "w2": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + } + }, + "hidden": 7168, + "router": { + "bias": false, + "gate": { + "dtype": "bf16", + "logits": "fp32" + }, + "renormalize": true, + "scale": 2.5, + "scoring": "sqrtsoftplus", + "select": { + "kind": "hash", + "vocab": 129280 + } + }, + "routing": { + "distribution": "natural", + "seed": 0 + }, + "shared": { + "a1": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 128 + ], + "static": false + } + }, + "count": 1, + "inter": 3072, + "w1": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + }, + "w2": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + } + }, + "parallel": { + "tp": 8 + } + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro/ffn" + ] + }, + { + "type": "moe", + "args": { + "tokens": 256, + "activation": { + "kind": "silu", + "limit": 10.0 + }, + "experts": { + "a1": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "a2": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "inter": 3072, + "num": 384, + "top_k": 6, + "w1": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + }, + "w2": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + } + }, + "hidden": 7168, + "router": { + "bias": false, + "gate": { + "dtype": "bf16", + "logits": "fp32" + }, + "renormalize": true, + "scale": 2.5, + "scoring": "sqrtsoftplus", + "select": { + "kind": "hash", + "vocab": 129280 + } + }, + "routing": { + "distribution": "natural", + "seed": 0 + }, + "shared": { + "a1": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 128 + ], + "static": false + } + }, + "count": 1, + "inter": 3072, + "w1": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + }, + "w2": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + } + }, + "parallel": { + "dp": 4, + "ep": 4 + } + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro/ffn" + ] + }, + { + "type": "moe", + "args": { + "tokens": 256, + "activation": { + "kind": "silu", + "limit": 10.0 + }, + "experts": { + "a1": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "a2": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "inter": 3072, + "num": 384, + "top_k": 6, + "w1": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + }, + "w2": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + } + }, + "hidden": 7168, + "router": { + "bias": false, + "gate": { + "dtype": "bf16", + "logits": "fp32" + }, + "renormalize": true, + "scale": 2.5, + "scoring": "sqrtsoftplus", + "select": { + "kind": "hash", + "vocab": 129280 + } + }, + "routing": { + "distribution": "natural", + "seed": 0 + }, + "shared": { + "a1": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 128 + ], + "static": false + } + }, + "count": 1, + "inter": 3072, + "w1": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + }, + "w2": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + } + }, + "parallel": { + "dp": 8, + "ep": 8 + } + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro/ffn" + ] + }, + { + "type": "moe", + "args": { + "tokens": 256, + "activation": { + "kind": "silu", + "limit": 10.0 + }, + "experts": { + "a1": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "a2": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "inter": 3072, + "num": 384, + "top_k": 6, + "w1": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + }, + "w2": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + } + }, + "hidden": 7168, + "router": { + "bias": false, + "gate": { + "dtype": "bf16", + "logits": "fp32" + }, + "renormalize": true, + "scale": 2.5, + "scoring": "sqrtsoftplus", + "select": { + "kind": "hash", + "vocab": 129280 + } + }, + "routing": { + "distribution": "natural", + "seed": 0 + }, + "shared": { + "a1": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 128 + ], + "static": false + } + }, + "count": 1, + "inter": 3072, + "w1": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + }, + "w2": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + } + }, + "parallel": { + "dp": 8 + } + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn" + ] + }, + { + "type": "moe", + "args": { + "tokens": 8, + "activation": { + "kind": "silu", + "limit": 10.0 + }, + "experts": { + "a1": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "a2": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "inter": 2304, + "num": 384, + "top_k": 6, + "w1": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + }, + "w2": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + } + }, + "hidden": 5120, + "router": { + "bias": true, + "gate": { + "dtype": "bf16", + "logits": "fp32" + }, + "renormalize": true, + "scale": 1.5, + "scoring": "sqrtsoftplus", + "select": { + "kind": "topk" + } + }, + "routing": { + "distribution": "natural", + "seed": 0 + }, + "shared": { + "a1": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "count": 1, + "inter": 2304, + "w1": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + }, + "w2": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + }, + "parallel": { + "tp": 2 + } + }, + "sources": [ + "deepseek-ai/DeepSeek-V4.1-Flash/ffn" + ] + }, + { + "type": "moe", + "args": { + "tokens": 8, + "activation": { + "kind": "silu", + "limit": 10.0 + }, + "experts": { + "a1": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "a2": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "inter": 2304, + "num": 384, + "top_k": 6, + "w1": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + }, + "w2": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + } + }, + "hidden": 5120, + "router": { + "bias": true, + "gate": { + "dtype": "bf16", + "logits": "fp32" + }, + "renormalize": true, + "scale": 1.5, + "scoring": "sqrtsoftplus", + "select": { + "kind": "topk" + } + }, + "routing": { + "distribution": "natural", + "seed": 0 + }, + "shared": { + "a1": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "count": 1, + "inter": 2304, + "w1": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + }, + "w2": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + }, + "parallel": { + "tp": 4 + } + }, + "sources": [ + "deepseek-ai/DeepSeek-V4.1-Flash/ffn" + ] + }, + { + "type": "moe", + "args": { + "tokens": 8, + "activation": { + "kind": "silu", + "limit": 10.0 + }, + "experts": { + "a1": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "a2": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "inter": 2304, + "num": 384, + "top_k": 6, + "w1": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + }, + "w2": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + } + }, + "hidden": 5120, + "router": { + "bias": true, + "gate": { + "dtype": "bf16", + "logits": "fp32" + }, + "renormalize": true, + "scale": 1.5, + "scoring": "sqrtsoftplus", + "select": { + "kind": "topk" + } + }, + "routing": { + "distribution": "natural", + "seed": 0 + }, + "shared": { + "a1": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "count": 1, + "inter": 2304, + "w1": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + }, + "w2": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + }, + "parallel": { + "tp": 8 + } + }, + "sources": [ + "deepseek-ai/DeepSeek-V4.1-Flash/ffn" + ] + }, + { + "type": "moe", + "args": { + "tokens": 8, + "activation": { + "kind": "silu", + "limit": 10.0 + }, + "experts": { + "a1": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "a2": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "inter": 2304, + "num": 384, + "top_k": 6, + "w1": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + }, + "w2": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + } + }, + "hidden": 5120, + "router": { + "bias": true, + "gate": { + "dtype": "bf16", + "logits": "fp32" + }, + "renormalize": true, + "scale": 1.5, + "scoring": "sqrtsoftplus", + "select": { + "kind": "topk" + } + }, + "routing": { + "distribution": "natural", + "seed": 0 + }, + "shared": { + "a1": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "count": 1, + "inter": 2304, + "w1": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + }, + "w2": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + }, + "parallel": { + "tp": 2, + "ep": 2 + } + }, + "sources": [ + "deepseek-ai/DeepSeek-V4.1-Flash/ffn" + ] + }, + { + "type": "moe", + "args": { + "tokens": 8, + "activation": { + "kind": "silu", + "limit": 10.0 + }, + "experts": { + "a1": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "a2": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "inter": 2304, + "num": 384, + "top_k": 6, + "w1": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + }, + "w2": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + } + }, + "hidden": 5120, + "router": { + "bias": true, + "gate": { + "dtype": "bf16", + "logits": "fp32" + }, + "renormalize": true, + "scale": 1.5, + "scoring": "sqrtsoftplus", + "select": { + "kind": "topk" + } + }, + "routing": { + "distribution": "natural", + "seed": 0 + }, + "shared": { + "a1": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "count": 1, + "inter": 2304, + "w1": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + }, + "w2": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + }, + "parallel": { + "tp": 4, + "ep": 4 + } + }, + "sources": [ + "deepseek-ai/DeepSeek-V4.1-Flash/ffn" + ] + }, + { + "type": "moe", + "args": { + "tokens": 8, + "activation": { + "kind": "silu", + "limit": 10.0 + }, + "experts": { + "a1": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "a2": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "inter": 2304, + "num": 384, + "top_k": 6, + "w1": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + }, + "w2": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + } + }, + "hidden": 5120, + "router": { + "bias": true, + "gate": { + "dtype": "bf16", + "logits": "fp32" + }, + "renormalize": true, + "scale": 1.5, + "scoring": "sqrtsoftplus", + "select": { + "kind": "topk" + } + }, + "routing": { + "distribution": "natural", + "seed": 0 + }, + "shared": { + "a1": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "count": 1, + "inter": 2304, + "w1": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + }, + "w2": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + }, + "parallel": { + "tp": 8, + "ep": 8 + } + }, + "sources": [ + "deepseek-ai/DeepSeek-V4.1-Flash/ffn" + ] + }, + { + "type": "moe", + "args": { + "tokens": 8, + "activation": { + "kind": "silu", + "limit": 10.0 + }, + "experts": { + "a1": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "a2": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "inter": 2304, + "num": 384, + "top_k": 6, + "w1": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + }, + "w2": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + } + }, + "hidden": 5120, + "router": { + "bias": true, + "gate": { + "dtype": "bf16", + "logits": "fp32" + }, + "renormalize": true, + "scale": 1.5, + "scoring": "sqrtsoftplus", + "select": { + "kind": "topk" + } + }, + "routing": { + "distribution": "natural", + "seed": 0 + }, + "shared": { + "a1": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "count": 1, + "inter": 2304, + "w1": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + }, + "w2": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + }, + "parallel": { + "dp": 8, + "ep": 8 + } + }, + "sources": [ + "deepseek-ai/DeepSeek-V4.1-Flash/ffn" + ] + }, + { + "type": "moe", + "args": { + "tokens": 256, + "activation": { + "kind": "silu", + "limit": 10.0 + }, + "experts": { + "a1": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "a2": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "inter": 2304, + "num": 384, + "top_k": 6, + "w1": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + }, + "w2": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + } + }, + "hidden": 5120, + "router": { + "bias": true, + "gate": { + "dtype": "bf16", + "logits": "fp32" + }, + "renormalize": true, + "scale": 1.5, + "scoring": "sqrtsoftplus", + "select": { + "kind": "topk" + } + }, + "routing": { + "distribution": "natural", + "seed": 0 + }, + "shared": { + "a1": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "count": 1, + "inter": 2304, + "w1": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + }, + "w2": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + }, + "parallel": { + "tp": 2 + } + }, + "sources": [ + "deepseek-ai/DeepSeek-V4.1-Flash/ffn" + ] + }, + { + "type": "moe", + "args": { + "tokens": 256, + "activation": { + "kind": "silu", + "limit": 10.0 + }, + "experts": { + "a1": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "a2": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "inter": 2304, + "num": 384, + "top_k": 6, + "w1": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + }, + "w2": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + } + }, + "hidden": 5120, + "router": { + "bias": true, + "gate": { + "dtype": "bf16", + "logits": "fp32" + }, + "renormalize": true, + "scale": 1.5, + "scoring": "sqrtsoftplus", + "select": { + "kind": "topk" + } + }, + "routing": { + "distribution": "natural", + "seed": 0 + }, + "shared": { + "a1": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "count": 1, + "inter": 2304, + "w1": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + }, + "w2": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + }, + "parallel": { + "tp": 4 + } + }, + "sources": [ + "deepseek-ai/DeepSeek-V4.1-Flash/ffn" + ] + }, + { + "type": "moe", + "args": { + "tokens": 256, + "activation": { + "kind": "silu", + "limit": 10.0 + }, + "experts": { + "a1": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "a2": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "inter": 2304, + "num": 384, + "top_k": 6, + "w1": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + }, + "w2": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + } + }, + "hidden": 5120, + "router": { + "bias": true, + "gate": { + "dtype": "bf16", + "logits": "fp32" + }, + "renormalize": true, + "scale": 1.5, + "scoring": "sqrtsoftplus", + "select": { + "kind": "topk" + } + }, + "routing": { + "distribution": "natural", + "seed": 0 + }, + "shared": { + "a1": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "count": 1, + "inter": 2304, + "w1": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + }, + "w2": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + }, + "parallel": { + "tp": 8 + } + }, + "sources": [ + "deepseek-ai/DeepSeek-V4.1-Flash/ffn" + ] + }, + { + "type": "moe", + "args": { + "tokens": 256, + "activation": { + "kind": "silu", + "limit": 10.0 + }, + "experts": { + "a1": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "a2": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "inter": 2304, + "num": 384, + "top_k": 6, + "w1": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + }, + "w2": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + } + }, + "hidden": 5120, + "router": { + "bias": true, + "gate": { + "dtype": "bf16", + "logits": "fp32" + }, + "renormalize": true, + "scale": 1.5, + "scoring": "sqrtsoftplus", + "select": { + "kind": "topk" + } + }, + "routing": { + "distribution": "natural", + "seed": 0 + }, + "shared": { + "a1": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "count": 1, + "inter": 2304, + "w1": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + }, + "w2": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + }, + "parallel": { + "tp": 2, + "ep": 2 + } + }, + "sources": [ + "deepseek-ai/DeepSeek-V4.1-Flash/ffn" + ] + }, + { + "type": "moe", + "args": { + "tokens": 256, + "activation": { + "kind": "silu", + "limit": 10.0 + }, + "experts": { + "a1": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "a2": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "inter": 2304, + "num": 384, + "top_k": 6, + "w1": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + }, + "w2": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + } + }, + "hidden": 5120, + "router": { + "bias": true, + "gate": { + "dtype": "bf16", + "logits": "fp32" + }, + "renormalize": true, + "scale": 1.5, + "scoring": "sqrtsoftplus", + "select": { + "kind": "topk" + } + }, + "routing": { + "distribution": "natural", + "seed": 0 + }, + "shared": { + "a1": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "count": 1, + "inter": 2304, + "w1": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + }, + "w2": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + }, + "parallel": { + "tp": 4, + "ep": 4 + } + }, + "sources": [ + "deepseek-ai/DeepSeek-V4.1-Flash/ffn" + ] + }, + { + "type": "moe", + "args": { + "tokens": 256, + "activation": { + "kind": "silu", + "limit": 10.0 + }, + "experts": { + "a1": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "a2": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "inter": 2304, + "num": 384, + "top_k": 6, + "w1": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + }, + "w2": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + } + }, + "hidden": 5120, + "router": { + "bias": true, + "gate": { + "dtype": "bf16", + "logits": "fp32" + }, + "renormalize": true, + "scale": 1.5, + "scoring": "sqrtsoftplus", + "select": { + "kind": "topk" + } + }, + "routing": { + "distribution": "natural", + "seed": 0 + }, + "shared": { + "a1": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "count": 1, + "inter": 2304, + "w1": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + }, + "w2": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + }, + "parallel": { + "tp": 8, + "ep": 8 + } + }, + "sources": [ + "deepseek-ai/DeepSeek-V4.1-Flash/ffn" + ] + }, + { + "type": "moe", + "args": { + "tokens": 256, + "activation": { + "kind": "silu", + "limit": 10.0 + }, + "experts": { + "a1": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "a2": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "inter": 2304, + "num": 384, + "top_k": 6, + "w1": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + }, + "w2": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + } + }, + "hidden": 5120, + "router": { + "bias": true, + "gate": { + "dtype": "bf16", + "logits": "fp32" + }, + "renormalize": true, + "scale": 1.5, + "scoring": "sqrtsoftplus", + "select": { + "kind": "topk" + } + }, + "routing": { + "distribution": "natural", + "seed": 0 + }, + "shared": { + "a1": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "count": 1, + "inter": 2304, + "w1": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + }, + "w2": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + }, + "parallel": { + "dp": 8, + "ep": 8 + } + }, + "sources": [ + "deepseek-ai/DeepSeek-V4.1-Flash/ffn" + ] + }, + { + "type": "moe", + "args": { + "tokens": 8, + "activation": { + "alpha": 4.0, + "beta": 25.0, + "kind": "situ" + }, + "experts": { + "a1": { + "dtype": "bf16" + }, + "a2": { + "dtype": "bf16" + }, + "inter": 3072, + "latent": 3584, + "latent_norm": true, + "num": 896, + "top_k": 16, + "w1": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + }, + "w2": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + } + }, + "hidden": 7168, + "router": { + "bias": true, + "gate": { + "dtype": "bf16", + "logits": "fp32" + }, + "renormalize": true, + "scale": 1.0, + "scoring": "sigmoid", + "select": { + "groups": 1, + "kind": "grouped_topk", + "topk_groups": 1 + } + }, + "routing": { + "distribution": "natural", + "seed": 0 + }, + "shared": { + "a1": { + "dtype": "bf16" + }, + "count": 2, + "inter": 3072, + "w1": { + "dtype": "bf16" + }, + "w2": { + "dtype": "bf16" + } + }, + "parallel": { + "tp": 8 + } + }, + "sources": [ + "moonshotai/Kimi-K3/block_sparse_moe" + ] + }, + { + "type": "moe", + "args": { + "tokens": 256, + "activation": { + "alpha": 4.0, + "beta": 25.0, + "kind": "situ" + }, + "experts": { + "a1": { + "dtype": "bf16" + }, + "a2": { + "dtype": "bf16" + }, + "inter": 3072, + "latent": 3584, + "latent_norm": true, + "num": 896, + "top_k": 16, + "w1": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + }, + "w2": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + } + }, + "hidden": 7168, + "router": { + "bias": true, + "gate": { + "dtype": "bf16", + "logits": "fp32" + }, + "renormalize": true, + "scale": 1.0, + "scoring": "sigmoid", + "select": { + "groups": 1, + "kind": "grouped_topk", + "topk_groups": 1 + } + }, + "routing": { + "distribution": "natural", + "seed": 0 + }, + "shared": { + "a1": { + "dtype": "bf16" + }, + "count": 2, + "inter": 3072, + "w1": { + "dtype": "bf16" + }, + "w2": { + "dtype": "bf16" + } + }, + "parallel": { + "tp": 8 + } + }, + "sources": [ + "moonshotai/Kimi-K3/block_sparse_moe" + ] + } +] \ No newline at end of file diff --git a/operatorx/testlists/moe_small.json b/operatorx/testlists/moe_small.json new file mode 100644 index 0000000000..dc85519c60 --- /dev/null +++ b/operatorx/testlists/moe_small.json @@ -0,0 +1,6924 @@ +[ + { + "type": "moe", + "args": { + "tokens": 1, + "hidden": 7168, + "experts": { + "num": 256, + "top_k": 8, + "inter": 2048, + "a1": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "w1": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "w2": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "a2": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + } + }, + "router": { + "gate": { + "dtype": "bf16" + }, + "scoring": "sigmoid", + "select": { + "kind": "grouped_topk", + "groups": 8, + "topk_groups": 4 + }, + "bias": true, + "renormalize": true, + "scale": 2.5 + }, + "activation": { + "kind": "silu" + }, + "shared": { + "count": 1, + "inter": 2048, + "a1": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "w1": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "w2": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + } + }, + "routing": { + "distribution": "natural", + "seed": 0 + } + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/mlp" + ] + }, + { + "type": "moe", + "args": { + "tokens": 1, + "hidden": 7168, + "experts": { + "num": 256, + "top_k": 8, + "inter": 2048, + "a1": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "w1": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "w2": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "a2": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + } + }, + "router": { + "gate": { + "dtype": "bf16" + }, + "scoring": "sigmoid", + "select": { + "kind": "grouped_topk", + "groups": 8, + "topk_groups": 4 + }, + "bias": true, + "renormalize": true, + "scale": 2.5 + }, + "activation": { + "kind": "silu" + }, + "shared": { + "count": 1, + "inter": 2048, + "a1": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "w1": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "w2": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + } + }, + "routing": { + "distribution": "natural", + "seed": 0 + } + }, + "sources": [ + "nvidia/DeepSeek-R1-0528-NVFP4-v2/mlp" + ] + }, + { + "type": "moe", + "args": { + "tokens": 1, + "hidden": 7168, + "experts": { + "num": 256, + "top_k": 8, + "inter": 2048, + "a1": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 32 + ] + } + }, + "w1": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 1, + 32 + ] + } + }, + "w2": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 1, + 32 + ] + } + }, + "a2": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 32 + ] + } + } + }, + "router": { + "gate": { + "dtype": "bf16" + }, + "scoring": "sigmoid", + "select": { + "kind": "grouped_topk", + "groups": 8, + "topk_groups": 4 + }, + "bias": true, + "renormalize": true, + "scale": 2.5 + }, + "activation": { + "kind": "silu" + }, + "shared": { + "count": 1, + "inter": 2048, + "a1": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 32 + ] + } + }, + "w1": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 1, + 32 + ] + } + }, + "w2": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 1, + 32 + ] + } + } + }, + "routing": { + "distribution": "natural", + "seed": 0 + } + }, + "sources": [ + "amd/DeepSeek-R1-0528-MXFP4/mlp" + ] + }, + { + "type": "moe", + "args": { + "tokens": 1, + "hidden": 7168, + "experts": { + "num": 384, + "top_k": 6, + "inter": 3072, + "a1": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 32 + ] + } + }, + "w1": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 1, + 32 + ] + } + }, + "w2": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 1, + 32 + ] + } + }, + "a2": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 32 + ] + } + } + }, + "router": { + "gate": { + "dtype": "bf16", + "logits": "fp32" + }, + "scoring": "sqrtsoftplus", + "select": { + "kind": "topk" + }, + "bias": true, + "renormalize": true, + "scale": 2.5 + }, + "activation": { + "kind": "silu", + "limit": 10.0 + }, + "shared": { + "count": 1, + "inter": 3072, + "a1": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "w1": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "w2": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + } + }, + "routing": { + "distribution": "natural", + "seed": 0 + } + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro/ffn" + ] + }, + { + "type": "moe", + "args": { + "tokens": 1, + "hidden": 7168, + "experts": { + "num": 384, + "top_k": 6, + "inter": 3072, + "a1": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 32 + ] + } + }, + "w1": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 1, + 32 + ] + } + }, + "w2": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 1, + 32 + ] + } + }, + "a2": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 32 + ] + } + } + }, + "router": { + "gate": { + "dtype": "bf16", + "logits": "fp32" + }, + "scoring": "sqrtsoftplus", + "select": { + "kind": "hash", + "vocab": 129280 + }, + "bias": false, + "renormalize": true, + "scale": 2.5 + }, + "activation": { + "kind": "silu", + "limit": 10.0 + }, + "shared": { + "count": 1, + "inter": 3072, + "a1": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "w1": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "w2": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + } + }, + "routing": { + "distribution": "natural", + "seed": 0 + } + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro/ffn" + ] + }, + { + "type": "moe", + "args": { + "tokens": 1, + "hidden": 5120, + "experts": { + "num": 384, + "top_k": 6, + "inter": 2304, + "a1": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 32 + ] + } + }, + "w1": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 1, + 32 + ] + } + }, + "w2": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 1, + 32 + ] + } + }, + "a2": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 32 + ] + } + } + }, + "router": { + "gate": { + "dtype": "bf16", + "logits": "fp32" + }, + "scoring": "sqrtsoftplus", + "select": { + "kind": "topk" + }, + "bias": true, + "renormalize": true, + "scale": 1.5 + }, + "activation": { + "kind": "silu", + "limit": 10.0 + }, + "shared": { + "count": 1, + "inter": 2304, + "a1": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 32 + ] + } + }, + "w1": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 32, + 32 + ] + } + }, + "w2": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 32, + 32 + ] + } + } + }, + "routing": { + "distribution": "natural", + "seed": 0 + } + }, + "sources": [ + "deepseek-ai/DeepSeek-V4.1-Flash/ffn" + ] + }, + { + "type": "moe", + "args": { + "tokens": 1, + "hidden": 4096, + "experts": { + "num": 512, + "top_k": 10, + "inter": 1024, + "a1": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "w1": { + "dtype": "e4m3", + "scale": { + "dtype": "bf16", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "w2": { + "dtype": "e4m3", + "scale": { + "dtype": "bf16", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "a2": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + } + }, + "router": { + "gate": { + "dtype": "bf16" + }, + "scoring": "softmax", + "select": { + "kind": "topk" + }, + "renormalize": true + }, + "activation": { + "kind": "silu" + }, + "shared": { + "count": 1, + "inter": 1024, + "a1": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "w1": { + "dtype": "e4m3", + "scale": { + "dtype": "bf16", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "w2": { + "dtype": "e4m3", + "scale": { + "dtype": "bf16", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "gate": "sigmoid" + }, + "routing": { + "distribution": "natural", + "seed": 0 + } + }, + "sources": [ + "Qwen/Qwen3.5-397B-A17B-FP8/mlp" + ] + }, + { + "type": "moe", + "args": { + "tokens": 1, + "hidden": 4096, + "experts": { + "num": 512, + "top_k": 10, + "inter": 1024, + "a1": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "w1": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "w2": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "a2": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + } + }, + "router": { + "gate": { + "dtype": "bf16" + }, + "scoring": "softmax", + "select": { + "kind": "topk" + }, + "renormalize": true + }, + "activation": { + "kind": "silu" + }, + "shared": { + "count": 1, + "inter": 1024, + "a1": { + "dtype": "bf16" + }, + "w1": { + "dtype": "bf16" + }, + "w2": { + "dtype": "bf16" + }, + "gate": "sigmoid" + }, + "routing": { + "distribution": "natural", + "seed": 0 + } + }, + "sources": [ + "nvidia/Qwen3.5-397B-A17B-NVFP4/mlp" + ] + }, + { + "type": "moe", + "args": { + "tokens": 1, + "hidden": 4096, + "experts": { + "num": 512, + "top_k": 10, + "inter": 1024, + "a1": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "w1": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "w2": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "a2": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + } + }, + "router": { + "gate": { + "dtype": "bf16" + }, + "scoring": "softmax", + "select": { + "kind": "topk" + }, + "renormalize": true + }, + "activation": { + "kind": "silu" + }, + "shared": { + "count": 1, + "inter": 1024, + "a1": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "w1": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "w2": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "gate": "sigmoid" + }, + "routing": { + "distribution": "natural", + "seed": 0 + } + }, + "sources": [ + "nvidia/Qwen3.5-397B-A17B-NVFP4-V2/mlp" + ] + }, + { + "type": "moe", + "args": { + "tokens": 1, + "hidden": 4096, + "experts": { + "num": 512, + "top_k": 10, + "inter": 1024, + "a1": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 32 + ] + } + }, + "w1": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 1, + 32 + ] + } + }, + "w2": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 1, + 32 + ] + } + }, + "a2": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 32 + ] + } + } + }, + "router": { + "gate": { + "dtype": "bf16" + }, + "scoring": "softmax", + "select": { + "kind": "topk" + }, + "renormalize": true + }, + "activation": { + "kind": "silu" + }, + "shared": { + "count": 1, + "inter": 1024, + "a1": { + "dtype": "bf16" + }, + "w1": { + "dtype": "bf16" + }, + "w2": { + "dtype": "bf16" + }, + "gate": "sigmoid" + }, + "routing": { + "distribution": "natural", + "seed": 0 + } + }, + "sources": [ + "amd/Qwen3.5-397B-A17B-MXFP4/mlp" + ] + }, + { + "type": "moe", + "args": { + "tokens": 1, + "hidden": 4096, + "experts": { + "num": 512, + "top_k": 10, + "inter": 1024, + "a1": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 32 + ] + } + }, + "w1": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 1, + 32 + ] + } + }, + "w2": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 1, + 32 + ] + } + }, + "a2": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 32 + ] + } + } + }, + "router": { + "gate": { + "dtype": "bf16" + }, + "scoring": "softmax", + "select": { + "kind": "topk" + }, + "renormalize": true + }, + "activation": { + "kind": "silu" + }, + "shared": { + "count": 1, + "inter": 1024, + "a1": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 32 + ] + } + }, + "w1": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 1, + 32 + ] + } + }, + "w2": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 1, + 32 + ] + } + }, + "gate": "sigmoid" + }, + "routing": { + "distribution": "natural", + "seed": 0 + } + }, + "sources": [ + "amd/Qwen3.5-397B-A17B-MXFP4-AttnFP8-V2/mlp" + ] + }, + { + "type": "moe", + "args": { + "tokens": 1, + "hidden": 2560, + "experts": { + "num": 512, + "top_k": 10, + "inter": 640, + "a1": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "w1": { + "dtype": "e4m3", + "scale": { + "dtype": "bf16", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "w2": { + "dtype": "e4m3", + "scale": { + "dtype": "bf16", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "a2": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + } + }, + "router": { + "gate": { + "dtype": "bf16" + }, + "scoring": "softmax", + "select": { + "kind": "topk" + }, + "renormalize": true + }, + "activation": { + "kind": "silu" + }, + "shared": { + "count": 1, + "inter": 640, + "a1": { + "dtype": "bf16" + }, + "w1": { + "dtype": "bf16" + }, + "w2": { + "dtype": "bf16" + }, + "gate": "sigmoid" + }, + "routing": { + "distribution": "natural", + "seed": 0 + } + }, + "sources": [ + "Qwen/Qwen3.8-Flash-Next-FP8/mlp" + ] + }, + { + "type": "moe", + "args": { + "tokens": 1, + "hidden": 2560, + "experts": { + "num": 512, + "top_k": 10, + "inter": 640, + "a1": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "w1": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "w2": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "a2": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + } + }, + "router": { + "gate": { + "dtype": "bf16" + }, + "scoring": "softmax", + "select": { + "kind": "topk" + }, + "renormalize": true + }, + "activation": { + "kind": "silu" + }, + "shared": { + "count": 1, + "inter": 640, + "a1": { + "dtype": "bf16" + }, + "w1": { + "dtype": "bf16" + }, + "w2": { + "dtype": "bf16" + }, + "gate": "sigmoid" + }, + "routing": { + "distribution": "natural", + "seed": 0 + } + }, + "sources": [ + "RadixArk/Qwen3.8-Flash-Next-NVFP4/mlp" + ] + }, + { + "type": "moe", + "args": { + "tokens": 1, + "hidden": 6144, + "experts": { + "num": 256, + "top_k": 8, + "inter": 2048, + "a1": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "w1": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "w2": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "a2": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + } + }, + "router": { + "gate": { + "dtype": "bf16", + "logits": "fp32" + }, + "scoring": "sigmoid", + "select": { + "kind": "grouped_topk", + "groups": 1, + "topk_groups": 1 + }, + "bias": true, + "renormalize": true, + "scale": 2.5 + }, + "activation": { + "kind": "silu" + }, + "shared": { + "count": 1, + "inter": 2048, + "a1": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "w1": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "w2": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + } + }, + "routing": { + "distribution": "natural", + "seed": 0 + } + }, + "sources": [ + "zai-org/GLM-5.2-FP8/mlp", + "zai-org/GLM-5-FP8/mlp", + "zai-org/GLM-5.1-FP8/mlp", + "zai-org/GLM-5.3/mlp" + ] + }, + { + "type": "moe", + "args": { + "tokens": 1, + "hidden": 6144, + "experts": { + "num": 256, + "top_k": 8, + "inter": 2048, + "a1": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "w1": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "w2": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "a2": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + } + }, + "router": { + "gate": { + "dtype": "bf16", + "logits": "fp32" + }, + "scoring": "sigmoid", + "select": { + "kind": "grouped_topk", + "groups": 1, + "topk_groups": 1 + }, + "bias": true, + "renormalize": true, + "scale": 2.5 + }, + "activation": { + "kind": "silu" + }, + "shared": { + "count": 1, + "inter": 2048, + "a1": { + "dtype": "bf16" + }, + "w1": { + "dtype": "bf16" + }, + "w2": { + "dtype": "bf16" + } + }, + "routing": { + "distribution": "natural", + "seed": 0 + } + }, + "sources": [ + "nvidia/GLM-5.2-NVFP4/mlp", + "nvidia/GLM-5-NVFP4/mlp", + "nvidia/GLM-5.1-NVFP4/mlp" + ] + }, + { + "type": "moe", + "args": { + "tokens": 1, + "hidden": 6144, + "experts": { + "num": 256, + "top_k": 8, + "inter": 2048, + "a1": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 32 + ] + } + }, + "w1": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 1, + 32 + ] + } + }, + "w2": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 1, + 32 + ] + } + }, + "a2": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 32 + ] + } + } + }, + "router": { + "gate": { + "dtype": "bf16", + "logits": "fp32" + }, + "scoring": "sigmoid", + "select": { + "kind": "grouped_topk", + "groups": 1, + "topk_groups": 1 + }, + "bias": true, + "renormalize": true, + "scale": 2.5 + }, + "activation": { + "kind": "silu" + }, + "shared": { + "count": 1, + "inter": 2048, + "a1": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 32 + ] + } + }, + "w1": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 1, + 32 + ] + } + }, + "w2": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 1, + 32 + ] + } + } + }, + "routing": { + "distribution": "natural", + "seed": 0 + } + }, + "sources": [ + "amd/GLM-5.2-MXFP4/mlp", + "amd/GLM-5.1-MXFP4/mlp" + ] + }, + { + "type": "moe", + "args": { + "tokens": 1, + "hidden": 7168, + "experts": { + "num": 896, + "top_k": 16, + "inter": 3072, + "latent": 3584, + "latent_norm": true, + "a1": { + "dtype": "bf16" + }, + "w1": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 1, + 32 + ] + } + }, + "w2": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 1, + 32 + ] + } + }, + "a2": { + "dtype": "bf16" + } + }, + "router": { + "gate": { + "dtype": "bf16", + "logits": "fp32" + }, + "scoring": "sigmoid", + "select": { + "kind": "grouped_topk", + "groups": 1, + "topk_groups": 1 + }, + "bias": true, + "renormalize": true, + "scale": 1.0 + }, + "activation": { + "kind": "situ", + "alpha": 4.0, + "beta": 25.0 + }, + "shared": { + "count": 2, + "inter": 3072, + "a1": { + "dtype": "bf16" + }, + "w1": { + "dtype": "bf16" + }, + "w2": { + "dtype": "bf16" + } + }, + "routing": { + "distribution": "natural", + "seed": 0 + } + }, + "sources": [ + "moonshotai/Kimi-K3/block_sparse_moe" + ], + "recipes": [ + "kimik3-fp4-mi355x-vllm-agentic-mtp" + ] + }, + { + "type": "moe", + "args": { + "tokens": 1, + "hidden": 6144, + "experts": { + "num": 128, + "top_k": 4, + "inter": 3072, + "a1": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 32 + ] + } + }, + "w1": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 1, + 32 + ] + } + }, + "w2": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 1, + 32 + ] + } + }, + "a2": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 32 + ] + } + } + }, + "router": { + "gate": { + "dtype": "fp32" + }, + "scoring": "sigmoid", + "select": { + "kind": "topk" + }, + "bias": true, + "renormalize": true, + "scale": 2.0 + }, + "activation": { + "kind": "swigluoai", + "alpha": 1.702, + "beta": 1.0, + "limit": 7.0 + }, + "shared": { + "count": 1, + "inter": 3072, + "a1": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 32 + ] + } + }, + "w1": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 1, + 32 + ] + } + }, + "w2": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 1, + 32 + ] + } + } + }, + "routing": { + "distribution": "natural", + "seed": 0 + } + }, + "sources": [ + "MiniMaxAI/MiniMax-M3-MXFP8/block_sparse_moe" + ] + }, + { + "type": "moe", + "args": { + "tokens": 1, + "hidden": 6144, + "experts": { + "num": 128, + "top_k": 4, + "inter": 3072, + "a1": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "w1": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "w2": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "a2": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + } + }, + "router": { + "gate": { + "dtype": "fp32" + }, + "scoring": "sigmoid", + "select": { + "kind": "topk" + }, + "bias": true, + "renormalize": true, + "scale": 2.0 + }, + "activation": { + "kind": "swigluoai", + "alpha": 1.702, + "beta": 1.0, + "limit": 7.0 + }, + "shared": { + "count": 1, + "inter": 3072, + "a1": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 32 + ] + } + }, + "w1": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 1, + 32 + ] + } + }, + "w2": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 1, + 32 + ] + } + } + }, + "routing": { + "distribution": "natural", + "seed": 0 + } + }, + "sources": [ + "nvidia/MiniMax-M3-NVFP4/block_sparse_moe" + ] + }, + { + "type": "moe", + "args": { + "tokens": 1, + "hidden": 6144, + "experts": { + "num": 128, + "top_k": 4, + "inter": 3072, + "a1": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 32 + ] + } + }, + "w1": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 1, + 32 + ] + } + }, + "w2": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 1, + 32 + ] + } + }, + "a2": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 32 + ] + } + } + }, + "router": { + "gate": { + "dtype": "fp32" + }, + "scoring": "sigmoid", + "select": { + "kind": "topk" + }, + "bias": true, + "renormalize": true, + "scale": 2.0 + }, + "activation": { + "kind": "swigluoai", + "alpha": 1.702, + "beta": 1.0, + "limit": 7.0 + }, + "shared": { + "count": 1, + "inter": 3072, + "a1": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 32 + ] + } + }, + "w1": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 1, + 32 + ] + } + }, + "w2": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 1, + 32 + ] + } + } + }, + "routing": { + "distribution": "natural", + "seed": 0 + } + }, + "sources": [ + "amd/MiniMax-M3-MXFP4/block_sparse_moe" + ] + }, + { + "type": "moe", + "args": { + "tokens": 1, + "hidden": 4096, + "experts": { + "num": 512, + "top_k": 10, + "inter": 1024, + "a1": { + "dtype": "bf16" + }, + "w1": { + "dtype": "bf16" + }, + "w2": { + "dtype": "bf16" + }, + "a2": { + "dtype": "bf16" + } + }, + "router": { + "gate": { + "dtype": "bf16" + }, + "scoring": "softmax", + "select": { + "kind": "topk" + }, + "renormalize": true + }, + "activation": { + "kind": "silu" + }, + "shared": { + "count": 1, + "inter": 1024, + "a1": { + "dtype": "bf16" + }, + "w1": { + "dtype": "bf16" + }, + "w2": { + "dtype": "bf16" + }, + "gate": "sigmoid" + }, + "routing": { + "distribution": "natural", + "seed": 0 + } + }, + "sources": [ + "Qwen/Qwen3.5-397B-A17B/mlp" + ] + }, + { + "type": "moe", + "args": { + "tokens": 1, + "hidden": 7168, + "experts": { + "num": 384, + "top_k": 8, + "inter": 2048, + "a1": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "w1": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "w2": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "a2": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + } + }, + "router": { + "gate": { + "dtype": "bf16" + }, + "scoring": "sigmoid", + "select": { + "kind": "grouped_topk", + "groups": 1, + "topk_groups": 1 + }, + "bias": true, + "renormalize": true, + "scale": 2.827 + }, + "activation": { + "kind": "silu" + }, + "shared": { + "count": 1, + "inter": 2048, + "a1": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "w1": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "w2": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + } + }, + "routing": { + "distribution": "natural", + "seed": 0 + } + }, + "sources": [ + "nvidia/Kimi-K2.5-NVFP4/mlp", + "nvidia/Kimi-K2.6-NVFP4/mlp" + ] + }, + { + "type": "moe", + "args": { + "tokens": 1, + "hidden": 7168, + "experts": { + "num": 384, + "top_k": 8, + "inter": 2048, + "a1": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 32 + ] + } + }, + "w1": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 1, + 32 + ] + } + }, + "w2": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 1, + 32 + ] + } + }, + "a2": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 32 + ] + } + } + }, + "router": { + "gate": { + "dtype": "bf16" + }, + "scoring": "sigmoid", + "select": { + "kind": "grouped_topk", + "groups": 1, + "topk_groups": 1 + }, + "bias": true, + "renormalize": true, + "scale": 2.827 + }, + "activation": { + "kind": "silu" + }, + "shared": { + "count": 1, + "inter": 2048, + "a1": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 32 + ] + } + }, + "w1": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 1, + 32 + ] + } + }, + "w2": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 1, + 32 + ] + } + } + }, + "routing": { + "distribution": "natural", + "seed": 0 + } + }, + "sources": [ + "amd/Kimi-K2.5-MXFP4/mlp" + ] + }, + { + "type": "moe", + "args": { + "tokens": 1, + "hidden": 3072, + "experts": { + "num": 256, + "top_k": 8, + "inter": 1536, + "a1": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "w1": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "w2": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "a2": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + } + }, + "router": { + "gate": { + "dtype": "bf16" + }, + "scoring": "sigmoid", + "select": { + "kind": "topk" + }, + "bias": true, + "renormalize": true + }, + "activation": { + "kind": "silu" + }, + "routing": { + "distribution": "natural", + "seed": 0 + } + }, + "sources": [ + "MiniMaxAI/MiniMax-M2.5/block_sparse_moe" + ] + }, + { + "type": "moe", + "args": { + "tokens": 1, + "hidden": 3072, + "experts": { + "num": 256, + "top_k": 8, + "inter": 1536, + "a1": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "w1": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "w2": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": true, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + }, + "a2": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "static": false, + "group": [ + 1, + 16 + ] + }, + "scale2": { + "dtype": "fp32", + "static": true, + "group": [ + -1, + -1 + ] + } + } + }, + "router": { + "gate": { + "dtype": "bf16" + }, + "scoring": "sigmoid", + "select": { + "kind": "topk" + }, + "bias": true, + "renormalize": true + }, + "activation": { + "kind": "silu" + }, + "routing": { + "distribution": "natural", + "seed": 0 + } + }, + "sources": [ + "nvidia/MiniMax-M2.5-NVFP4/block_sparse_moe" + ] + }, + { + "type": "moe", + "args": { + "tokens": 1, + "hidden": 3072, + "experts": { + "num": 256, + "top_k": 8, + "inter": 1536, + "a1": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 32 + ] + } + }, + "w1": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 1, + 32 + ] + } + }, + "w2": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 1, + 32 + ] + } + }, + "a2": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 32 + ] + } + } + }, + "router": { + "gate": { + "dtype": "bf16" + }, + "scoring": "sigmoid", + "select": { + "kind": "topk" + }, + "bias": true, + "renormalize": true + }, + "activation": { + "kind": "silu" + }, + "routing": { + "distribution": "natural", + "seed": 0 + } + }, + "sources": [ + "amd/MiniMax-M2.5-MXFP4/block_sparse_moe" + ] + }, + { + "type": "moe", + "args": { + "tokens": 256, + "hidden": 7168, + "experts": { + "num": 256, + "top_k": 8, + "inter": 2048, + "a1": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "w1": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "w2": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "a2": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + } + }, + "router": { + "gate": { + "dtype": "bf16" + }, + "scoring": "sigmoid", + "select": { + "kind": "grouped_topk", + "groups": 8, + "topk_groups": 4 + }, + "bias": true, + "renormalize": true, + "scale": 2.5 + }, + "activation": { + "kind": "silu" + }, + "shared": { + "count": 1, + "inter": 2048, + "a1": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "w1": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "w2": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + } + }, + "routing": { + "distribution": "natural", + "seed": 0 + } + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/mlp" + ] + }, + { + "type": "moe", + "args": { + "tokens": 256, + "hidden": 7168, + "experts": { + "num": 256, + "top_k": 8, + "inter": 2048, + "a1": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "w1": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "w2": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "a2": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + } + }, + "router": { + "gate": { + "dtype": "bf16" + }, + "scoring": "sigmoid", + "select": { + "kind": "grouped_topk", + "groups": 8, + "topk_groups": 4 + }, + "bias": true, + "renormalize": true, + "scale": 2.5 + }, + "activation": { + "kind": "silu" + }, + "shared": { + "count": 1, + "inter": 2048, + "a1": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "w1": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "w2": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + } + }, + "routing": { + "distribution": "balanced", + "seed": 0 + } + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/mlp" + ] + }, + { + "type": "moe", + "args": { + "tokens": 256, + "hidden": 7168, + "experts": { + "num": 256, + "top_k": 8, + "inter": 2048, + "a1": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "w1": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "w2": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "a2": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + } + }, + "router": { + "gate": { + "dtype": "bf16" + }, + "scoring": "sigmoid", + "select": { + "kind": "grouped_topk", + "groups": 8, + "topk_groups": 4 + }, + "bias": true, + "renormalize": true, + "scale": 2.5 + }, + "activation": { + "kind": "silu" + }, + "shared": { + "count": 1, + "inter": 2048, + "a1": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "w1": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "w2": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + } + }, + "routing": { + "distribution": { + "kind": "zipf", + "s": 1.2 + }, + "seed": 0 + } + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/mlp" + ] + }, + { + "type": "moe", + "args": { + "tokens": 256, + "hidden": 7168, + "experts": { + "num": 256, + "top_k": 8, + "inter": 2048, + "a1": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "w1": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "w2": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "a2": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + } + }, + "router": { + "gate": { + "dtype": "bf16" + }, + "scoring": "sigmoid", + "select": { + "kind": "grouped_topk", + "groups": 8, + "topk_groups": 4 + }, + "bias": true, + "renormalize": true, + "scale": 2.5 + }, + "activation": { + "kind": "silu" + }, + "shared": { + "count": 1, + "inter": 2048, + "a1": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "w1": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "w2": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + } + }, + "routing": { + "distribution": "single_hot", + "seed": 0 + } + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/mlp" + ] + }, + { + "type": "moe", + "args": { + "tokens": 256, + "hidden": 4096, + "experts": { + "num": 512, + "top_k": 10, + "inter": 1024, + "a1": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "w1": { + "dtype": "e4m3", + "scale": { + "dtype": "bf16", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "w2": { + "dtype": "e4m3", + "scale": { + "dtype": "bf16", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "a2": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + } + }, + "router": { + "gate": { + "dtype": "bf16" + }, + "scoring": "softmax", + "select": { + "kind": "topk" + }, + "renormalize": true + }, + "activation": { + "kind": "silu" + }, + "shared": { + "count": 1, + "inter": 1024, + "a1": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "w1": { + "dtype": "e4m3", + "scale": { + "dtype": "bf16", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "w2": { + "dtype": "e4m3", + "scale": { + "dtype": "bf16", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "gate": "sigmoid" + }, + "routing": { + "distribution": "natural", + "seed": 0 + } + }, + "sources": [ + "Qwen/Qwen3.5-397B-A17B-FP8/mlp" + ] + }, + { + "type": "moe", + "args": { + "tokens": 256, + "hidden": 4096, + "experts": { + "num": 512, + "top_k": 10, + "inter": 1024, + "a1": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "w1": { + "dtype": "e4m3", + "scale": { + "dtype": "bf16", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "w2": { + "dtype": "e4m3", + "scale": { + "dtype": "bf16", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "a2": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + } + }, + "router": { + "gate": { + "dtype": "bf16" + }, + "scoring": "softmax", + "select": { + "kind": "topk" + }, + "renormalize": true + }, + "activation": { + "kind": "silu" + }, + "shared": { + "count": 1, + "inter": 1024, + "a1": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "w1": { + "dtype": "e4m3", + "scale": { + "dtype": "bf16", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "w2": { + "dtype": "e4m3", + "scale": { + "dtype": "bf16", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "gate": "sigmoid" + }, + "routing": { + "distribution": "balanced", + "seed": 0 + } + }, + "sources": [ + "Qwen/Qwen3.5-397B-A17B-FP8/mlp" + ] + }, + { + "type": "moe", + "args": { + "tokens": 256, + "hidden": 4096, + "experts": { + "num": 512, + "top_k": 10, + "inter": 1024, + "a1": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "w1": { + "dtype": "e4m3", + "scale": { + "dtype": "bf16", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "w2": { + "dtype": "e4m3", + "scale": { + "dtype": "bf16", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "a2": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + } + }, + "router": { + "gate": { + "dtype": "bf16" + }, + "scoring": "softmax", + "select": { + "kind": "topk" + }, + "renormalize": true + }, + "activation": { + "kind": "silu" + }, + "shared": { + "count": 1, + "inter": 1024, + "a1": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "w1": { + "dtype": "e4m3", + "scale": { + "dtype": "bf16", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "w2": { + "dtype": "e4m3", + "scale": { + "dtype": "bf16", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "gate": "sigmoid" + }, + "routing": { + "distribution": { + "kind": "zipf", + "s": 1.2 + }, + "seed": 0 + } + }, + "sources": [ + "Qwen/Qwen3.5-397B-A17B-FP8/mlp" + ] + }, + { + "type": "moe", + "args": { + "tokens": 256, + "hidden": 4096, + "experts": { + "num": 512, + "top_k": 10, + "inter": 1024, + "a1": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "w1": { + "dtype": "e4m3", + "scale": { + "dtype": "bf16", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "w2": { + "dtype": "e4m3", + "scale": { + "dtype": "bf16", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "a2": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + } + }, + "router": { + "gate": { + "dtype": "bf16" + }, + "scoring": "softmax", + "select": { + "kind": "topk" + }, + "renormalize": true + }, + "activation": { + "kind": "silu" + }, + "shared": { + "count": 1, + "inter": 1024, + "a1": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "w1": { + "dtype": "e4m3", + "scale": { + "dtype": "bf16", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "w2": { + "dtype": "e4m3", + "scale": { + "dtype": "bf16", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "gate": "sigmoid" + }, + "routing": { + "distribution": "single_hot", + "seed": 0 + } + }, + "sources": [ + "Qwen/Qwen3.5-397B-A17B-FP8/mlp" + ] + }, + { + "type": "moe", + "args": { + "tokens": 8, + "hidden": 7168, + "experts": { + "num": 256, + "top_k": 8, + "inter": 2048, + "a1": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "w1": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "w2": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "a2": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + } + }, + "router": { + "gate": { + "dtype": "bf16" + }, + "scoring": "sigmoid", + "select": { + "kind": "grouped_topk", + "groups": 8, + "topk_groups": 4 + }, + "bias": true, + "renormalize": true, + "scale": 2.5 + }, + "activation": { + "kind": "silu" + }, + "shared": { + "count": 1, + "inter": 2048, + "a1": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "w1": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "w2": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + } + }, + "routing": { + "distribution": "natural", + "seed": 0 + } + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/mlp" + ] + }, + { + "type": "moe", + "args": { + "tokens": 64, + "hidden": 7168, + "experts": { + "num": 256, + "top_k": 8, + "inter": 2048, + "a1": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "w1": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "w2": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "a2": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + } + }, + "router": { + "gate": { + "dtype": "bf16" + }, + "scoring": "sigmoid", + "select": { + "kind": "grouped_topk", + "groups": 8, + "topk_groups": 4 + }, + "bias": true, + "renormalize": true, + "scale": 2.5 + }, + "activation": { + "kind": "silu" + }, + "shared": { + "count": 1, + "inter": 2048, + "a1": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "w1": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "w2": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + } + }, + "routing": { + "distribution": "natural", + "seed": 0 + } + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/mlp" + ] + }, + { + "type": "moe", + "args": { + "tokens": 512, + "hidden": 7168, + "experts": { + "num": 256, + "top_k": 8, + "inter": 2048, + "a1": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "w1": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "w2": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "a2": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + } + }, + "router": { + "gate": { + "dtype": "bf16" + }, + "scoring": "sigmoid", + "select": { + "kind": "grouped_topk", + "groups": 8, + "topk_groups": 4 + }, + "bias": true, + "renormalize": true, + "scale": 2.5 + }, + "activation": { + "kind": "silu" + }, + "shared": { + "count": 1, + "inter": 2048, + "a1": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "w1": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "w2": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + } + }, + "routing": { + "distribution": "natural", + "seed": 0 + } + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/mlp" + ] + }, + { + "type": "moe", + "args": { + "tokens": 2048, + "hidden": 7168, + "experts": { + "num": 256, + "top_k": 8, + "inter": 2048, + "a1": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "w1": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "w2": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "a2": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + } + }, + "router": { + "gate": { + "dtype": "bf16" + }, + "scoring": "sigmoid", + "select": { + "kind": "grouped_topk", + "groups": 8, + "topk_groups": 4 + }, + "bias": true, + "renormalize": true, + "scale": 2.5 + }, + "activation": { + "kind": "silu" + }, + "shared": { + "count": 1, + "inter": 2048, + "a1": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "w1": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "w2": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + } + }, + "routing": { + "distribution": "natural", + "seed": 0 + } + }, + "sources": [ + "deepseek-ai/DeepSeek-R1-0528/mlp" + ] + }, + { + "type": "moe", + "args": { + "tokens": 8, + "hidden": 7168, + "experts": { + "num": 384, + "top_k": 6, + "inter": 3072, + "a1": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 32 + ] + } + }, + "w1": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 1, + 32 + ] + } + }, + "w2": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 1, + 32 + ] + } + }, + "a2": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 32 + ] + } + } + }, + "router": { + "gate": { + "dtype": "bf16", + "logits": "fp32" + }, + "scoring": "sqrtsoftplus", + "select": { + "kind": "topk" + }, + "bias": true, + "renormalize": true, + "scale": 2.5 + }, + "activation": { + "kind": "silu", + "limit": 10.0 + }, + "shared": { + "count": 1, + "inter": 3072, + "a1": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "w1": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "w2": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + } + }, + "routing": { + "distribution": "natural", + "seed": 0 + } + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro/ffn" + ] + }, + { + "type": "moe", + "args": { + "tokens": 64, + "hidden": 7168, + "experts": { + "num": 384, + "top_k": 6, + "inter": 3072, + "a1": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 32 + ] + } + }, + "w1": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 1, + 32 + ] + } + }, + "w2": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 1, + 32 + ] + } + }, + "a2": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 32 + ] + } + } + }, + "router": { + "gate": { + "dtype": "bf16", + "logits": "fp32" + }, + "scoring": "sqrtsoftplus", + "select": { + "kind": "topk" + }, + "bias": true, + "renormalize": true, + "scale": 2.5 + }, + "activation": { + "kind": "silu", + "limit": 10.0 + }, + "shared": { + "count": 1, + "inter": 3072, + "a1": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "w1": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "w2": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + } + }, + "routing": { + "distribution": "natural", + "seed": 0 + } + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro/ffn" + ] + }, + { + "type": "moe", + "args": { + "tokens": 512, + "hidden": 7168, + "experts": { + "num": 384, + "top_k": 6, + "inter": 3072, + "a1": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 32 + ] + } + }, + "w1": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 1, + 32 + ] + } + }, + "w2": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 1, + 32 + ] + } + }, + "a2": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 32 + ] + } + } + }, + "router": { + "gate": { + "dtype": "bf16", + "logits": "fp32" + }, + "scoring": "sqrtsoftplus", + "select": { + "kind": "topk" + }, + "bias": true, + "renormalize": true, + "scale": 2.5 + }, + "activation": { + "kind": "silu", + "limit": 10.0 + }, + "shared": { + "count": 1, + "inter": 3072, + "a1": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "w1": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "w2": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + } + }, + "routing": { + "distribution": "natural", + "seed": 0 + } + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro/ffn" + ] + }, + { + "type": "moe", + "args": { + "tokens": 2048, + "hidden": 7168, + "experts": { + "num": 384, + "top_k": 6, + "inter": 3072, + "a1": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 32 + ] + } + }, + "w1": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 1, + 32 + ] + } + }, + "w2": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 1, + 32 + ] + } + }, + "a2": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 32 + ] + } + } + }, + "router": { + "gate": { + "dtype": "bf16", + "logits": "fp32" + }, + "scoring": "sqrtsoftplus", + "select": { + "kind": "topk" + }, + "bias": true, + "renormalize": true, + "scale": 2.5 + }, + "activation": { + "kind": "silu", + "limit": 10.0 + }, + "shared": { + "count": 1, + "inter": 3072, + "a1": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "w1": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "w2": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 128, + 128 + ] + } + } + }, + "routing": { + "distribution": "natural", + "seed": 0 + } + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro/ffn" + ] + }, + { + "type": "moe", + "args": { + "tokens": 8, + "hidden": 4096, + "experts": { + "num": 512, + "top_k": 10, + "inter": 1024, + "a1": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "w1": { + "dtype": "e4m3", + "scale": { + "dtype": "bf16", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "w2": { + "dtype": "e4m3", + "scale": { + "dtype": "bf16", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "a2": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + } + }, + "router": { + "gate": { + "dtype": "bf16" + }, + "scoring": "softmax", + "select": { + "kind": "topk" + }, + "renormalize": true + }, + "activation": { + "kind": "silu" + }, + "shared": { + "count": 1, + "inter": 1024, + "a1": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "w1": { + "dtype": "e4m3", + "scale": { + "dtype": "bf16", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "w2": { + "dtype": "e4m3", + "scale": { + "dtype": "bf16", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "gate": "sigmoid" + }, + "routing": { + "distribution": "natural", + "seed": 0 + } + }, + "sources": [ + "Qwen/Qwen3.5-397B-A17B-FP8/mlp" + ] + }, + { + "type": "moe", + "args": { + "tokens": 64, + "hidden": 4096, + "experts": { + "num": 512, + "top_k": 10, + "inter": 1024, + "a1": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "w1": { + "dtype": "e4m3", + "scale": { + "dtype": "bf16", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "w2": { + "dtype": "e4m3", + "scale": { + "dtype": "bf16", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "a2": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + } + }, + "router": { + "gate": { + "dtype": "bf16" + }, + "scoring": "softmax", + "select": { + "kind": "topk" + }, + "renormalize": true + }, + "activation": { + "kind": "silu" + }, + "shared": { + "count": 1, + "inter": 1024, + "a1": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "w1": { + "dtype": "e4m3", + "scale": { + "dtype": "bf16", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "w2": { + "dtype": "e4m3", + "scale": { + "dtype": "bf16", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "gate": "sigmoid" + }, + "routing": { + "distribution": "natural", + "seed": 0 + } + }, + "sources": [ + "Qwen/Qwen3.5-397B-A17B-FP8/mlp" + ] + }, + { + "type": "moe", + "args": { + "tokens": 512, + "hidden": 4096, + "experts": { + "num": 512, + "top_k": 10, + "inter": 1024, + "a1": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "w1": { + "dtype": "e4m3", + "scale": { + "dtype": "bf16", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "w2": { + "dtype": "e4m3", + "scale": { + "dtype": "bf16", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "a2": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + } + }, + "router": { + "gate": { + "dtype": "bf16" + }, + "scoring": "softmax", + "select": { + "kind": "topk" + }, + "renormalize": true + }, + "activation": { + "kind": "silu" + }, + "shared": { + "count": 1, + "inter": 1024, + "a1": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "w1": { + "dtype": "e4m3", + "scale": { + "dtype": "bf16", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "w2": { + "dtype": "e4m3", + "scale": { + "dtype": "bf16", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "gate": "sigmoid" + }, + "routing": { + "distribution": "natural", + "seed": 0 + } + }, + "sources": [ + "Qwen/Qwen3.5-397B-A17B-FP8/mlp" + ] + }, + { + "type": "moe", + "args": { + "tokens": 2048, + "hidden": 4096, + "experts": { + "num": 512, + "top_k": 10, + "inter": 1024, + "a1": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "w1": { + "dtype": "e4m3", + "scale": { + "dtype": "bf16", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "w2": { + "dtype": "e4m3", + "scale": { + "dtype": "bf16", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "a2": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + } + }, + "router": { + "gate": { + "dtype": "bf16" + }, + "scoring": "softmax", + "select": { + "kind": "topk" + }, + "renormalize": true + }, + "activation": { + "kind": "silu" + }, + "shared": { + "count": 1, + "inter": 1024, + "a1": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "w1": { + "dtype": "e4m3", + "scale": { + "dtype": "bf16", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "w2": { + "dtype": "e4m3", + "scale": { + "dtype": "bf16", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "gate": "sigmoid" + }, + "routing": { + "distribution": "natural", + "seed": 0 + } + }, + "sources": [ + "Qwen/Qwen3.5-397B-A17B-FP8/mlp" + ] + }, + { + "type": "moe", + "args": { + "tokens": 8, + "hidden": 6144, + "experts": { + "num": 256, + "top_k": 8, + "inter": 2048, + "a1": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "w1": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "w2": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "a2": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + } + }, + "router": { + "gate": { + "dtype": "bf16", + "logits": "fp32" + }, + "scoring": "sigmoid", + "select": { + "kind": "grouped_topk", + "groups": 1, + "topk_groups": 1 + }, + "bias": true, + "renormalize": true, + "scale": 2.5 + }, + "activation": { + "kind": "silu" + }, + "shared": { + "count": 1, + "inter": 2048, + "a1": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "w1": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "w2": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + } + }, + "routing": { + "distribution": "natural", + "seed": 0 + } + }, + "sources": [ + "zai-org/GLM-5.2-FP8/mlp" + ] + }, + { + "type": "moe", + "args": { + "tokens": 64, + "hidden": 6144, + "experts": { + "num": 256, + "top_k": 8, + "inter": 2048, + "a1": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "w1": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "w2": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "a2": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + } + }, + "router": { + "gate": { + "dtype": "bf16", + "logits": "fp32" + }, + "scoring": "sigmoid", + "select": { + "kind": "grouped_topk", + "groups": 1, + "topk_groups": 1 + }, + "bias": true, + "renormalize": true, + "scale": 2.5 + }, + "activation": { + "kind": "silu" + }, + "shared": { + "count": 1, + "inter": 2048, + "a1": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "w1": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "w2": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + } + }, + "routing": { + "distribution": "natural", + "seed": 0 + } + }, + "sources": [ + "zai-org/GLM-5.2-FP8/mlp" + ] + }, + { + "type": "moe", + "args": { + "tokens": 512, + "hidden": 6144, + "experts": { + "num": 256, + "top_k": 8, + "inter": 2048, + "a1": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "w1": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "w2": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "a2": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + } + }, + "router": { + "gate": { + "dtype": "bf16", + "logits": "fp32" + }, + "scoring": "sigmoid", + "select": { + "kind": "grouped_topk", + "groups": 1, + "topk_groups": 1 + }, + "bias": true, + "renormalize": true, + "scale": 2.5 + }, + "activation": { + "kind": "silu" + }, + "shared": { + "count": 1, + "inter": 2048, + "a1": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "w1": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "w2": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + } + }, + "routing": { + "distribution": "natural", + "seed": 0 + } + }, + "sources": [ + "zai-org/GLM-5.2-FP8/mlp" + ] + }, + { + "type": "moe", + "args": { + "tokens": 2048, + "hidden": 6144, + "experts": { + "num": 256, + "top_k": 8, + "inter": 2048, + "a1": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "w1": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "w2": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "a2": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + } + }, + "router": { + "gate": { + "dtype": "bf16", + "logits": "fp32" + }, + "scoring": "sigmoid", + "select": { + "kind": "grouped_topk", + "groups": 1, + "topk_groups": 1 + }, + "bias": true, + "renormalize": true, + "scale": 2.5 + }, + "activation": { + "kind": "silu" + }, + "shared": { + "count": 1, + "inter": 2048, + "a1": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "w1": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "w2": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + } + }, + "routing": { + "distribution": "natural", + "seed": 0 + } + }, + "sources": [ + "zai-org/GLM-5.2-FP8/mlp" + ] + }, + { + "type": "moe", + "args": { + "tokens": 8, + "hidden": 7168, + "experts": { + "num": 896, + "top_k": 16, + "inter": 3072, + "latent": 3584, + "latent_norm": true, + "a1": { + "dtype": "bf16" + }, + "w1": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 1, + 32 + ] + } + }, + "w2": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 1, + 32 + ] + } + }, + "a2": { + "dtype": "bf16" + } + }, + "router": { + "gate": { + "dtype": "bf16", + "logits": "fp32" + }, + "scoring": "sigmoid", + "select": { + "kind": "grouped_topk", + "groups": 1, + "topk_groups": 1 + }, + "bias": true, + "renormalize": true, + "scale": 1.0 + }, + "activation": { + "kind": "situ", + "alpha": 4.0, + "beta": 25.0 + }, + "shared": { + "count": 2, + "inter": 3072, + "a1": { + "dtype": "bf16" + }, + "w1": { + "dtype": "bf16" + }, + "w2": { + "dtype": "bf16" + } + }, + "routing": { + "distribution": "natural", + "seed": 0 + } + }, + "sources": [ + "moonshotai/Kimi-K3/block_sparse_moe" + ], + "recipes": [ + "kimik3-fp4-mi355x-vllm-agentic-mtp" + ] + }, + { + "type": "moe", + "args": { + "tokens": 64, + "hidden": 7168, + "experts": { + "num": 896, + "top_k": 16, + "inter": 3072, + "latent": 3584, + "latent_norm": true, + "a1": { + "dtype": "bf16" + }, + "w1": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 1, + 32 + ] + } + }, + "w2": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 1, + 32 + ] + } + }, + "a2": { + "dtype": "bf16" + } + }, + "router": { + "gate": { + "dtype": "bf16", + "logits": "fp32" + }, + "scoring": "sigmoid", + "select": { + "kind": "grouped_topk", + "groups": 1, + "topk_groups": 1 + }, + "bias": true, + "renormalize": true, + "scale": 1.0 + }, + "activation": { + "kind": "situ", + "alpha": 4.0, + "beta": 25.0 + }, + "shared": { + "count": 2, + "inter": 3072, + "a1": { + "dtype": "bf16" + }, + "w1": { + "dtype": "bf16" + }, + "w2": { + "dtype": "bf16" + } + }, + "routing": { + "distribution": "natural", + "seed": 0 + } + }, + "sources": [ + "moonshotai/Kimi-K3/block_sparse_moe" + ], + "recipes": [ + "kimik3-fp4-mi355x-vllm-agentic-mtp" + ] + }, + { + "type": "moe", + "args": { + "tokens": 512, + "hidden": 7168, + "experts": { + "num": 896, + "top_k": 16, + "inter": 3072, + "latent": 3584, + "latent_norm": true, + "a1": { + "dtype": "bf16" + }, + "w1": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 1, + 32 + ] + } + }, + "w2": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 1, + 32 + ] + } + }, + "a2": { + "dtype": "bf16" + } + }, + "router": { + "gate": { + "dtype": "bf16", + "logits": "fp32" + }, + "scoring": "sigmoid", + "select": { + "kind": "grouped_topk", + "groups": 1, + "topk_groups": 1 + }, + "bias": true, + "renormalize": true, + "scale": 1.0 + }, + "activation": { + "kind": "situ", + "alpha": 4.0, + "beta": 25.0 + }, + "shared": { + "count": 2, + "inter": 3072, + "a1": { + "dtype": "bf16" + }, + "w1": { + "dtype": "bf16" + }, + "w2": { + "dtype": "bf16" + } + }, + "routing": { + "distribution": "natural", + "seed": 0 + } + }, + "sources": [ + "moonshotai/Kimi-K3/block_sparse_moe" + ], + "recipes": [ + "kimik3-fp4-mi355x-vllm-agentic-mtp" + ] + }, + { + "type": "moe", + "args": { + "tokens": 2048, + "hidden": 7168, + "experts": { + "num": 896, + "top_k": 16, + "inter": 3072, + "latent": 3584, + "latent_norm": true, + "a1": { + "dtype": "bf16" + }, + "w1": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 1, + 32 + ] + } + }, + "w2": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 1, + 32 + ] + } + }, + "a2": { + "dtype": "bf16" + } + }, + "router": { + "gate": { + "dtype": "bf16", + "logits": "fp32" + }, + "scoring": "sigmoid", + "select": { + "kind": "grouped_topk", + "groups": 1, + "topk_groups": 1 + }, + "bias": true, + "renormalize": true, + "scale": 1.0 + }, + "activation": { + "kind": "situ", + "alpha": 4.0, + "beta": 25.0 + }, + "shared": { + "count": 2, + "inter": 3072, + "a1": { + "dtype": "bf16" + }, + "w1": { + "dtype": "bf16" + }, + "w2": { + "dtype": "bf16" + } + }, + "routing": { + "distribution": "natural", + "seed": 0 + } + }, + "sources": [ + "moonshotai/Kimi-K3/block_sparse_moe" + ], + "recipes": [ + "kimik3-fp4-mi355x-vllm-agentic-mtp" + ] + }, + { + "type": "moe", + "args": { + "tokens": 8, + "hidden": 6144, + "experts": { + "num": 128, + "top_k": 4, + "inter": 3072, + "a1": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 32 + ] + } + }, + "w1": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 1, + 32 + ] + } + }, + "w2": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 1, + 32 + ] + } + }, + "a2": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 32 + ] + } + } + }, + "router": { + "gate": { + "dtype": "fp32" + }, + "scoring": "sigmoid", + "select": { + "kind": "topk" + }, + "bias": true, + "renormalize": true, + "scale": 2.0 + }, + "activation": { + "kind": "swigluoai", + "alpha": 1.702, + "beta": 1.0, + "limit": 7.0 + }, + "shared": { + "count": 1, + "inter": 3072, + "a1": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 32 + ] + } + }, + "w1": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 1, + 32 + ] + } + }, + "w2": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 1, + 32 + ] + } + } + }, + "routing": { + "distribution": "natural", + "seed": 0 + } + }, + "sources": [ + "MiniMaxAI/MiniMax-M3-MXFP8/block_sparse_moe" + ] + }, + { + "type": "moe", + "args": { + "tokens": 64, + "hidden": 6144, + "experts": { + "num": 128, + "top_k": 4, + "inter": 3072, + "a1": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 32 + ] + } + }, + "w1": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 1, + 32 + ] + } + }, + "w2": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 1, + 32 + ] + } + }, + "a2": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 32 + ] + } + } + }, + "router": { + "gate": { + "dtype": "fp32" + }, + "scoring": "sigmoid", + "select": { + "kind": "topk" + }, + "bias": true, + "renormalize": true, + "scale": 2.0 + }, + "activation": { + "kind": "swigluoai", + "alpha": 1.702, + "beta": 1.0, + "limit": 7.0 + }, + "shared": { + "count": 1, + "inter": 3072, + "a1": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 32 + ] + } + }, + "w1": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 1, + 32 + ] + } + }, + "w2": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 1, + 32 + ] + } + } + }, + "routing": { + "distribution": "natural", + "seed": 0 + } + }, + "sources": [ + "MiniMaxAI/MiniMax-M3-MXFP8/block_sparse_moe" + ] + }, + { + "type": "moe", + "args": { + "tokens": 512, + "hidden": 6144, + "experts": { + "num": 128, + "top_k": 4, + "inter": 3072, + "a1": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 32 + ] + } + }, + "w1": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 1, + 32 + ] + } + }, + "w2": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 1, + 32 + ] + } + }, + "a2": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 32 + ] + } + } + }, + "router": { + "gate": { + "dtype": "fp32" + }, + "scoring": "sigmoid", + "select": { + "kind": "topk" + }, + "bias": true, + "renormalize": true, + "scale": 2.0 + }, + "activation": { + "kind": "swigluoai", + "alpha": 1.702, + "beta": 1.0, + "limit": 7.0 + }, + "shared": { + "count": 1, + "inter": 3072, + "a1": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 32 + ] + } + }, + "w1": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 1, + 32 + ] + } + }, + "w2": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 1, + 32 + ] + } + } + }, + "routing": { + "distribution": "natural", + "seed": 0 + } + }, + "sources": [ + "MiniMaxAI/MiniMax-M3-MXFP8/block_sparse_moe" + ] + }, + { + "type": "moe", + "args": { + "tokens": 2048, + "hidden": 6144, + "experts": { + "num": 128, + "top_k": 4, + "inter": 3072, + "a1": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 32 + ] + } + }, + "w1": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 1, + 32 + ] + } + }, + "w2": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 1, + 32 + ] + } + }, + "a2": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 32 + ] + } + } + }, + "router": { + "gate": { + "dtype": "fp32" + }, + "scoring": "sigmoid", + "select": { + "kind": "topk" + }, + "bias": true, + "renormalize": true, + "scale": 2.0 + }, + "activation": { + "kind": "swigluoai", + "alpha": 1.702, + "beta": 1.0, + "limit": 7.0 + }, + "shared": { + "count": 1, + "inter": 3072, + "a1": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": false, + "group": [ + 1, + 32 + ] + } + }, + "w1": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 1, + 32 + ] + } + }, + "w2": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "static": true, + "group": [ + 1, + 32 + ] + } + } + }, + "routing": { + "distribution": "natural", + "seed": 0 + } + }, + "sources": [ + "MiniMaxAI/MiniMax-M3-MXFP8/block_sparse_moe" + ] + }, + { + "type": "moe", + "args": { + "tokens": 8, + "hidden": 3072, + "experts": { + "num": 256, + "top_k": 8, + "inter": 1536, + "a1": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "w1": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "w2": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "a2": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + } + }, + "router": { + "gate": { + "dtype": "bf16" + }, + "scoring": "sigmoid", + "select": { + "kind": "topk" + }, + "bias": true, + "renormalize": true + }, + "activation": { + "kind": "silu" + }, + "routing": { + "distribution": "natural", + "seed": 0 + } + }, + "sources": [ + "MiniMaxAI/MiniMax-M2.5/block_sparse_moe" + ] + }, + { + "type": "moe", + "args": { + "tokens": 64, + "hidden": 3072, + "experts": { + "num": 256, + "top_k": 8, + "inter": 1536, + "a1": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "w1": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "w2": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "a2": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + } + }, + "router": { + "gate": { + "dtype": "bf16" + }, + "scoring": "sigmoid", + "select": { + "kind": "topk" + }, + "bias": true, + "renormalize": true + }, + "activation": { + "kind": "silu" + }, + "routing": { + "distribution": "natural", + "seed": 0 + } + }, + "sources": [ + "MiniMaxAI/MiniMax-M2.5/block_sparse_moe" + ] + }, + { + "type": "moe", + "args": { + "tokens": 512, + "hidden": 3072, + "experts": { + "num": 256, + "top_k": 8, + "inter": 1536, + "a1": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "w1": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "w2": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "a2": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + } + }, + "router": { + "gate": { + "dtype": "bf16" + }, + "scoring": "sigmoid", + "select": { + "kind": "topk" + }, + "bias": true, + "renormalize": true + }, + "activation": { + "kind": "silu" + }, + "routing": { + "distribution": "natural", + "seed": 0 + } + }, + "sources": [ + "MiniMaxAI/MiniMax-M2.5/block_sparse_moe" + ] + }, + { + "type": "moe", + "args": { + "tokens": 2048, + "hidden": 3072, + "experts": { + "num": 256, + "top_k": 8, + "inter": 1536, + "a1": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + }, + "w1": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "w2": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": true, + "group": [ + 128, + 128 + ] + } + }, + "a2": { + "dtype": "e4m3", + "scale": { + "dtype": "fp32", + "static": false, + "group": [ + 1, + 128 + ] + } + } + }, + "router": { + "gate": { + "dtype": "bf16" + }, + "scoring": "sigmoid", + "select": { + "kind": "topk" + }, + "bias": true, + "renormalize": true + }, + "activation": { + "kind": "silu" + }, + "routing": { + "distribution": "natural", + "seed": 0 + } + }, + "sources": [ + "MiniMaxAI/MiniMax-M2.5/block_sparse_moe" + ] + } +] diff --git a/operatorx/testlists/moe_subset.json b/operatorx/testlists/moe_subset.json deleted file mode 100644 index 308e02ec21..0000000000 --- a/operatorx/testlists/moe_subset.json +++ /dev/null @@ -1,5708 +0,0 @@ -[ - { - "type": "moe_forward", - "args": { - "num_tokens": 1243136, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 4, - "expert_parallel_size": 4, - "routed_tensor_parallel_size": 1, - "shared_tensor_parallel_size": 1, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 2428, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 4, - "expert_parallel_size": 4, - "routed_tensor_parallel_size": 1, - "shared_tensor_parallel_size": 1, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 896000, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 4, - "expert_parallel_size": 4, - "routed_tensor_parallel_size": 1, - "shared_tensor_parallel_size": 1, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 1750, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 4, - "expert_parallel_size": 4, - "routed_tensor_parallel_size": 1, - "shared_tensor_parallel_size": 1, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 6144, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 4, - "expert_parallel_size": 4, - "routed_tensor_parallel_size": 1, - "shared_tensor_parallel_size": 1, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 12, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 4, - "expert_parallel_size": 4, - "routed_tensor_parallel_size": 1, - "shared_tensor_parallel_size": 1, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 10240, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 4, - "expert_parallel_size": 4, - "routed_tensor_parallel_size": 1, - "shared_tensor_parallel_size": 1, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 20, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 4, - "expert_parallel_size": 4, - "routed_tensor_parallel_size": 1, - "shared_tensor_parallel_size": 1, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 15360, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 4, - "expert_parallel_size": 4, - "routed_tensor_parallel_size": 1, - "shared_tensor_parallel_size": 1, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 30, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 4, - "expert_parallel_size": 4, - "routed_tensor_parallel_size": 1, - "shared_tensor_parallel_size": 1, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 25600, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 4, - "expert_parallel_size": 4, - "routed_tensor_parallel_size": 1, - "shared_tensor_parallel_size": 1, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 50, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 4, - "expert_parallel_size": 4, - "routed_tensor_parallel_size": 1, - "shared_tensor_parallel_size": 1, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 46080, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 4, - "expert_parallel_size": 4, - "routed_tensor_parallel_size": 1, - "shared_tensor_parallel_size": 1, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 90, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 4, - "expert_parallel_size": 4, - "routed_tensor_parallel_size": 1, - "shared_tensor_parallel_size": 1, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 92160, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 4, - "expert_parallel_size": 4, - "routed_tensor_parallel_size": 1, - "shared_tensor_parallel_size": 1, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 180, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 4, - "expert_parallel_size": 4, - "routed_tensor_parallel_size": 1, - "shared_tensor_parallel_size": 1, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 184320, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 4, - "expert_parallel_size": 4, - "routed_tensor_parallel_size": 1, - "shared_tensor_parallel_size": 1, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 360, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 4, - "expert_parallel_size": 4, - "routed_tensor_parallel_size": 1, - "shared_tensor_parallel_size": 1, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 5087232, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 4, - "expert_parallel_size": 4, - "routed_tensor_parallel_size": 1, - "shared_tensor_parallel_size": 1, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 9936, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 4, - "expert_parallel_size": 4, - "routed_tensor_parallel_size": 1, - "shared_tensor_parallel_size": 1, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 11120640, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 4, - "expert_parallel_size": 4, - "routed_tensor_parallel_size": 1, - "shared_tensor_parallel_size": 1, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 21720, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 4, - "expert_parallel_size": 4, - "routed_tensor_parallel_size": 1, - "shared_tensor_parallel_size": 1, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 4194304, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 4, - "expert_parallel_size": 4, - "routed_tensor_parallel_size": 1, - "shared_tensor_parallel_size": 1, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 4096, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 4, - "expert_parallel_size": 4, - "routed_tensor_parallel_size": 1, - "shared_tensor_parallel_size": 1, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 2244608, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 4, - "expert_parallel_size": 4, - "routed_tensor_parallel_size": 1, - "shared_tensor_parallel_size": 1, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 2192, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 4, - "expert_parallel_size": 4, - "routed_tensor_parallel_size": 1, - "shared_tensor_parallel_size": 1, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 1397760, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 4, - "expert_parallel_size": 4, - "routed_tensor_parallel_size": 1, - "shared_tensor_parallel_size": 1, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 1365, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 4, - "expert_parallel_size": 4, - "routed_tensor_parallel_size": 1, - "shared_tensor_parallel_size": 1, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 6, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 4, - "expert_parallel_size": 4, - "routed_tensor_parallel_size": 1, - "shared_tensor_parallel_size": 1, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 10, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 4, - "expert_parallel_size": 4, - "routed_tensor_parallel_size": 1, - "shared_tensor_parallel_size": 1, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 15, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 4, - "expert_parallel_size": 4, - "routed_tensor_parallel_size": 1, - "shared_tensor_parallel_size": 1, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 25, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 4, - "expert_parallel_size": 4, - "routed_tensor_parallel_size": 1, - "shared_tensor_parallel_size": 1, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 45, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 4, - "expert_parallel_size": 4, - "routed_tensor_parallel_size": 1, - "shared_tensor_parallel_size": 1, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 460800, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 4, - "expert_parallel_size": 4, - "routed_tensor_parallel_size": 1, - "shared_tensor_parallel_size": 1, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 450, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 4, - "expert_parallel_size": 4, - "routed_tensor_parallel_size": 1, - "shared_tensor_parallel_size": 1, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 737280, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 4, - "expert_parallel_size": 4, - "routed_tensor_parallel_size": 1, - "shared_tensor_parallel_size": 1, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 540672, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 4, - "expert_parallel_size": 4, - "routed_tensor_parallel_size": 1, - "shared_tensor_parallel_size": 1, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 132, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 4, - "expert_parallel_size": 4, - "routed_tensor_parallel_size": 1, - "shared_tensor_parallel_size": 1, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 49152, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 4, - "expert_parallel_size": 4, - "routed_tensor_parallel_size": 1, - "shared_tensor_parallel_size": 1, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 81920, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 4, - "expert_parallel_size": 4, - "routed_tensor_parallel_size": 1, - "shared_tensor_parallel_size": 1, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 122880, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 4, - "expert_parallel_size": 4, - "routed_tensor_parallel_size": 1, - "shared_tensor_parallel_size": 1, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 245760, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 4, - "expert_parallel_size": 4, - "routed_tensor_parallel_size": 1, - "shared_tensor_parallel_size": 1, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 60, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 4, - "expert_parallel_size": 4, - "routed_tensor_parallel_size": 1, - "shared_tensor_parallel_size": 1, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 491520, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 4, - "expert_parallel_size": 4, - "routed_tensor_parallel_size": 1, - "shared_tensor_parallel_size": 1, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 120, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 4, - "expert_parallel_size": 4, - "routed_tensor_parallel_size": 1, - "shared_tensor_parallel_size": 1, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 4489216, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 4, - "expert_parallel_size": 4, - "routed_tensor_parallel_size": 1, - "shared_tensor_parallel_size": 1, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 1096, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 4, - "expert_parallel_size": 4, - "routed_tensor_parallel_size": 1, - "shared_tensor_parallel_size": 1, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 8978432, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 4, - "expert_parallel_size": 4, - "routed_tensor_parallel_size": 1, - "shared_tensor_parallel_size": 1, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 13852672, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 4, - "expert_parallel_size": 4, - "routed_tensor_parallel_size": 1, - "shared_tensor_parallel_size": 1, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 3382, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 4, - "expert_parallel_size": 4, - "routed_tensor_parallel_size": 1, - "shared_tensor_parallel_size": 1, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 5390336, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 4, - "expert_parallel_size": 4, - "routed_tensor_parallel_size": 1, - "shared_tensor_parallel_size": 1, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 1316, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 4, - "expert_parallel_size": 4, - "routed_tensor_parallel_size": 1, - "shared_tensor_parallel_size": 1, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 204800, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 4, - "expert_parallel_size": 4, - "routed_tensor_parallel_size": 1, - "shared_tensor_parallel_size": 1, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 409600, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 4, - "expert_parallel_size": 4, - "routed_tensor_parallel_size": 1, - "shared_tensor_parallel_size": 1, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 819200, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 4, - "expert_parallel_size": 4, - "routed_tensor_parallel_size": 1, - "shared_tensor_parallel_size": 1, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 100, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 4, - "expert_parallel_size": 4, - "routed_tensor_parallel_size": 1, - "shared_tensor_parallel_size": 1, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 3031040, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 4, - "expert_parallel_size": 4, - "routed_tensor_parallel_size": 1, - "shared_tensor_parallel_size": 1, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 370, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 4, - "expert_parallel_size": 4, - "routed_tensor_parallel_size": 1, - "shared_tensor_parallel_size": 1, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 13156352, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 4, - "expert_parallel_size": 4, - "routed_tensor_parallel_size": 1, - "shared_tensor_parallel_size": 1, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 1606, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 4, - "expert_parallel_size": 4, - "routed_tensor_parallel_size": 1, - "shared_tensor_parallel_size": 1, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 6856704, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 4, - "expert_parallel_size": 4, - "routed_tensor_parallel_size": 1, - "shared_tensor_parallel_size": 1, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 837, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 4, - "expert_parallel_size": 4, - "routed_tensor_parallel_size": 1, - "shared_tensor_parallel_size": 1, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 18202624, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 4, - "expert_parallel_size": 4, - "routed_tensor_parallel_size": 1, - "shared_tensor_parallel_size": 1, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 2222, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 4, - "expert_parallel_size": 4, - "routed_tensor_parallel_size": 1, - "shared_tensor_parallel_size": 1, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 8192, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 4, - "expert_parallel_size": 4, - "routed_tensor_parallel_size": 1, - "shared_tensor_parallel_size": 1, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 1, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 4, - "expert_parallel_size": 4, - "routed_tensor_parallel_size": 1, - "shared_tensor_parallel_size": 1, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 16384, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 4, - "expert_parallel_size": 4, - "routed_tensor_parallel_size": 1, - "shared_tensor_parallel_size": 1, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 2, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 4, - "expert_parallel_size": 4, - "routed_tensor_parallel_size": 1, - "shared_tensor_parallel_size": 1, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 32768, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 4, - "expert_parallel_size": 4, - "routed_tensor_parallel_size": 1, - "shared_tensor_parallel_size": 1, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 4, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 4, - "expert_parallel_size": 4, - "routed_tensor_parallel_size": 1, - "shared_tensor_parallel_size": 1, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 65536, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 4, - "expert_parallel_size": 4, - "routed_tensor_parallel_size": 1, - "shared_tensor_parallel_size": 1, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 8, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 4, - "expert_parallel_size": 4, - "routed_tensor_parallel_size": 1, - "shared_tensor_parallel_size": 1, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 131072, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 4, - "expert_parallel_size": 4, - "routed_tensor_parallel_size": 1, - "shared_tensor_parallel_size": 1, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 16, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 4, - "expert_parallel_size": 4, - "routed_tensor_parallel_size": 1, - "shared_tensor_parallel_size": 1, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 262144, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 4, - "expert_parallel_size": 4, - "routed_tensor_parallel_size": 1, - "shared_tensor_parallel_size": 1, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 32, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 4, - "expert_parallel_size": 4, - "routed_tensor_parallel_size": 1, - "shared_tensor_parallel_size": 1, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 5120, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 4, - "expert_parallel_size": 4, - "routed_tensor_parallel_size": 1, - "shared_tensor_parallel_size": 1, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 5, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 4, - "expert_parallel_size": 4, - "routed_tensor_parallel_size": 1, - "shared_tensor_parallel_size": 1, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 425984, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 4, - "expert_parallel_size": 4, - "routed_tensor_parallel_size": 1, - "shared_tensor_parallel_size": 1, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 104, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 4, - "expert_parallel_size": 4, - "routed_tensor_parallel_size": 1, - "shared_tensor_parallel_size": 1, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 860160, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 4, - "expert_parallel_size": 4, - "routed_tensor_parallel_size": 1, - "shared_tensor_parallel_size": 1, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 105, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 4, - "expert_parallel_size": 4, - "routed_tensor_parallel_size": 1, - "shared_tensor_parallel_size": 1, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 98304, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 4, - "expert_parallel_size": 4, - "routed_tensor_parallel_size": 1, - "shared_tensor_parallel_size": 1, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 64, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 4, - "expert_parallel_size": 4, - "routed_tensor_parallel_size": 1, - "shared_tensor_parallel_size": 1, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 128, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 4, - "expert_parallel_size": 4, - "routed_tensor_parallel_size": 1, - "shared_tensor_parallel_size": 1, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 524288, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 4, - "expert_parallel_size": 4, - "routed_tensor_parallel_size": 1, - "shared_tensor_parallel_size": 1, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 1048576, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 4, - "expert_parallel_size": 4, - "routed_tensor_parallel_size": 1, - "shared_tensor_parallel_size": 1, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 196608, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 4, - "expert_parallel_size": 4, - "routed_tensor_parallel_size": 1, - "shared_tensor_parallel_size": 1, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 24, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 4, - "expert_parallel_size": 4, - "routed_tensor_parallel_size": 1, - "shared_tensor_parallel_size": 1, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 393216, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 4, - "expert_parallel_size": 4, - "routed_tensor_parallel_size": 1, - "shared_tensor_parallel_size": 1, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 48, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 4, - "expert_parallel_size": 4, - "routed_tensor_parallel_size": 1, - "shared_tensor_parallel_size": 1, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 2097152, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 4, - "expert_parallel_size": 4, - "routed_tensor_parallel_size": 1, - "shared_tensor_parallel_size": 1, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 256, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 4, - "expert_parallel_size": 4, - "routed_tensor_parallel_size": 1, - "shared_tensor_parallel_size": 1, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 512, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 4, - "expert_parallel_size": 4, - "routed_tensor_parallel_size": 1, - "shared_tensor_parallel_size": 1, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 524288, - "hidden": 7168, - "intermediate": 3072, - "num_experts": 384, - "top_k": 6, - "dtype_act": "nvfp4", - "dtype_weight": "mxfp4", - "world_size": 4, - "expert_parallel_size": 4, - "routed_tensor_parallel_size": 1, - "shared_tensor_parallel_size": 1, - "n_shared_experts": 1 - }, - "name": "dsv4" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 512, - "hidden": 7168, - "intermediate": 3072, - "num_experts": 384, - "top_k": 6, - "dtype_act": "nvfp4", - "dtype_weight": "mxfp4", - "world_size": 4, - "expert_parallel_size": 4, - "routed_tensor_parallel_size": 1, - "shared_tensor_parallel_size": 1, - "n_shared_experts": 1 - }, - "name": "dsv4" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 4194304, - "hidden": 7168, - "intermediate": 3072, - "num_experts": 384, - "top_k": 6, - "dtype_act": "nvfp4", - "dtype_weight": "mxfp4", - "world_size": 4, - "expert_parallel_size": 4, - "routed_tensor_parallel_size": 1, - "shared_tensor_parallel_size": 1, - "n_shared_experts": 1 - }, - "name": "dsv4" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 262144, - "hidden": 7168, - "intermediate": 3072, - "num_experts": 384, - "top_k": 6, - "dtype_act": "nvfp4", - "dtype_weight": "mxfp4", - "world_size": 4, - "expert_parallel_size": 4, - "routed_tensor_parallel_size": 1, - "shared_tensor_parallel_size": 1, - "n_shared_experts": 1 - }, - "name": "dsv4" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 256, - "hidden": 7168, - "intermediate": 3072, - "num_experts": 384, - "top_k": 6, - "dtype_act": "nvfp4", - "dtype_weight": "mxfp4", - "world_size": 4, - "expert_parallel_size": 4, - "routed_tensor_parallel_size": 1, - "shared_tensor_parallel_size": 1, - "n_shared_experts": 1 - }, - "name": "dsv4" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 2097152, - "hidden": 7168, - "intermediate": 3072, - "num_experts": 384, - "top_k": 6, - "dtype_act": "nvfp4", - "dtype_weight": "mxfp4", - "world_size": 4, - "expert_parallel_size": 4, - "routed_tensor_parallel_size": 1, - "shared_tensor_parallel_size": 1, - "n_shared_experts": 1 - }, - "name": "dsv4" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 2048, - "hidden": 7168, - "intermediate": 3072, - "num_experts": 384, - "top_k": 6, - "dtype_act": "nvfp4", - "dtype_weight": "mxfp4", - "world_size": 4, - "expert_parallel_size": 4, - "routed_tensor_parallel_size": 1, - "shared_tensor_parallel_size": 1, - "n_shared_experts": 1 - }, - "name": "dsv4" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 65536, - "hidden": 7168, - "intermediate": 3072, - "num_experts": 384, - "top_k": 6, - "dtype_act": "nvfp4", - "dtype_weight": "mxfp4", - "world_size": 4, - "expert_parallel_size": 4, - "routed_tensor_parallel_size": 1, - "shared_tensor_parallel_size": 1, - "n_shared_experts": 1 - }, - "name": "dsv4" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 64, - "hidden": 7168, - "intermediate": 3072, - "num_experts": 384, - "top_k": 6, - "dtype_act": "nvfp4", - "dtype_weight": "mxfp4", - "world_size": 4, - "expert_parallel_size": 4, - "routed_tensor_parallel_size": 1, - "shared_tensor_parallel_size": 1, - "n_shared_experts": 1 - }, - "name": "dsv4" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 131072, - "hidden": 7168, - "intermediate": 3072, - "num_experts": 384, - "top_k": 6, - "dtype_act": "nvfp4", - "dtype_weight": "mxfp4", - "world_size": 4, - "expert_parallel_size": 4, - "routed_tensor_parallel_size": 1, - "shared_tensor_parallel_size": 1, - "n_shared_experts": 1 - }, - "name": "dsv4" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 128, - "hidden": 7168, - "intermediate": 3072, - "num_experts": 384, - "top_k": 6, - "dtype_act": "nvfp4", - "dtype_weight": "mxfp4", - "world_size": 4, - "expert_parallel_size": 4, - "routed_tensor_parallel_size": 1, - "shared_tensor_parallel_size": 1, - "n_shared_experts": 1 - }, - "name": "dsv4" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 1048576, - "hidden": 7168, - "intermediate": 3072, - "num_experts": 384, - "top_k": 6, - "dtype_act": "nvfp4", - "dtype_weight": "mxfp4", - "world_size": 4, - "expert_parallel_size": 4, - "routed_tensor_parallel_size": 1, - "shared_tensor_parallel_size": 1, - "n_shared_experts": 1 - }, - "name": "dsv4" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 65536, - "hidden": 2880, - "intermediate": 2880, - "num_experts": 128, - "top_k": 4, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 4, - "expert_parallel_size": 4, - "routed_tensor_parallel_size": 1, - "shared_tensor_parallel_size": 1, - "n_shared_experts": 0 - }, - "name": "gptoss" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 64, - "hidden": 2880, - "intermediate": 2880, - "num_experts": 128, - "top_k": 4, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 4, - "expert_parallel_size": 4, - "routed_tensor_parallel_size": 1, - "shared_tensor_parallel_size": 1, - "n_shared_experts": 0 - }, - "name": "gptoss" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 262144, - "hidden": 3072, - "intermediate": 1536, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "world_size": 4, - "expert_parallel_size": 4, - "routed_tensor_parallel_size": 1, - "shared_tensor_parallel_size": 1, - "n_shared_experts": 0 - }, - "name": "minimaxm2.5" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 256, - "hidden": 3072, - "intermediate": 1536, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "world_size": 4, - "expert_parallel_size": 4, - "routed_tensor_parallel_size": 1, - "shared_tensor_parallel_size": 1, - "n_shared_experts": 0 - }, - "name": "minimaxm2.5" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 524288, - "hidden": 3072, - "intermediate": 1536, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "world_size": 4, - "expert_parallel_size": 4, - "routed_tensor_parallel_size": 1, - "shared_tensor_parallel_size": 1, - "n_shared_experts": 0 - }, - "name": "minimaxm2.5" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 512, - "hidden": 3072, - "intermediate": 1536, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "world_size": 4, - "expert_parallel_size": 4, - "routed_tensor_parallel_size": 1, - "shared_tensor_parallel_size": 1, - "n_shared_experts": 0 - }, - "name": "minimaxm2.5" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 65536, - "hidden": 3072, - "intermediate": 1536, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 4, - "expert_parallel_size": 4, - "routed_tensor_parallel_size": 1, - "shared_tensor_parallel_size": 1, - "n_shared_experts": 0 - }, - "name": "minimaxm2.5" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 64, - "hidden": 3072, - "intermediate": 1536, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 4, - "expert_parallel_size": 4, - "routed_tensor_parallel_size": 1, - "shared_tensor_parallel_size": 1, - "n_shared_experts": 0 - }, - "name": "minimaxm2.5" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 131072, - "hidden": 3072, - "intermediate": 1536, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 4, - "expert_parallel_size": 4, - "routed_tensor_parallel_size": 1, - "shared_tensor_parallel_size": 1, - "n_shared_experts": 0 - }, - "name": "minimaxm2.5" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 128, - "hidden": 3072, - "intermediate": 1536, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 4, - "expert_parallel_size": 4, - "routed_tensor_parallel_size": 1, - "shared_tensor_parallel_size": 1, - "n_shared_experts": 0 - }, - "name": "minimaxm2.5" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 4096, - "hidden": 3072, - "intermediate": 1536, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "world_size": 4, - "expert_parallel_size": 4, - "routed_tensor_parallel_size": 1, - "shared_tensor_parallel_size": 1, - "n_shared_experts": 0 - }, - "name": "minimaxm2.5" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 4, - "hidden": 3072, - "intermediate": 1536, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "world_size": 4, - "expert_parallel_size": 4, - "routed_tensor_parallel_size": 1, - "shared_tensor_parallel_size": 1, - "n_shared_experts": 0 - }, - "name": "minimaxm2.5" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 8192, - "hidden": 3072, - "intermediate": 1536, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "world_size": 4, - "expert_parallel_size": 4, - "routed_tensor_parallel_size": 1, - "shared_tensor_parallel_size": 1, - "n_shared_experts": 0 - }, - "name": "minimaxm2.5" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 8, - "hidden": 3072, - "intermediate": 1536, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "world_size": 4, - "expert_parallel_size": 4, - "routed_tensor_parallel_size": 1, - "shared_tensor_parallel_size": 1, - "n_shared_experts": 0 - }, - "name": "minimaxm2.5" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 16384, - "hidden": 3072, - "intermediate": 1536, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "world_size": 4, - "expert_parallel_size": 4, - "routed_tensor_parallel_size": 1, - "shared_tensor_parallel_size": 1, - "n_shared_experts": 0 - }, - "name": "minimaxm2.5" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 16, - "hidden": 3072, - "intermediate": 1536, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "world_size": 4, - "expert_parallel_size": 4, - "routed_tensor_parallel_size": 1, - "shared_tensor_parallel_size": 1, - "n_shared_experts": 0 - }, - "name": "minimaxm2.5" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 32768, - "hidden": 3072, - "intermediate": 1536, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "world_size": 4, - "expert_parallel_size": 4, - "routed_tensor_parallel_size": 1, - "shared_tensor_parallel_size": 1, - "n_shared_experts": 0 - }, - "name": "minimaxm2.5" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 32, - "hidden": 3072, - "intermediate": 1536, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "world_size": 4, - "expert_parallel_size": 4, - "routed_tensor_parallel_size": 1, - "shared_tensor_parallel_size": 1, - "n_shared_experts": 0 - }, - "name": "minimaxm2.5" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 65536, - "hidden": 3072, - "intermediate": 1536, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "world_size": 4, - "expert_parallel_size": 4, - "routed_tensor_parallel_size": 1, - "shared_tensor_parallel_size": 1, - "n_shared_experts": 0 - }, - "name": "minimaxm2.5" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 64, - "hidden": 3072, - "intermediate": 1536, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "world_size": 4, - "expert_parallel_size": 4, - "routed_tensor_parallel_size": 1, - "shared_tensor_parallel_size": 1, - "n_shared_experts": 0 - }, - "name": "minimaxm2.5" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 131072, - "hidden": 3072, - "intermediate": 1536, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "world_size": 4, - "expert_parallel_size": 4, - "routed_tensor_parallel_size": 1, - "shared_tensor_parallel_size": 1, - "n_shared_experts": 0 - }, - "name": "minimaxm2.5" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 4096, - "hidden": 2880, - "intermediate": 2880, - "num_experts": 128, - "top_k": 4, - "dtype_act": "bf16", - "dtype_weight": "mxfp4", - "world_size": 4, - "expert_parallel_size": 4, - "routed_tensor_parallel_size": 1, - "shared_tensor_parallel_size": 1, - "n_shared_experts": 0 - }, - "name": "gptoss" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 4, - "hidden": 2880, - "intermediate": 2880, - "num_experts": 128, - "top_k": 4, - "dtype_act": "bf16", - "dtype_weight": "mxfp4", - "world_size": 4, - "expert_parallel_size": 4, - "routed_tensor_parallel_size": 1, - "shared_tensor_parallel_size": 1, - "n_shared_experts": 0 - }, - "name": "gptoss" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 8192, - "hidden": 2880, - "intermediate": 2880, - "num_experts": 128, - "top_k": 4, - "dtype_act": "bf16", - "dtype_weight": "mxfp4", - "world_size": 4, - "expert_parallel_size": 4, - "routed_tensor_parallel_size": 1, - "shared_tensor_parallel_size": 1, - "n_shared_experts": 0 - }, - "name": "gptoss" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 8, - "hidden": 2880, - "intermediate": 2880, - "num_experts": 128, - "top_k": 4, - "dtype_act": "bf16", - "dtype_weight": "mxfp4", - "world_size": 4, - "expert_parallel_size": 4, - "routed_tensor_parallel_size": 1, - "shared_tensor_parallel_size": 1, - "n_shared_experts": 0 - }, - "name": "gptoss" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 16384, - "hidden": 2880, - "intermediate": 2880, - "num_experts": 128, - "top_k": 4, - "dtype_act": "bf16", - "dtype_weight": "mxfp4", - "world_size": 4, - "expert_parallel_size": 4, - "routed_tensor_parallel_size": 1, - "shared_tensor_parallel_size": 1, - "n_shared_experts": 0 - }, - "name": "gptoss" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 16, - "hidden": 2880, - "intermediate": 2880, - "num_experts": 128, - "top_k": 4, - "dtype_act": "bf16", - "dtype_weight": "mxfp4", - "world_size": 4, - "expert_parallel_size": 4, - "routed_tensor_parallel_size": 1, - "shared_tensor_parallel_size": 1, - "n_shared_experts": 0 - }, - "name": "gptoss" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 32768, - "hidden": 2880, - "intermediate": 2880, - "num_experts": 128, - "top_k": 4, - "dtype_act": "bf16", - "dtype_weight": "mxfp4", - "world_size": 4, - "expert_parallel_size": 4, - "routed_tensor_parallel_size": 1, - "shared_tensor_parallel_size": 1, - "n_shared_experts": 0 - }, - "name": "gptoss" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 32, - "hidden": 2880, - "intermediate": 2880, - "num_experts": 128, - "top_k": 4, - "dtype_act": "bf16", - "dtype_weight": "mxfp4", - "world_size": 4, - "expert_parallel_size": 4, - "routed_tensor_parallel_size": 1, - "shared_tensor_parallel_size": 1, - "n_shared_experts": 0 - }, - "name": "gptoss" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 65536, - "hidden": 2880, - "intermediate": 2880, - "num_experts": 128, - "top_k": 4, - "dtype_act": "bf16", - "dtype_weight": "mxfp4", - "world_size": 4, - "expert_parallel_size": 4, - "routed_tensor_parallel_size": 1, - "shared_tensor_parallel_size": 1, - "n_shared_experts": 0 - }, - "name": "gptoss" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 131072, - "hidden": 2880, - "intermediate": 2880, - "num_experts": 128, - "top_k": 4, - "dtype_act": "bf16", - "dtype_weight": "mxfp4", - "world_size": 4, - "expert_parallel_size": 4, - "routed_tensor_parallel_size": 1, - "shared_tensor_parallel_size": 1, - "n_shared_experts": 0 - }, - "name": "gptoss" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 262144, - "hidden": 2880, - "intermediate": 2880, - "num_experts": 128, - "top_k": 4, - "dtype_act": "bf16", - "dtype_weight": "mxfp4", - "world_size": 4, - "expert_parallel_size": 4, - "routed_tensor_parallel_size": 1, - "shared_tensor_parallel_size": 1, - "n_shared_experts": 0 - }, - "name": "gptoss" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 524288, - "hidden": 2880, - "intermediate": 2880, - "num_experts": 128, - "top_k": 4, - "dtype_act": "bf16", - "dtype_weight": "mxfp4", - "world_size": 4, - "expert_parallel_size": 4, - "routed_tensor_parallel_size": 1, - "shared_tensor_parallel_size": 1, - "n_shared_experts": 0 - }, - "name": "gptoss" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 64, - "hidden": 2880, - "intermediate": 2880, - "num_experts": 128, - "top_k": 4, - "dtype_act": "bf16", - "dtype_weight": "mxfp4", - "world_size": 4, - "expert_parallel_size": 4, - "routed_tensor_parallel_size": 1, - "shared_tensor_parallel_size": 1, - "n_shared_experts": 0 - }, - "name": "gptoss" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 12288, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 4, - "expert_parallel_size": 4, - "routed_tensor_parallel_size": 1, - "shared_tensor_parallel_size": 1, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 24576, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 4, - "expert_parallel_size": 4, - "routed_tensor_parallel_size": 1, - "shared_tensor_parallel_size": 1, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 96, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 4, - "expert_parallel_size": 4, - "routed_tensor_parallel_size": 1, - "shared_tensor_parallel_size": 1, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 4404224, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 4, - "expert_parallel_size": 4, - "routed_tensor_parallel_size": 1, - "shared_tensor_parallel_size": 1, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 8602, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 4, - "expert_parallel_size": 4, - "routed_tensor_parallel_size": 1, - "shared_tensor_parallel_size": 1, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 2307072, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 4, - "expert_parallel_size": 4, - "routed_tensor_parallel_size": 1, - "shared_tensor_parallel_size": 1, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 4506, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 4, - "expert_parallel_size": 4, - "routed_tensor_parallel_size": 1, - "shared_tensor_parallel_size": 1, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 16517120, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 4, - "expert_parallel_size": 4, - "routed_tensor_parallel_size": 1, - "shared_tensor_parallel_size": 1, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 32260, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 4, - "expert_parallel_size": 4, - "routed_tensor_parallel_size": 1, - "shared_tensor_parallel_size": 1, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 4301, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 4, - "expert_parallel_size": 4, - "routed_tensor_parallel_size": 1, - "shared_tensor_parallel_size": 1, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 681984, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 4, - "expert_parallel_size": 4, - "routed_tensor_parallel_size": 1, - "shared_tensor_parallel_size": 1, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 666, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 4, - "expert_parallel_size": 4, - "routed_tensor_parallel_size": 1, - "shared_tensor_parallel_size": 1, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 6291456, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 4, - "expert_parallel_size": 4, - "routed_tensor_parallel_size": 1, - "shared_tensor_parallel_size": 1, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 192, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 4, - "expert_parallel_size": 4, - "routed_tensor_parallel_size": 1, - "shared_tensor_parallel_size": 1, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 2253, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 4, - "expert_parallel_size": 4, - "routed_tensor_parallel_size": 1, - "shared_tensor_parallel_size": 1, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 1474560, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 4, - "expert_parallel_size": 4, - "routed_tensor_parallel_size": 1, - "shared_tensor_parallel_size": 1, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 10067968, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 4, - "expert_parallel_size": 4, - "routed_tensor_parallel_size": 1, - "shared_tensor_parallel_size": 1, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 2458, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 4, - "expert_parallel_size": 4, - "routed_tensor_parallel_size": 1, - "shared_tensor_parallel_size": 1, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 5455872, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 4, - "expert_parallel_size": 4, - "routed_tensor_parallel_size": 1, - "shared_tensor_parallel_size": 1, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 1332, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 4, - "expert_parallel_size": 4, - "routed_tensor_parallel_size": 1, - "shared_tensor_parallel_size": 1, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 35233792, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 4, - "expert_parallel_size": 4, - "routed_tensor_parallel_size": 1, - "shared_tensor_parallel_size": 1, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 360448, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 4, - "expert_parallel_size": 4, - "routed_tensor_parallel_size": 1, - "shared_tensor_parallel_size": 1, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 44, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 4, - "expert_parallel_size": 4, - "routed_tensor_parallel_size": 1, - "shared_tensor_parallel_size": 1, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 622592, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 4, - "expert_parallel_size": 4, - "routed_tensor_parallel_size": 1, - "shared_tensor_parallel_size": 1, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 76, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 4, - "expert_parallel_size": 4, - "routed_tensor_parallel_size": 1, - "shared_tensor_parallel_size": 1, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 40960, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 4, - "expert_parallel_size": 4, - "routed_tensor_parallel_size": 1, - "shared_tensor_parallel_size": 1, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 2727936, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 4, - "expert_parallel_size": 4, - "routed_tensor_parallel_size": 1, - "shared_tensor_parallel_size": 1, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 333, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 4, - "expert_parallel_size": 4, - "routed_tensor_parallel_size": 1, - "shared_tensor_parallel_size": 1, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 1229, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 4, - "expert_parallel_size": 4, - "routed_tensor_parallel_size": 1, - "shared_tensor_parallel_size": 1, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 18456576, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 4, - "expert_parallel_size": 4, - "routed_tensor_parallel_size": 1, - "shared_tensor_parallel_size": 1, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 33554432, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 4, - "expert_parallel_size": 4, - "routed_tensor_parallel_size": 1, - "shared_tensor_parallel_size": 1, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 2048, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 4, - "expert_parallel_size": 4, - "routed_tensor_parallel_size": 1, - "shared_tensor_parallel_size": 1, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 8388608, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 4, - "expert_parallel_size": 4, - "routed_tensor_parallel_size": 1, - "shared_tensor_parallel_size": 1, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 3303424, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 4, - "expert_parallel_size": 4, - "routed_tensor_parallel_size": 1, - "shared_tensor_parallel_size": 1, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 6452, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 4, - "expert_parallel_size": 4, - "routed_tensor_parallel_size": 1, - "shared_tensor_parallel_size": 1, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 26443776, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 4, - "expert_parallel_size": 4, - "routed_tensor_parallel_size": 1, - "shared_tensor_parallel_size": 1, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 3228, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 4, - "expert_parallel_size": 4, - "routed_tensor_parallel_size": 1, - "shared_tensor_parallel_size": 1, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 8192, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "world_size": 4, - "expert_parallel_size": 4, - "routed_tensor_parallel_size": 1, - "shared_tensor_parallel_size": 1, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 16, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "world_size": 4, - "expert_parallel_size": 4, - "routed_tensor_parallel_size": 1, - "shared_tensor_parallel_size": 1, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 24576, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "world_size": 4, - "expert_parallel_size": 4, - "routed_tensor_parallel_size": 1, - "shared_tensor_parallel_size": 1, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 48, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "world_size": 4, - "expert_parallel_size": 4, - "routed_tensor_parallel_size": 1, - "shared_tensor_parallel_size": 1, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 184320, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "world_size": 4, - "expert_parallel_size": 4, - "routed_tensor_parallel_size": 1, - "shared_tensor_parallel_size": 1, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 360, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "world_size": 4, - "expert_parallel_size": 4, - "routed_tensor_parallel_size": 1, - "shared_tensor_parallel_size": 1, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 577536, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "world_size": 4, - "expert_parallel_size": 4, - "routed_tensor_parallel_size": 1, - "shared_tensor_parallel_size": 1, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 1128, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "world_size": 4, - "expert_parallel_size": 4, - "routed_tensor_parallel_size": 1, - "shared_tensor_parallel_size": 1, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 681984, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "world_size": 4, - "expert_parallel_size": 4, - "routed_tensor_parallel_size": 1, - "shared_tensor_parallel_size": 1, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 1332, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "world_size": 4, - "expert_parallel_size": 4, - "routed_tensor_parallel_size": 1, - "shared_tensor_parallel_size": 1, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 2307072, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "world_size": 4, - "expert_parallel_size": 4, - "routed_tensor_parallel_size": 1, - "shared_tensor_parallel_size": 1, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 4506, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "world_size": 4, - "expert_parallel_size": 4, - "routed_tensor_parallel_size": 1, - "shared_tensor_parallel_size": 1, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 8388608, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "world_size": 4, - "expert_parallel_size": 4, - "routed_tensor_parallel_size": 1, - "shared_tensor_parallel_size": 1, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 16384, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "world_size": 4, - "expert_parallel_size": 4, - "routed_tensor_parallel_size": 1, - "shared_tensor_parallel_size": 1, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 4096, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "world_size": 4, - "expert_parallel_size": 4, - "routed_tensor_parallel_size": 1, - "shared_tensor_parallel_size": 1, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 4, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "world_size": 4, - "expert_parallel_size": 4, - "routed_tensor_parallel_size": 1, - "shared_tensor_parallel_size": 1, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 24, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "world_size": 4, - "expert_parallel_size": 4, - "routed_tensor_parallel_size": 1, - "shared_tensor_parallel_size": 1, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 86016, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "world_size": 4, - "expert_parallel_size": 4, - "routed_tensor_parallel_size": 1, - "shared_tensor_parallel_size": 1, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 84, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "world_size": 4, - "expert_parallel_size": 4, - "routed_tensor_parallel_size": 1, - "shared_tensor_parallel_size": 1, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 1258496, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "world_size": 4, - "expert_parallel_size": 4, - "routed_tensor_parallel_size": 1, - "shared_tensor_parallel_size": 1, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 1229, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "world_size": 4, - "expert_parallel_size": 4, - "routed_tensor_parallel_size": 1, - "shared_tensor_parallel_size": 1, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 2253, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "world_size": 4, - "expert_parallel_size": 4, - "routed_tensor_parallel_size": 1, - "shared_tensor_parallel_size": 1, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 8808448, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "world_size": 4, - "expert_parallel_size": 4, - "routed_tensor_parallel_size": 1, - "shared_tensor_parallel_size": 1, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 8602, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "world_size": 4, - "expert_parallel_size": 4, - "routed_tensor_parallel_size": 1, - "shared_tensor_parallel_size": 1, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 12582912, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "world_size": 4, - "expert_parallel_size": 4, - "routed_tensor_parallel_size": 1, - "shared_tensor_parallel_size": 1, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 12288, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "world_size": 4, - "expert_parallel_size": 4, - "routed_tensor_parallel_size": 1, - "shared_tensor_parallel_size": 1, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 65536, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "world_size": 4, - "expert_parallel_size": 4, - "routed_tensor_parallel_size": 1, - "shared_tensor_parallel_size": 1, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 196608, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "world_size": 4, - "expert_parallel_size": 4, - "routed_tensor_parallel_size": 1, - "shared_tensor_parallel_size": 1, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 2727936, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "world_size": 4, - "expert_parallel_size": 4, - "routed_tensor_parallel_size": 1, - "shared_tensor_parallel_size": 1, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 666, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "world_size": 4, - "expert_parallel_size": 4, - "routed_tensor_parallel_size": 1, - "shared_tensor_parallel_size": 1, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 5455872, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "world_size": 4, - "expert_parallel_size": 4, - "routed_tensor_parallel_size": 1, - "shared_tensor_parallel_size": 1, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 10067968, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "world_size": 4, - "expert_parallel_size": 4, - "routed_tensor_parallel_size": 1, - "shared_tensor_parallel_size": 1, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 2458, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "world_size": 4, - "expert_parallel_size": 4, - "routed_tensor_parallel_size": 1, - "shared_tensor_parallel_size": 1, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 32768, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "world_size": 4, - "expert_parallel_size": 4, - "routed_tensor_parallel_size": 1, - "shared_tensor_parallel_size": 1, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 294912, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "world_size": 4, - "expert_parallel_size": 4, - "routed_tensor_parallel_size": 1, - "shared_tensor_parallel_size": 1, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 36, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "world_size": 4, - "expert_parallel_size": 4, - "routed_tensor_parallel_size": 1, - "shared_tensor_parallel_size": 1, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 4194304, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "world_size": 4, - "expert_parallel_size": 4, - "routed_tensor_parallel_size": 1, - "shared_tensor_parallel_size": 1, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 512, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "world_size": 4, - "expert_parallel_size": 4, - "routed_tensor_parallel_size": 1, - "shared_tensor_parallel_size": 1, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 17620992, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "world_size": 4, - "expert_parallel_size": 4, - "routed_tensor_parallel_size": 1, - "shared_tensor_parallel_size": 1, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 2151, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "world_size": 4, - "expert_parallel_size": 4, - "routed_tensor_parallel_size": 1, - "shared_tensor_parallel_size": 1, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 262144, - "hidden": 2880, - "intermediate": 2880, - "num_experts": 128, - "top_k": 4, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 4, - "expert_parallel_size": 4, - "routed_tensor_parallel_size": 1, - "shared_tensor_parallel_size": 1, - "n_shared_experts": 0 - }, - "name": "gptoss" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 256, - "hidden": 2880, - "intermediate": 2880, - "num_experts": 128, - "top_k": 4, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 4, - "expert_parallel_size": 4, - "routed_tensor_parallel_size": 1, - "shared_tensor_parallel_size": 1, - "n_shared_experts": 0 - }, - "name": "gptoss" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 1048576, - "hidden": 2880, - "intermediate": 2880, - "num_experts": 128, - "top_k": 4, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 4, - "expert_parallel_size": 4, - "routed_tensor_parallel_size": 1, - "shared_tensor_parallel_size": 1, - "n_shared_experts": 0 - }, - "name": "gptoss" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 1024, - "hidden": 2880, - "intermediate": 2880, - "num_experts": 128, - "top_k": 4, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 4, - "expert_parallel_size": 4, - "routed_tensor_parallel_size": 1, - "shared_tensor_parallel_size": 1, - "n_shared_experts": 0 - }, - "name": "gptoss" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 1572864, - "hidden": 2880, - "intermediate": 2880, - "num_experts": 128, - "top_k": 4, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 4, - "expert_parallel_size": 4, - "routed_tensor_parallel_size": 1, - "shared_tensor_parallel_size": 1, - "n_shared_experts": 0 - }, - "name": "gptoss" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 1536, - "hidden": 2880, - "intermediate": 2880, - "num_experts": 128, - "top_k": 4, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 4, - "expert_parallel_size": 4, - "routed_tensor_parallel_size": 1, - "shared_tensor_parallel_size": 1, - "n_shared_experts": 0 - }, - "name": "gptoss" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 3145728, - "hidden": 2880, - "intermediate": 2880, - "num_experts": 128, - "top_k": 4, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 4, - "expert_parallel_size": 4, - "routed_tensor_parallel_size": 1, - "shared_tensor_parallel_size": 1, - "n_shared_experts": 0 - }, - "name": "gptoss" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 3072, - "hidden": 2880, - "intermediate": 2880, - "num_experts": 128, - "top_k": 4, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 4, - "expert_parallel_size": 4, - "routed_tensor_parallel_size": 1, - "shared_tensor_parallel_size": 1, - "n_shared_experts": 0 - }, - "name": "gptoss" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 1024, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 4, - "expert_parallel_size": 4, - "routed_tensor_parallel_size": 1, - "shared_tensor_parallel_size": 1, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 16777216, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 4, - "expert_parallel_size": 4, - "routed_tensor_parallel_size": 1, - "shared_tensor_parallel_size": 1, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 4096, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 384, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 4, - "expert_parallel_size": 4, - "routed_tensor_parallel_size": 1, - "shared_tensor_parallel_size": 1, - "n_shared_experts": 1 - }, - "name": "kimik2.5" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 4, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 384, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 4, - "expert_parallel_size": 4, - "routed_tensor_parallel_size": 1, - "shared_tensor_parallel_size": 1, - "n_shared_experts": 1 - }, - "name": "kimik2.5" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 196608, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 384, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 4, - "expert_parallel_size": 4, - "routed_tensor_parallel_size": 1, - "shared_tensor_parallel_size": 1, - "n_shared_experts": 1 - }, - "name": "kimik2.5" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 192, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 384, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 4, - "expert_parallel_size": 4, - "routed_tensor_parallel_size": 1, - "shared_tensor_parallel_size": 1, - "n_shared_experts": 1 - }, - "name": "kimik2.5" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 368640, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 384, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 4, - "expert_parallel_size": 4, - "routed_tensor_parallel_size": 1, - "shared_tensor_parallel_size": 1, - "n_shared_experts": 1 - }, - "name": "kimik2.5" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 360, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 384, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 4, - "expert_parallel_size": 4, - "routed_tensor_parallel_size": 1, - "shared_tensor_parallel_size": 1, - "n_shared_experts": 1 - }, - "name": "kimik2.5" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 684032, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 384, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 4, - "expert_parallel_size": 4, - "routed_tensor_parallel_size": 1, - "shared_tensor_parallel_size": 1, - "n_shared_experts": 1 - }, - "name": "kimik2.5" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 668, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 384, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 4, - "expert_parallel_size": 4, - "routed_tensor_parallel_size": 1, - "shared_tensor_parallel_size": 1, - "n_shared_experts": 1 - }, - "name": "kimik2.5" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 5120, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 384, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 4, - "expert_parallel_size": 4, - "routed_tensor_parallel_size": 1, - "shared_tensor_parallel_size": 1, - "n_shared_experts": 1 - }, - "name": "kimik2.5" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 5, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 384, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 4, - "expert_parallel_size": 4, - "routed_tensor_parallel_size": 1, - "shared_tensor_parallel_size": 1, - "n_shared_experts": 1 - }, - "name": "kimik2.5" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 15360, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 384, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 4, - "expert_parallel_size": 4, - "routed_tensor_parallel_size": 1, - "shared_tensor_parallel_size": 1, - "n_shared_experts": 1 - }, - "name": "kimik2.5" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 15, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 384, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 4, - "expert_parallel_size": 4, - "routed_tensor_parallel_size": 1, - "shared_tensor_parallel_size": 1, - "n_shared_experts": 1 - }, - "name": "kimik2.5" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 30720, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 384, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 4, - "expert_parallel_size": 4, - "routed_tensor_parallel_size": 1, - "shared_tensor_parallel_size": 1, - "n_shared_experts": 1 - }, - "name": "kimik2.5" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 30, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 384, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 4, - "expert_parallel_size": 4, - "routed_tensor_parallel_size": 1, - "shared_tensor_parallel_size": 1, - "n_shared_experts": 1 - }, - "name": "kimik2.5" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 56320, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 384, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 4, - "expert_parallel_size": 4, - "routed_tensor_parallel_size": 1, - "shared_tensor_parallel_size": 1, - "n_shared_experts": 1 - }, - "name": "kimik2.5" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 55, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 384, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 4, - "expert_parallel_size": 4, - "routed_tensor_parallel_size": 1, - "shared_tensor_parallel_size": 1, - "n_shared_experts": 1 - }, - "name": "kimik2.5" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 681984, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 384, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 4, - "expert_parallel_size": 4, - "routed_tensor_parallel_size": 1, - "shared_tensor_parallel_size": 1, - "n_shared_experts": 1 - }, - "name": "kimik2.5" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 666, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 384, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 4, - "expert_parallel_size": 4, - "routed_tensor_parallel_size": 1, - "shared_tensor_parallel_size": 1, - "n_shared_experts": 1 - }, - "name": "kimik2.5" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 2307072, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 384, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 4, - "expert_parallel_size": 4, - "routed_tensor_parallel_size": 1, - "shared_tensor_parallel_size": 1, - "n_shared_experts": 1 - }, - "name": "kimik2.5" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 2253, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 384, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 4, - "expert_parallel_size": 4, - "routed_tensor_parallel_size": 1, - "shared_tensor_parallel_size": 1, - "n_shared_experts": 1 - }, - "name": "kimik2.5" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 4404224, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 384, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 4, - "expert_parallel_size": 4, - "routed_tensor_parallel_size": 1, - "shared_tensor_parallel_size": 1, - "n_shared_experts": 1 - }, - "name": "kimik2.5" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 4301, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 384, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 4, - "expert_parallel_size": 4, - "routed_tensor_parallel_size": 1, - "shared_tensor_parallel_size": 1, - "n_shared_experts": 1 - }, - "name": "kimik2.5" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 6606848, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 384, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 4, - "expert_parallel_size": 4, - "routed_tensor_parallel_size": 1, - "shared_tensor_parallel_size": 1, - "n_shared_experts": 1 - }, - "name": "kimik2.5" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 6452, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 384, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 4, - "expert_parallel_size": 4, - "routed_tensor_parallel_size": 1, - "shared_tensor_parallel_size": 1, - "n_shared_experts": 1 - }, - "name": "kimik2.5" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 32768, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 384, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 4, - "expert_parallel_size": 4, - "routed_tensor_parallel_size": 1, - "shared_tensor_parallel_size": 1, - "n_shared_experts": 1 - }, - "name": "kimik2.5" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 1277952, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 384, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 4, - "expert_parallel_size": 4, - "routed_tensor_parallel_size": 1, - "shared_tensor_parallel_size": 1, - "n_shared_experts": 1 - }, - "name": "kimik2.5" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 156, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 384, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 4, - "expert_parallel_size": 4, - "routed_tensor_parallel_size": 1, - "shared_tensor_parallel_size": 1, - "n_shared_experts": 1 - }, - "name": "kimik2.5" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 40960, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 384, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 4, - "expert_parallel_size": 4, - "routed_tensor_parallel_size": 1, - "shared_tensor_parallel_size": 1, - "n_shared_experts": 1 - }, - "name": "kimik2.5" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 122880, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 384, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 4, - "expert_parallel_size": 4, - "routed_tensor_parallel_size": 1, - "shared_tensor_parallel_size": 1, - "n_shared_experts": 1 - }, - "name": "kimik2.5" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 245760, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 384, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 4, - "expert_parallel_size": 4, - "routed_tensor_parallel_size": 1, - "shared_tensor_parallel_size": 1, - "n_shared_experts": 1 - }, - "name": "kimik2.5" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 491520, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 384, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 4, - "expert_parallel_size": 4, - "routed_tensor_parallel_size": 1, - "shared_tensor_parallel_size": 1, - "n_shared_experts": 1 - }, - "name": "kimik2.5" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 60, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 384, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 4, - "expert_parallel_size": 4, - "routed_tensor_parallel_size": 1, - "shared_tensor_parallel_size": 1, - "n_shared_experts": 1 - }, - "name": "kimik2.5" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 860160, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 384, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 4, - "expert_parallel_size": 4, - "routed_tensor_parallel_size": 1, - "shared_tensor_parallel_size": 1, - "n_shared_experts": 1 - }, - "name": "kimik2.5" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 105, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 384, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 4, - "expert_parallel_size": 4, - "routed_tensor_parallel_size": 1, - "shared_tensor_parallel_size": 1, - "n_shared_experts": 1 - }, - "name": "kimik2.5" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 2727936, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 384, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 4, - "expert_parallel_size": 4, - "routed_tensor_parallel_size": 1, - "shared_tensor_parallel_size": 1, - "n_shared_experts": 1 - }, - "name": "kimik2.5" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 333, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 384, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 4, - "expert_parallel_size": 4, - "routed_tensor_parallel_size": 1, - "shared_tensor_parallel_size": 1, - "n_shared_experts": 1 - }, - "name": "kimik2.5" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 5038080, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 384, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 4, - "expert_parallel_size": 4, - "routed_tensor_parallel_size": 1, - "shared_tensor_parallel_size": 1, - "n_shared_experts": 1 - }, - "name": "kimik2.5" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 615, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 384, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 4, - "expert_parallel_size": 4, - "routed_tensor_parallel_size": 1, - "shared_tensor_parallel_size": 1, - "n_shared_experts": 1 - }, - "name": "kimik2.5" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 17620992, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 384, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 4, - "expert_parallel_size": 4, - "routed_tensor_parallel_size": 1, - "shared_tensor_parallel_size": 1, - "n_shared_experts": 1 - }, - "name": "kimik2.5" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 2151, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 384, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 4, - "expert_parallel_size": 4, - "routed_tensor_parallel_size": 1, - "shared_tensor_parallel_size": 1, - "n_shared_experts": 1 - }, - "name": "kimik2.5" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 18456576, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 384, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 4, - "expert_parallel_size": 4, - "routed_tensor_parallel_size": 1, - "shared_tensor_parallel_size": 1, - "n_shared_experts": 1 - }, - "name": "kimik2.5" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 262144, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 384, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 4, - "expert_parallel_size": 4, - "routed_tensor_parallel_size": 1, - "shared_tensor_parallel_size": 1, - "n_shared_experts": 1 - }, - "name": "kimik2.5" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 256, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 384, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 4, - "expert_parallel_size": 4, - "routed_tensor_parallel_size": 1, - "shared_tensor_parallel_size": 1, - "n_shared_experts": 1 - }, - "name": "kimik2.5" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 524288, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 384, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 4, - "expert_parallel_size": 4, - "routed_tensor_parallel_size": 1, - "shared_tensor_parallel_size": 1, - "n_shared_experts": 1 - }, - "name": "kimik2.5" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 512, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 384, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 4, - "expert_parallel_size": 4, - "routed_tensor_parallel_size": 1, - "shared_tensor_parallel_size": 1, - "n_shared_experts": 1 - }, - "name": "kimik2.5" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 1048576, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 384, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 4, - "expert_parallel_size": 4, - "routed_tensor_parallel_size": 1, - "shared_tensor_parallel_size": 1, - "n_shared_experts": 1 - }, - "name": "kimik2.5" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 1024, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 384, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 4, - "expert_parallel_size": 4, - "routed_tensor_parallel_size": 1, - "shared_tensor_parallel_size": 1, - "n_shared_experts": 1 - }, - "name": "kimik2.5" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 2097152, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 384, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 4, - "expert_parallel_size": 4, - "routed_tensor_parallel_size": 1, - "shared_tensor_parallel_size": 1, - "n_shared_experts": 1 - }, - "name": "kimik2.5" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 2048, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 384, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 4, - "expert_parallel_size": 4, - "routed_tensor_parallel_size": 1, - "shared_tensor_parallel_size": 1, - "n_shared_experts": 1 - }, - "name": "kimik2.5" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 3145728, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 384, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 4, - "expert_parallel_size": 4, - "routed_tensor_parallel_size": 1, - "shared_tensor_parallel_size": 1, - "n_shared_experts": 1 - }, - "name": "kimik2.5" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 3072, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 384, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 4, - "expert_parallel_size": 4, - "routed_tensor_parallel_size": 1, - "shared_tensor_parallel_size": 1, - "n_shared_experts": 1 - }, - "name": "kimik2.5" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 4194304, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 384, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 4, - "expert_parallel_size": 4, - "routed_tensor_parallel_size": 1, - "shared_tensor_parallel_size": 1, - "n_shared_experts": 1 - }, - "name": "kimik2.5" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 8192, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 384, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 4, - "expert_parallel_size": 4, - "routed_tensor_parallel_size": 1, - "shared_tensor_parallel_size": 1, - "n_shared_experts": 1 - }, - "name": "kimik2.5" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 8, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 384, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 4, - "expert_parallel_size": 4, - "routed_tensor_parallel_size": 1, - "shared_tensor_parallel_size": 1, - "n_shared_experts": 1 - }, - "name": "kimik2.5" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 16384, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 384, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 4, - "expert_parallel_size": 4, - "routed_tensor_parallel_size": 1, - "shared_tensor_parallel_size": 1, - "n_shared_experts": 1 - }, - "name": "kimik2.5" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 16, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 384, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 4, - "expert_parallel_size": 4, - "routed_tensor_parallel_size": 1, - "shared_tensor_parallel_size": 1, - "n_shared_experts": 1 - }, - "name": "kimik2.5" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 32, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 384, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 4, - "expert_parallel_size": 4, - "routed_tensor_parallel_size": 1, - "shared_tensor_parallel_size": 1, - "n_shared_experts": 1 - }, - "name": "kimik2.5" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 65536, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 384, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 4, - "expert_parallel_size": 4, - "routed_tensor_parallel_size": 1, - "shared_tensor_parallel_size": 1, - "n_shared_experts": 1 - }, - "name": "kimik2.5" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 64, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 384, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 4, - "expert_parallel_size": 4, - "routed_tensor_parallel_size": 1, - "shared_tensor_parallel_size": 1, - "n_shared_experts": 1 - }, - "name": "kimik2.5" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 131072, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 384, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 4, - "expert_parallel_size": 4, - "routed_tensor_parallel_size": 1, - "shared_tensor_parallel_size": 1, - "n_shared_experts": 1 - }, - "name": "kimik2.5" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 128, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 384, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 4, - "expert_parallel_size": 4, - "routed_tensor_parallel_size": 1, - "shared_tensor_parallel_size": 1, - "n_shared_experts": 1 - }, - "name": "kimik2.5" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 8388608, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 384, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 4, - "expert_parallel_size": 4, - "routed_tensor_parallel_size": 1, - "shared_tensor_parallel_size": 1, - "n_shared_experts": 1 - }, - "name": "kimik2.5" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 16777216, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 384, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 4, - "expert_parallel_size": 4, - "routed_tensor_parallel_size": 1, - "shared_tensor_parallel_size": 1, - "n_shared_experts": 1 - }, - "name": "kimik2.5" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 25165824, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 384, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 4, - "expert_parallel_size": 4, - "routed_tensor_parallel_size": 1, - "shared_tensor_parallel_size": 1, - "n_shared_experts": 1 - }, - "name": "kimik2.5" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 33554432, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 384, - "top_k": 8, - "dtype_act": "nvfp4", - "dtype_weight": "nvfp4", - "world_size": 4, - "expert_parallel_size": 4, - "routed_tensor_parallel_size": 1, - "shared_tensor_parallel_size": 1, - "n_shared_experts": 1 - }, - "name": "kimik2.5" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 1572864, - "hidden": 7168, - "intermediate": 3072, - "num_experts": 384, - "top_k": 6, - "dtype_act": "nvfp4", - "dtype_weight": "mxfp4", - "world_size": 4, - "expert_parallel_size": 4, - "routed_tensor_parallel_size": 1, - "shared_tensor_parallel_size": 1, - "n_shared_experts": 1 - }, - "name": "dsv4" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 192, - "hidden": 7168, - "intermediate": 3072, - "num_experts": 384, - "top_k": 6, - "dtype_act": "nvfp4", - "dtype_weight": "mxfp4", - "world_size": 4, - "expert_parallel_size": 4, - "routed_tensor_parallel_size": 1, - "shared_tensor_parallel_size": 1, - "n_shared_experts": 1 - }, - "name": "dsv4" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 147456, - "hidden": 7168, - "intermediate": 3072, - "num_experts": 384, - "top_k": 6, - "dtype_act": "nvfp4", - "dtype_weight": "mxfp4", - "world_size": 4, - "expert_parallel_size": 4, - "routed_tensor_parallel_size": 1, - "shared_tensor_parallel_size": 1, - "n_shared_experts": 1 - }, - "name": "dsv4" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 18, - "hidden": 7168, - "intermediate": 3072, - "num_experts": 384, - "top_k": 6, - "dtype_act": "nvfp4", - "dtype_weight": "mxfp4", - "world_size": 4, - "expert_parallel_size": 4, - "routed_tensor_parallel_size": 1, - "shared_tensor_parallel_size": 1, - "n_shared_experts": 1 - }, - "name": "dsv4" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 33554432, - "hidden": 7168, - "intermediate": 3072, - "num_experts": 384, - "top_k": 6, - "dtype_act": "nvfp4", - "dtype_weight": "mxfp4", - "world_size": 4, - "expert_parallel_size": 4, - "routed_tensor_parallel_size": 1, - "shared_tensor_parallel_size": 1, - "n_shared_experts": 1 - }, - "name": "dsv4" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 4096, - "hidden": 7168, - "intermediate": 3072, - "num_experts": 384, - "top_k": 6, - "dtype_act": "nvfp4", - "dtype_weight": "mxfp4", - "world_size": 4, - "expert_parallel_size": 4, - "routed_tensor_parallel_size": 1, - "shared_tensor_parallel_size": 1, - "n_shared_experts": 1 - }, - "name": "dsv4" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 25165824, - "hidden": 7168, - "intermediate": 3072, - "num_experts": 384, - "top_k": 6, - "dtype_act": "nvfp4", - "dtype_weight": "mxfp4", - "world_size": 4, - "expert_parallel_size": 4, - "routed_tensor_parallel_size": 1, - "shared_tensor_parallel_size": 1, - "n_shared_experts": 1 - }, - "name": "dsv4" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 3072, - "hidden": 7168, - "intermediate": 3072, - "num_experts": 384, - "top_k": 6, - "dtype_act": "nvfp4", - "dtype_weight": "mxfp4", - "world_size": 4, - "expert_parallel_size": 4, - "routed_tensor_parallel_size": 1, - "shared_tensor_parallel_size": 1, - "n_shared_experts": 1 - }, - "name": "dsv4" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 8388608, - "hidden": 7168, - "intermediate": 3072, - "num_experts": 384, - "top_k": 6, - "dtype_act": "nvfp4", - "dtype_weight": "mxfp4", - "world_size": 4, - "expert_parallel_size": 4, - "routed_tensor_parallel_size": 1, - "shared_tensor_parallel_size": 1, - "n_shared_experts": 1 - }, - "name": "dsv4" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 1024, - "hidden": 7168, - "intermediate": 3072, - "num_experts": 384, - "top_k": 6, - "dtype_act": "nvfp4", - "dtype_weight": "mxfp4", - "world_size": 4, - "expert_parallel_size": 4, - "routed_tensor_parallel_size": 1, - "shared_tensor_parallel_size": 1, - "n_shared_experts": 1 - }, - "name": "dsv4" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 67108864, - "hidden": 7168, - "intermediate": 3072, - "num_experts": 384, - "top_k": 6, - "dtype_act": "nvfp4", - "dtype_weight": "mxfp4", - "world_size": 4, - "expert_parallel_size": 4, - "routed_tensor_parallel_size": 1, - "shared_tensor_parallel_size": 1, - "n_shared_experts": 1 - }, - "name": "dsv4" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 8192, - "hidden": 7168, - "intermediate": 3072, - "num_experts": 384, - "top_k": 6, - "dtype_act": "nvfp4", - "dtype_weight": "mxfp4", - "world_size": 4, - "expert_parallel_size": 4, - "routed_tensor_parallel_size": 1, - "shared_tensor_parallel_size": 1, - "n_shared_experts": 1 - }, - "name": "dsv4" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 176160768, - "hidden": 7168, - "intermediate": 3072, - "num_experts": 384, - "top_k": 6, - "dtype_act": "nvfp4", - "dtype_weight": "mxfp4", - "world_size": 4, - "expert_parallel_size": 4, - "routed_tensor_parallel_size": 1, - "shared_tensor_parallel_size": 1, - "n_shared_experts": 1 - }, - "name": "dsv4" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 21504, - "hidden": 7168, - "intermediate": 3072, - "num_experts": 384, - "top_k": 6, - "dtype_act": "nvfp4", - "dtype_weight": "mxfp4", - "world_size": 4, - "expert_parallel_size": 4, - "routed_tensor_parallel_size": 1, - "shared_tensor_parallel_size": 1, - "n_shared_experts": 1 - }, - "name": "dsv4" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 128, - "hidden": 3072, - "intermediate": 1536, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "world_size": 4, - "expert_parallel_size": 4, - "routed_tensor_parallel_size": 1, - "shared_tensor_parallel_size": 1, - "n_shared_experts": 0 - }, - "name": "minimaxm2.5" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 1048576, - "hidden": 3072, - "intermediate": 1536, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "world_size": 4, - "expert_parallel_size": 4, - "routed_tensor_parallel_size": 1, - "shared_tensor_parallel_size": 1, - "n_shared_experts": 0 - }, - "name": "minimaxm2.5" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 2097152, - "hidden": 3072, - "intermediate": 1536, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "world_size": 4, - "expert_parallel_size": 4, - "routed_tensor_parallel_size": 1, - "shared_tensor_parallel_size": 1, - "n_shared_experts": 0 - }, - "name": "minimaxm2.5" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 4194304, - "hidden": 3072, - "intermediate": 1536, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "fp8", - "world_size": 4, - "expert_parallel_size": 4, - "routed_tensor_parallel_size": 1, - "shared_tensor_parallel_size": 1, - "n_shared_experts": 0 - }, - "name": "minimaxm2.5" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 1048576, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "mxfp4", - "world_size": 4, - "expert_parallel_size": 4, - "routed_tensor_parallel_size": 1, - "shared_tensor_parallel_size": 1, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 1024, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "mxfp4", - "world_size": 4, - "expert_parallel_size": 4, - "routed_tensor_parallel_size": 1, - "shared_tensor_parallel_size": 1, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 2097152, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "mxfp4", - "world_size": 4, - "expert_parallel_size": 4, - "routed_tensor_parallel_size": 1, - "shared_tensor_parallel_size": 1, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 2048, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "mxfp4", - "world_size": 4, - "expert_parallel_size": 4, - "routed_tensor_parallel_size": 1, - "shared_tensor_parallel_size": 1, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 4194304, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "mxfp4", - "world_size": 4, - "expert_parallel_size": 4, - "routed_tensor_parallel_size": 1, - "shared_tensor_parallel_size": 1, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 4096, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "mxfp4", - "world_size": 4, - "expert_parallel_size": 4, - "routed_tensor_parallel_size": 1, - "shared_tensor_parallel_size": 1, - "n_shared_experts": 1 - }, - "name": "dsr1" - }, - { - "type": "moe_forward", - "args": { - "num_tokens": 8192, - "hidden": 7168, - "intermediate": 2048, - "num_experts": 256, - "top_k": 8, - "dtype_act": "fp8", - "dtype_weight": "mxfp4", - "world_size": 4, - "expert_parallel_size": 4, - "routed_tensor_parallel_size": 1, - "shared_tensor_parallel_size": 1, - "n_shared_experts": 1 - }, - "name": "dsr1" - } -] \ No newline at end of file diff --git a/operatorx/testlists/smoke_parallel.json b/operatorx/testlists/smoke_parallel.json new file mode 100644 index 0000000000..9487e31364 --- /dev/null +++ b/operatorx/testlists/smoke_parallel.json @@ -0,0 +1,2083 @@ +[ + { + "type": "gemm", + "args": { + "m": 8, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + }, + "k": 3072, + "n": 6144, + "out": "bf16", + "parallel": { + "tp": 8 + } + }, + "sources": [ + "MiniMaxAI/MiniMax-M3-MXFP8/block_sparse_moe.shared_experts.down_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 8, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + }, + "k": 3072, + "n": 6144, + "out": "bf16", + "parallel": { + "tp": 2 + } + }, + "sources": [ + "nvidia/MiniMax-M3-NVFP4/block_sparse_moe.shared_experts.down_proj" + ] + }, + { + "type": "gemm", + "args": { + "m": 8, + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + }, + "k": 3072, + "n": 6144, + "out": "bf16", + "parallel": { + "tp": 4 + } + }, + "sources": [ + "nvidia/MiniMax-M3-NVFP4/block_sparse_moe.shared_experts.down_proj" + ] + }, + { + "type": "moe", + "args": { + "tokens": 8, + "activation": { + "alpha": 1.702, + "beta": 1.0, + "kind": "swigluoai", + "limit": 7.0 + }, + "experts": { + "a1": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "a2": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "inter": 3072, + "num": 128, + "top_k": 4, + "w1": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + }, + "w2": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + } + }, + "hidden": 6144, + "router": { + "bias": true, + "gate": { + "dtype": "fp32" + }, + "renormalize": true, + "scale": 2.0, + "scoring": "sigmoid", + "select": { + "kind": "topk" + } + }, + "routing": { + "distribution": "natural", + "seed": 0 + }, + "shared": { + "a1": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "count": 1, + "inter": 3072, + "w1": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + }, + "w2": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + } + }, + "parallel": { + "tp": 8 + } + }, + "sources": [ + "MiniMaxAI/MiniMax-M3-MXFP8/block_sparse_moe" + ] + }, + { + "type": "moe", + "args": { + "tokens": 8, + "activation": { + "alpha": 1.702, + "beta": 1.0, + "kind": "swigluoai", + "limit": 7.0 + }, + "experts": { + "a1": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "group": [ + 1, + 16 + ], + "static": false + }, + "scale2": { + "dtype": "fp32", + "group": [ + -1, + -1 + ], + "static": true + } + }, + "a2": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "group": [ + 1, + 16 + ], + "static": false + }, + "scale2": { + "dtype": "fp32", + "group": [ + -1, + -1 + ], + "static": true + } + }, + "inter": 3072, + "num": 128, + "top_k": 4, + "w1": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "group": [ + 1, + 16 + ], + "static": true + }, + "scale2": { + "dtype": "fp32", + "group": [ + -1, + -1 + ], + "static": true + } + }, + "w2": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "group": [ + 1, + 16 + ], + "static": true + }, + "scale2": { + "dtype": "fp32", + "group": [ + -1, + -1 + ], + "static": true + } + } + }, + "hidden": 6144, + "router": { + "bias": true, + "gate": { + "dtype": "fp32" + }, + "renormalize": true, + "scale": 2.0, + "scoring": "sigmoid", + "select": { + "kind": "topk" + } + }, + "routing": { + "distribution": "natural", + "seed": 0 + }, + "shared": { + "a1": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "count": 1, + "inter": 3072, + "w1": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + }, + "w2": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + } + }, + "parallel": { + "tp": 2 + } + }, + "sources": [ + "nvidia/MiniMax-M3-NVFP4/block_sparse_moe" + ] + }, + { + "type": "moe", + "args": { + "tokens": 8, + "activation": { + "alpha": 1.702, + "beta": 1.0, + "kind": "swigluoai", + "limit": 7.0 + }, + "experts": { + "a1": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "group": [ + 1, + 16 + ], + "static": false + }, + "scale2": { + "dtype": "fp32", + "group": [ + -1, + -1 + ], + "static": true + } + }, + "a2": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "group": [ + 1, + 16 + ], + "static": false + }, + "scale2": { + "dtype": "fp32", + "group": [ + -1, + -1 + ], + "static": true + } + }, + "inter": 3072, + "num": 128, + "top_k": 4, + "w1": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "group": [ + 1, + 16 + ], + "static": true + }, + "scale2": { + "dtype": "fp32", + "group": [ + -1, + -1 + ], + "static": true + } + }, + "w2": { + "dtype": "e2m1", + "scale": { + "dtype": "e4m3", + "group": [ + 1, + 16 + ], + "static": true + }, + "scale2": { + "dtype": "fp32", + "group": [ + -1, + -1 + ], + "static": true + } + } + }, + "hidden": 6144, + "router": { + "bias": true, + "gate": { + "dtype": "fp32" + }, + "renormalize": true, + "scale": 2.0, + "scoring": "sigmoid", + "select": { + "kind": "topk" + } + }, + "routing": { + "distribution": "natural", + "seed": 0 + }, + "shared": { + "a1": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "count": 1, + "inter": 3072, + "w1": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + }, + "w2": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + } + }, + "parallel": { + "tp": 4 + } + }, + "sources": [ + "nvidia/MiniMax-M3-NVFP4/block_sparse_moe" + ] + }, + { + "type": "moe", + "args": { + "tokens": 8, + "activation": { + "kind": "silu", + "limit": 10.0 + }, + "experts": { + "a1": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "a2": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "inter": 3072, + "num": 384, + "top_k": 6, + "w1": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + }, + "w2": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + } + }, + "hidden": 7168, + "router": { + "bias": true, + "gate": { + "dtype": "bf16", + "logits": "fp32" + }, + "renormalize": true, + "scale": 2.5, + "scoring": "sqrtsoftplus", + "select": { + "kind": "topk" + } + }, + "routing": { + "distribution": "natural", + "seed": 0 + }, + "shared": { + "a1": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 128 + ], + "static": false + } + }, + "count": 1, + "inter": 3072, + "w1": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + }, + "w2": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + } + }, + "parallel": { + "dp": 4, + "ep": 4 + } + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro/ffn" + ] + }, + { + "type": "moe", + "args": { + "tokens": 8, + "activation": { + "kind": "silu", + "limit": 10.0 + }, + "experts": { + "a1": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "a2": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "inter": 3072, + "num": 384, + "top_k": 6, + "w1": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + }, + "w2": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + } + }, + "hidden": 7168, + "router": { + "bias": true, + "gate": { + "dtype": "bf16", + "logits": "fp32" + }, + "renormalize": true, + "scale": 2.5, + "scoring": "sqrtsoftplus", + "select": { + "kind": "topk" + } + }, + "routing": { + "distribution": "natural", + "seed": 0 + }, + "shared": { + "a1": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 128 + ], + "static": false + } + }, + "count": 1, + "inter": 3072, + "w1": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + }, + "w2": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + } + }, + "parallel": { + "dp": 8, + "ep": 8 + } + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro/ffn" + ] + }, + { + "type": "moe", + "args": { + "tokens": 8, + "activation": { + "kind": "silu", + "limit": 10.0 + }, + "experts": { + "a1": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "a2": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "inter": 3072, + "num": 384, + "top_k": 6, + "w1": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + }, + "w2": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + } + }, + "hidden": 7168, + "router": { + "bias": true, + "gate": { + "dtype": "bf16", + "logits": "fp32" + }, + "renormalize": true, + "scale": 2.5, + "scoring": "sqrtsoftplus", + "select": { + "kind": "topk" + } + }, + "routing": { + "distribution": "natural", + "seed": 0 + }, + "shared": { + "a1": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 128 + ], + "static": false + } + }, + "count": 1, + "inter": 3072, + "w1": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + }, + "w2": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + } + }, + "parallel": { + "dp": 8 + } + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro-0813/ffn" + ] + }, + { + "type": "moe", + "args": { + "tokens": 8, + "activation": { + "kind": "silu", + "limit": 10.0 + }, + "experts": { + "a1": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "a2": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "inter": 2304, + "num": 384, + "top_k": 6, + "w1": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + }, + "w2": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + } + }, + "hidden": 5120, + "router": { + "bias": true, + "gate": { + "dtype": "bf16", + "logits": "fp32" + }, + "renormalize": true, + "scale": 1.5, + "scoring": "sqrtsoftplus", + "select": { + "kind": "topk" + } + }, + "routing": { + "distribution": "natural", + "seed": 0 + }, + "shared": { + "a1": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "count": 1, + "inter": 2304, + "w1": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + }, + "w2": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + }, + "parallel": { + "tp": 2, + "ep": 2 + } + }, + "sources": [ + "deepseek-ai/DeepSeek-V4.1-Flash/ffn" + ] + }, + { + "type": "moe", + "args": { + "tokens": 8, + "activation": { + "kind": "silu", + "limit": 10.0 + }, + "experts": { + "a1": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "a2": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "inter": 2304, + "num": 384, + "top_k": 6, + "w1": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + }, + "w2": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + } + }, + "hidden": 5120, + "router": { + "bias": true, + "gate": { + "dtype": "bf16", + "logits": "fp32" + }, + "renormalize": true, + "scale": 1.5, + "scoring": "sqrtsoftplus", + "select": { + "kind": "topk" + } + }, + "routing": { + "distribution": "natural", + "seed": 0 + }, + "shared": { + "a1": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "count": 1, + "inter": 2304, + "w1": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + }, + "w2": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + }, + "parallel": { + "tp": 4, + "ep": 4 + } + }, + "sources": [ + "deepseek-ai/DeepSeek-V4.1-Flash/ffn" + ] + }, + { + "type": "moe", + "args": { + "tokens": 8, + "activation": { + "kind": "silu", + "limit": 10.0 + }, + "experts": { + "a1": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "a2": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "inter": 2304, + "num": 384, + "top_k": 6, + "w1": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + }, + "w2": { + "dtype": "e2m1", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": true + } + } + }, + "hidden": 5120, + "router": { + "bias": true, + "gate": { + "dtype": "bf16", + "logits": "fp32" + }, + "renormalize": true, + "scale": 1.5, + "scoring": "sqrtsoftplus", + "select": { + "kind": "topk" + } + }, + "routing": { + "distribution": "natural", + "seed": 0 + }, + "shared": { + "a1": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "count": 1, + "inter": 2304, + "w1": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + }, + "w2": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + }, + "parallel": { + "tp": 8, + "ep": 8 + } + }, + "sources": [ + "deepseek-ai/DeepSeek-V4.1-Flash/ffn" + ] + }, + { + "type": "dsv4_attn", + "args": { + "hidden": 7168, + "heads": 128, + "head_dim": 512, + "rope_dim": 64, + "q_lora_rank": 1536, + "window": 128, + "o_groups": 16, + "o_lora_rank": 1024, + "compress_rope_theta": 160000, + "rope_scaling": { + "type": "yarn", + "factor": 16, + "original_max": 65536, + "beta_fast": 32, + "beta_slow": 1 + }, + "kv_cache_dtype": "fp8", + "compress_ratio": 4, + "indexer": "own", + "topk": 1024, + "index_heads": 64, + "index_dim": 128, + "index_cache_dtype": "fp8", + "proj": { + "indexer.wq_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 128 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + } + }, + "wkv": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 128 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + } + }, + "wo_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 128 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + } + }, + "wq_a": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 128 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + } + }, + "wq_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 128 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + } + } + }, + "batch": { + "groups": [ + { + "count": 16, + "q": 1, + "ctx": 16384 + } + ] + }, + "parallel": { + "tp": 4 + } + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro/attn" + ] + }, + { + "type": "dsv4_attn", + "args": { + "hidden": 7168, + "heads": 128, + "head_dim": 512, + "rope_dim": 64, + "q_lora_rank": 1536, + "window": 128, + "o_groups": 16, + "o_lora_rank": 1024, + "compress_rope_theta": 160000, + "rope_scaling": { + "type": "yarn", + "factor": 16, + "original_max": 65536, + "beta_fast": 32, + "beta_slow": 1 + }, + "kv_cache_dtype": "fp8", + "compress_ratio": 4, + "indexer": "own", + "topk": 1024, + "index_heads": 64, + "index_dim": 128, + "index_cache_dtype": "fp8", + "proj": { + "indexer.wq_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 128 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + } + }, + "wkv": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 128 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + } + }, + "wo_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 128 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + } + }, + "wq_a": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 128 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + } + }, + "wq_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 128 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + } + } + }, + "batch": { + "groups": [ + { + "count": 16, + "q": 1, + "ctx": 16384 + } + ] + }, + "parallel": { + "tp": 8 + } + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro/attn" + ] + }, + { + "type": "dsv4_attn", + "args": { + "hidden": 7168, + "heads": 128, + "head_dim": 512, + "rope_dim": 64, + "q_lora_rank": 1536, + "window": 128, + "o_groups": 16, + "o_lora_rank": 1024, + "compress_rope_theta": 160000, + "rope_scaling": { + "type": "yarn", + "factor": 16, + "original_max": 65536, + "beta_fast": 32, + "beta_slow": 1 + }, + "kv_cache_dtype": "fp8", + "compress_ratio": 4, + "indexer": "own", + "topk": 1024, + "index_heads": 64, + "index_dim": 128, + "index_cache_dtype": "fp8", + "proj": { + "indexer.wq_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 128 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + } + }, + "wkv": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 128 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + } + }, + "wo_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 128 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + } + }, + "wq_a": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 128 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + } + }, + "wq_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 128 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + } + } + }, + "batch": { + "groups": [ + { + "count": 16, + "q": 1, + "ctx": 16384 + } + ] + }, + "parallel": { + "dp": 4 + } + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro/attn" + ] + }, + { + "type": "dsv4_attn", + "args": { + "hidden": 7168, + "heads": 128, + "head_dim": 512, + "rope_dim": 64, + "q_lora_rank": 1536, + "window": 128, + "o_groups": 16, + "o_lora_rank": 1024, + "compress_rope_theta": 160000, + "rope_scaling": { + "type": "yarn", + "factor": 16, + "original_max": 65536, + "beta_fast": 32, + "beta_slow": 1 + }, + "kv_cache_dtype": "fp8", + "compress_ratio": 4, + "indexer": "own", + "topk": 1024, + "index_heads": 64, + "index_dim": 128, + "index_cache_dtype": "fp8", + "proj": { + "indexer.wq_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 128 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + } + }, + "wkv": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 128 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + } + }, + "wo_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 128 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + } + }, + "wq_a": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 128 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + } + }, + "wq_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 128 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 128, + 128 + ], + "static": true + } + } + } + }, + "batch": { + "groups": [ + { + "count": 16, + "q": 1, + "ctx": 16384 + } + ] + }, + "parallel": { + "dp": 8 + } + }, + "sources": [ + "deepseek-ai/DeepSeek-V4-Pro/attn" + ] + }, + { + "type": "dsv4_attn", + "args": { + "hidden": 5120, + "heads": 64, + "head_dim": 512, + "rope_dim": 64, + "q_lora_rank": 1280, + "window": 128, + "o_groups": 8, + "o_lora_rank": 1024, + "compress_ratio": 0, + "proj": { + "wkv": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + }, + "wo_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + }, + "wq_a": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + }, + "wq_b": { + "a": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 1, + 32 + ], + "static": false + } + }, + "b": { + "dtype": "e4m3", + "scale": { + "dtype": "ue8m0", + "group": [ + 32, + 32 + ], + "static": true + } + } + } + }, + "batch": { + "groups": [ + { + "count": 16, + "q": 1, + "ctx": 16384 + } + ] + }, + "parallel": { + "tp": 2 + } + }, + "sources": [ + "deepseek-ai/DeepSeek-V4.1-Flash/attn" + ] + }, + { + "type": "mla", + "args": { + "hidden": 7168, + "heads": 96, + "q_lora_rank": 1536, + "kv_lora_rank": 512, + "nope": 128, + "rope_dim": 64, + "v": 128, + "rope": false, + "gate": true, + "kv_cache_dtype": "fp8", + "batch": { + "groups": [ + { + "count": 16, + "q": 1, + "ctx": 16384 + } + ] + }, + "parallel": { + "tp": 8, + "dcp": 8 + } + }, + "sources": [ + "moonshotai/Kimi-K3/self_attn" + ] + } +] \ No newline at end of file diff --git a/operatorx/tests/test_amd.py b/operatorx/tests/test_amd.py deleted file mode 100644 index 1990937db5..0000000000 --- a/operatorx/tests/test_amd.py +++ /dev/null @@ -1,129 +0,0 @@ -"""ROCm orchestration with real CPU tensor math and substituted GPU APIs.""" - -import sys -from pathlib import Path -from types import SimpleNamespace - -import pytest -import torch - -sys.path.insert(0, str(Path(__file__).resolve().parents[2])) -from operatorx.core import Op, UnsupportedOpError -from operatorx.runners.amd import runner -from operatorx.runners.amd.backends import torch as backend - - -def gemm(**overrides): - return Op( - "gemm", - { - "m": 2, - "n": 2, - "k": 2, - "dtype_a": "fp32", - "dtype_b": "fp32", - "dtype_out": "fp32", - **overrides, - }, - "torch", - ) - - -@pytest.mark.parametrize( - "bias,expected", - [(None, [[19.0, 22.0], [43.0, 50.0]]), ([3.0, 5.0], [[22.0, 27.0], [46.0, 55.0]])], -) -def test_gemm_computes_matrix_product_and_fused_bias(bias, expected): - ctx = { - "A": torch.tensor([[1.0, 2.0], [3.0, 4.0]]), - "B": torch.tensor([[5.0, 6.0], [7.0, 8.0]]), - "bias": None if bias is None else torch.tensor(bias), - "fp8": False, - } - backend.kernel(ctx) - torch.testing.assert_close(ctx["C"], torch.tensor(expected)) - - -def cpu_allocations(monkeypatch): - # Substitute the GPU allocator; tensor construction and math remain real. - for name in ("randn", "tensor", "empty"): - original = getattr(torch, name) - - def allocate(*args, _original=original, **kwargs): - kwargs["device"] = "cpu" - return _original(*args, **kwargs) - - monkeypatch.setattr(torch, name, allocate) - - -@pytest.mark.parametrize( - "arch,dtype", - [ - ("gfx942:sramecc+:xnack-", torch.float8_e4m3fnuz), - ("gfx950", torch.float8_e4m3fn), - ], -) -def test_fp8_preparation_selects_architecture_and_column_major_rhs( - monkeypatch, arch, dtype -): - cpu_allocations(monkeypatch) - monkeypatch.setattr(torch.cuda, "current_device", lambda: 0) - monkeypatch.setattr( - torch.cuda, "get_device_properties", lambda _: SimpleNamespace(gcnArchName=arch) - ) - ctx = backend.prepare( - gemm(m=2, n=3, k=4, dtype_a="fp8", dtype_b="fp8", dtype_out="bf16") - ) - assert ctx["A"].shape == (2, 4) - assert ctx["B"].shape == (4, 3) - assert ctx["B"].stride() == (1, 4) - assert ctx["A"].dtype == ctx["B"].dtype == dtype - assert ctx["scale_a"].item() == ctx["scale_b"].item() == 1.0 - assert ctx["out_dtype"] == torch.bfloat16 - - -@pytest.mark.parametrize( - "args", - [ - {"activation": "relu"}, - {"dtype_b": "bf16"}, - {"dtype_a": "nvfp4", "dtype_b": "nvfp4"}, - {"dtype_out": "fp8"}, - {"dtype_out": "bf16"}, - ], -) -def test_unsupported_gemm_requests_raise_before_gpu_allocation(args): - with pytest.raises(UnsupportedOpError): - backend.prepare(gemm(**args)) - - -def test_timing_converts_hip_events_to_upper_median_microseconds(monkeypatch): - cpu_allocations(monkeypatch) - monkeypatch.setattr(torch.version, "hip", "test-rocm") - monkeypatch.setattr(torch.cuda, "current_device", lambda: 0) - monkeypatch.setattr( - torch.cuda, "get_device_properties", lambda _: SimpleNamespace(L2_cache_size=16) - ) - monkeypatch.setattr(torch.cuda, "synchronize", lambda: None) - milliseconds = iter([9, 3, 4, 1, 8, 2, 7, 10, 5, 6]) - - class Event: - def __init__(self, **kwargs): - pass - - def record(self): - pass - - def elapsed_time(self, end): - return next(milliseconds) - - monkeypatch.setattr(torch.cuda, "Event", Event) - monkeypatch.setattr(runner, "_L2_BUF", {}) - result = runner.run(gemm()) - assert result.metrics == {"latency_us": 6000.0} - - -def test_cpu_build_cannot_emit_amd_gpu_measurement(monkeypatch): - monkeypatch.setattr(torch.version, "hip", None) - with pytest.raises(RuntimeError, match="ROCm PyTorch"): - runner.run(gemm()) diff --git a/operatorx/tests/test_attention.py b/operatorx/tests/test_attention.py deleted file mode 100644 index 15669c4d2a..0000000000 --- a/operatorx/tests/test_attention.py +++ /dev/null @@ -1,129 +0,0 @@ -"""Exercise real attention preparation and SDPA math with CPU tensors.""" - -import sys -from pathlib import Path - -import pytest -import torch - -sys.path.insert(0, str(Path(__file__).resolve().parents[2])) -from operatorx.core import Op, UnsupportedOpError -from operatorx.runners import attention - - -def mha(**overrides): - return Op( - "attention_mha", - { - "batch_size": 1, - "seq_len_q": 2, - "seq_len_kv": 5, - "num_heads": 4, - "num_heads_kv": 2, - "head_dim": 8, - "dtype_q": "fp32", - "dtype_k": "fp32", - "dtype_v": "fp32", - "dtype_o": "fp32", - "causal": True, - **overrides, - }, - "torch", - ) - - -@pytest.fixture -def cpu(monkeypatch): - for name in ("randn", "ones"): - original = getattr(torch, name) - - def allocate(*args, _original=original, **kwargs): - kwargs["device"] = "cpu" - return _original(*args, **kwargs) - - monkeypatch.setattr(torch, name, allocate) - - -@pytest.mark.parametrize( - "sq,sk,causal,expected", - [ - (2, 5, True, [4.0, 5.0]), - (1, 5, True, [5.0]), - (2, 2, True, [1.0, 2.0]), - (2, 5, False, [5.0, 5.0]), - (3, 2, True, [0.0, 1.0, 2.0]), - ], -) -def test_causal_decode_alignment(cpu, sq, sk, causal, expected): - ctx = attention.prepare(mha(seq_len_q=sq, seq_len_kv=sk, causal=causal)) - ctx["q"].zero_() - ctx["k"].zero_() - ctx["v"].copy_(torch.arange(1, 2 * sk, 2).reshape(1, 1, sk, 1)) - attention.kernel(ctx) - torch.testing.assert_close(ctx["out"][0, 0, :, 0], torch.tensor(expected)) - assert ctx["out"].shape == (1, 4, sq, 8) - - -def test_gqa_keeps_each_kv_head_and_defaults_output_to_bf16(cpu): - op = mha( - seq_len_q=1, seq_len_kv=1, dtype_q="bf16", dtype_k="bf16", dtype_v="bf16" - ) - args = dict(op.args) - del args["dtype_o"] - ctx = attention.prepare(Op(op.type, args, op.backend)) - # Preparation repeats head 0 for query heads 0/1, head 1 for heads 2/3. - torch.testing.assert_close(ctx["v"][:, 0], ctx["v"][:, 1]) - torch.testing.assert_close(ctx["v"][:, 2], ctx["v"][:, 3]) - ctx["q"].zero_() - ctx["k"].zero_() - ctx["v"][:, :2].fill_(3) - ctx["v"][:, 2:].fill_(7) - attention.kernel(ctx) - assert ctx["out"].dtype == torch.bfloat16 - assert ctx["out"][0, :, 0, 0].tolist() == [3, 3, 7, 7] - - -def test_mla_preserves_value_dimension_and_bshd_layout(cpu): - op = Op( - "attention_mla", - { - "batch_size": 1, - "seq_len_q": 2, - "seq_len_kv": 3, - "num_heads": 2, - "head_dim_qk_nope": 8, - "head_dim_qk_rope": 8, - "head_dim_v": 4, - "kv_lora_rank": 32, - "dtype_q": "fp32", - "dtype_kv": "fp32", - "dtype_o": "fp32", - }, - "torch", - ) - ctx = attention.prepare(op) - ctx["q"].zero_() - ctx["k"].zero_() - ctx["v"].fill_(6) - attention.kernel(ctx) - torch.testing.assert_close(ctx["out"], torch.full((1, 2, 2, 4), 6.0)) - packed = attention.prepare(op, layout="bshd") - assert packed["k"].shape == (1, 3, 2, 16) - assert packed["v"].shape == (1, 3, 2, 4) - assert packed["causal"] is True - - -@pytest.mark.parametrize( - "overrides,error", - [ - ({"dtype_o": "bf16"}, UnsupportedOpError), - ({"dtype_q": "fp8"}, UnsupportedOpError), - ({"kv_layout": "paged"}, UnsupportedOpError), - ({"sliding_window": 32}, UnsupportedOpError), - ({"num_heads": 3}, ValueError), - ({"num_heads_kv": 0}, ValueError), - ], -) -def test_invalid_attention_never_reaches_gpu(overrides, error): - with pytest.raises(error): - attention.prepare(mha(**overrides)) diff --git a/operatorx/tests/test_ci.py b/operatorx/tests/test_ci.py index 8f76888af2..3e3eafb341 100644 --- a/operatorx/tests/test_ci.py +++ b/operatorx/tests/test_ci.py @@ -17,28 +17,28 @@ from operatorx.core import Result, UnsupportedOpError -def platforms(pool="h100-dgxc", gpus=8, architecture="linux/amd64"): - return {pool: {"gpus_per_node": gpus, "image_platform": architecture}} +def platforms(runner="cluster:h100-dsxe", gpus=8, architecture="linux/amd64"): + return {runner: {"gpus_per_node": gpus, "image_platform": architecture}} -def test_plan_chunks_and_preserves_moe_groups(): +def test_plan_chunks_by_world_size(): ordinary = {"type": "gemm", "args": {"m": 2}} - moe = {"type": "moe_forward", "args": {"world_size": 4, "expert_parallel_size": 2}} - multi = {"type": "allreduce", "args": {"world_size": 16}} + wide = {"type": "gemm", "args": {"m": 2, "parallel": {"tp": 4}}} + multi = {"type": "gemm", "args": {"m": 2, "parallel": {"tp": 16}}} result = ci.plan( - "h100-dgxc", + "cluster:h100-dsxe", ["a", "b"], - {"small": [ordinary, ordinary, moe, multi]}, + {"small": [ordinary, ordinary, wide, multi]}, {"a": {"image": "same:1"}, "b": {"image": "same:1"}}, [1, 4], 1, platforms(), ) cells = result["include"] - assert [(c["world_size"], c["moe"], len(c["cases"])) for c in cells] == [ - (1, (), 1), - (1, (), 1), - (4, (2, 1, 1), 1), + assert [(c["world_size"], len(c["cases"])) for c in cells] == [ + (1, 1), + (1, 1), + (4, 1), ] assert result["excluded_shapes"] == 1 assert cells[2]["backends"] == ["a", "b"] @@ -46,19 +46,19 @@ def test_plan_chunks_and_preserves_moe_groups(): @pytest.mark.parametrize( - "pool,backends,worlds,shapes,chunk", + "runner,backends,worlds,shapes,chunk", [ - ("b200", ["a"], [1], [{"type": "gemm", "args": {}}], 1), - ("h100-dgxc", ["missing"], [1], [{"type": "gemm", "args": {}}], 1), - ("h100-dgxc", ["a"], [16], [{"type": "gemm", "args": {}}], 1), - ("h100-dgxc", ["a"], [1], [], 1), - ("h100-dgxc", ["a"], [1], [{"type": "gemm", "args": {}}] * 257, 1), + ("cluster:b200", ["a"], [1], [{"type": "gemm", "args": {}}], 1), + ("cluster:h100-dsxe", ["missing"], [1], [{"type": "gemm", "args": {}}], 1), + ("cluster:h100-dsxe", ["a"], [16], [{"type": "gemm", "args": {}}], 1), + ("cluster:h100-dsxe", ["a"], [1], [], 1), + ("cluster:h100-dsxe", ["a"], [1], [{"type": "gemm", "args": {}}] * 257, 1), ], ) -def test_plan_rejects_unexecutable_selection(pool, backends, worlds, shapes, chunk): +def test_plan_rejects_unexecutable_selection(runner, backends, worlds, shapes, chunk): with pytest.raises(ValueError): ci.plan( - pool, + runner, backends, {"tiny": shapes}, {"a": {"image": "image:1"}}, @@ -72,12 +72,12 @@ def test_plan_bounds_world_size_to_physical_arm_node(): shapes = { "tiny": [ {"type": "gemm", "args": {"m": 2}}, - {"type": "allreduce", "args": {"world_size": 8}}, + {"type": "allreduce", "args": {"parallel": {"tp": 8}}}, ] } - hardware = platforms("gb200", 4, "linux/arm64") + hardware = platforms("cluster:gb200-nv", 4, "linux/arm64") result = ci.plan( - "gb200", ["torch"], shapes, {"torch": {"image": "image:1"}}, [1], 50, hardware + "cluster:gb200-nv", ["torch"], shapes, {"torch": {"image": "image:1"}}, [1], 50, hardware ) assert result["excluded_shapes"] == 1 assert result["include"][0]["cases"] == [ @@ -86,7 +86,7 @@ def test_plan_bounds_world_size_to_physical_arm_node(): assert result["include"][0]["image_platform"] == "linux/arm64" with pytest.raises(ValueError, match="4-GPU"): ci.plan( - "gb200", + "cluster:gb200-nv", ["torch"], shapes, {"torch": {"image": "image:1"}}, @@ -115,7 +115,7 @@ def external(argv, **kwargs): monkeypatch.setattr(ci.subprocess, "run", external) digest = "sha256:" + "a" * 64 - # Only registry traffic is replaced; use the production reference parser. + # only registry traffic is faked; the real reference parser runs probe = ci.probe_module() monkeypatch.setattr(probe, "resolve_image_digest", lambda image: digest) monkeypatch.setattr(ci, "probe_module", lambda: probe) @@ -137,7 +137,7 @@ def external(argv, **kwargs): args.image_platform = architecture monkeypatch.setattr(ci.platform, "machine", lambda: machine) ci.import_image(args) - ci.import_image(args) # Reuse a validated cache on the same architecture. + ci.import_image(args) # reuses the validated cache images = list(args.cache.glob("*.sqsh")) assert len(images) == 2 assert all(image.read_bytes() == b"validated squash fixture" for image in images) @@ -151,15 +151,15 @@ def test_shared_storage_uses_only_configured_writable_roots(tmp_path, monkeypatc shared = tmp_path / "shared" shared.mkdir() assert ci.shared_base( - {"storage_roots": [str(tmp_path / "absent"), str(shared)]}, "gb200" + {"storage_roots": [str(tmp_path / "absent"), str(shared)]}, "cluster:gb200-nv" ) == (shared / f".operatorx-{os.getuid()}") monkeypatch.setenv("HOME", str(tmp_path / "runner-local-sandbox")) monkeypatch.setattr( ci.pwd, "getpwuid", lambda uid: types.SimpleNamespace(pw_dir=str(shared)) ) - assert ci.shared_base({}, "b300") == shared / f".operatorx-{os.getuid()}" + assert ci.shared_base({}, "cluster:b300-dsxe") == shared / f".operatorx-{os.getuid()}" with pytest.raises(ValueError, match="shared storage"): - ci.shared_base({"storage_roots": [str(tmp_path / "absent")]}, "gb200") + ci.shared_base({"storage_roots": [str(tmp_path / "absent")]}, "cluster:gb200-nv") def test_amd_staging_uses_shared_runner_root(tmp_path, monkeypatch): @@ -167,10 +167,10 @@ def test_amd_staging_uses_shared_runner_root(tmp_path, monkeypatch): (runner / "_work/_temp").mkdir(parents=True) monkeypatch.setenv("RUNNER_TEMP", str(runner / "_work/_temp")) monkeypatch.setenv("HOME", str(tmp_path / "private-home")) - assert ci.shared_base({}, "mi300x") == runner / f".operatorx-{os.getuid()}" + assert ci.shared_base({}, "cluster:mi300x-amd") == runner / f".operatorx-{os.getuid()}" monkeypatch.setenv("RUNNER_TEMP", str(tmp_path / "unrelated")) with pytest.raises(ValueError, match="shared runner"): - ci.shared_base({}, "mi355x") + ci.shared_base({}, "cluster:mi355x-amds") def test_platform_overlay_preserves_base_and_replaces_explicit_profile(tmp_path): @@ -179,7 +179,7 @@ def test_platform_overlay_preserves_base_and_replaces_explicit_profile(tmp_path) base, { "platforms": { - "fixture": {"gpus_per_node": 4, "operator": {"partition": "old"}} + "mi300x": {"gpus_per_node": 4, "operator": {"partition": "old"}} } }, ) @@ -188,11 +188,11 @@ def test_platform_overlay_preserves_base_and_replaces_explicit_profile(tmp_path) child, { "base": "base.json", - "platforms": {"fixture": {"operator": {"partition": "new"}}}, + "platforms": {"cluster:mi300x-amd": {"operator": {"partition": "new"}}}, }, ) assert ci.load_platforms(child) == { - "fixture": {"gpus_per_node": 4, "operator": {"partition": "new"}} + "cluster:mi300x-amd": {"gpus_per_node": 4, "operator": {"partition": "new"}} } @@ -201,15 +201,15 @@ def test_platform_overlay_preserves_base_and_replaces_explicit_profile(tmp_path) [("flashinfer", [1], "gemm"), ("torch", [2], "gemm"), ("torch", [1], "allreduce")], ) def test_amd_plan_rejects_unimplemented_execution(backend, worlds, kind): - with pytest.raises(ValueError, match="single-GPU torch GEMM"): + with pytest.raises(ValueError, match="AMD CI supports|single-device"): ci.plan( - "mi300x", + "cluster:mi300x-amd", [backend], {"tiny": [{"type": kind, "args": {}}]}, {backend: {"image": "rocm:1"}}, worlds, 50, - platforms("mi300x"), + platforms("cluster:mi300x-amd"), ) @@ -220,8 +220,7 @@ def test_amd_plan_rejects_unimplemented_execution(backend, worlds, kind): def test_strict_benchmark_writes_actual_status( tmp_path, monkeypatch, outcome, expected_rc ): - # The GPU kernel is an external collaborator; selection, exception handling, - # checkpointing, serialization and exit decisions execute the real main(). + # only the GPU kernel is faked; everything else is the real main() backend = types.ModuleType("operatorx.runners.testgpu.backends.kernel") backend.IMPLS = ( [] if outcome == "unclaimed" else [types.SimpleNamespace(op_type="gemm")] @@ -240,9 +239,10 @@ def kernel(op): monkeypatch.setitem(sys.modules, runner.__name__, runner) monkeypatch.setenv("WORLD_SIZE", "1") monkeypatch.setenv("RANK", "0") - monkeypatch.delenv("OPERATORX_MOE_PARALLELISM", raising=False) (tmp_path / "tiny.json").write_text( - json.dumps([{"type": "gemm", "args": {"m": 2}}]) + json.dumps([{"type": "gemm", "sources": ["org/model/q_proj"], + "args": {"m": 2, "n": 64, "k": 64, "a": {"dtype": "bf16"}, + "b": {"dtype": "bf16"}, "out": "bf16"}}]) ) monkeypatch.setattr( sys, @@ -271,30 +271,33 @@ def kernel(op): def test_testlist_loading_and_unknown_selection(tmp_path): - (tmp_path / "one.json").write_text('[{"type":"gemm","args":{"m":7}}]') + (tmp_path / "one.json").write_text('[{"type":"gemm","args":{"m":7},"sources":["org/model/o_proj"]}]') assert benchmark._load_testlists(["one"], tmp_path) == { - "one": [{"type": "gemm", "args": {"m": 7}}] + "one": [{"type": "gemm", "args": {"m": 7}, "sources": ["org/model/o_proj"]}] } + (tmp_path / "bare.json").write_text('[{"type":"gemm","args":{"m":7}}]') + with pytest.raises(SystemExit, match="'sources' list"): + benchmark._load_testlists(["bare"], tmp_path) with pytest.raises(SystemExit, match="unknown testlist"): benchmark._load_testlists(["two"], tmp_path) @pytest.mark.parametrize( - "exit_code,cancel,pool,gpus,architecture", + "exit_code,cancel,runner,gpus,architecture", [ - (0, False, "h100-dgxc", 8, "linux/amd64"), - (3, False, "h100-dgxc", 8, "linux/amd64"), - (0, True, "h100-dgxc", 8, "linux/amd64"), - (0, "queued", "h100-dgxc", 8, "linux/amd64"), - (0, False, "gb200", 4, "linux/arm64"), - (0, False, "gb300", 4, "linux/arm64"), - (0, False, "b300", 8, "linux/amd64"), - (0, False, "mi300x", 8, "linux/amd64"), - (0, False, "mi355x", 8, "linux/amd64"), + (0, False, "cluster:h100-dsxe", 8, "linux/amd64"), + (3, False, "cluster:h100-dsxe", 8, "linux/amd64"), + (0, True, "cluster:h100-dsxe", 8, "linux/amd64"), + (0, "queued", "cluster:h100-dsxe", 8, "linux/amd64"), + (0, False, "cluster:gb200-nv", 4, "linux/arm64"), + (0, False, "cluster:gb300-nv", 4, "linux/arm64"), + (0, False, "cluster:b300-dsxe", 8, "linux/amd64"), + (0, False, "cluster:mi300x-amd", 8, "linux/amd64"), + (0, False, "cluster:mi355x-amds", 8, "linux/amd64"), ], ) def test_allocation_completion_failure_and_cancellation( - tmp_path, exit_code, cancel, pool, gpus, architecture + tmp_path, exit_code, cancel, runner, gpus, architecture ): binaries = tmp_path / "bin" binaries.mkdir() @@ -344,7 +347,7 @@ def test_allocation_completion_failure_and_cancellation( json.dumps( { "platforms": { - pool: { + runner: { "gpus_per_node": gpus, "image_platform": architecture, "operator": { @@ -352,12 +355,12 @@ def test_allocation_completion_failure_and_cancellation( "stage_dir": str(tmp_path / "shared"), "account": "fixture", "cpus_per_node": 128, - **({"qos": "fixture-qos"} if pool != "b300" else {}), + **({"qos": "fixture-qos"} if runner != "cluster:b300-dsxe" else {}), "exclude_nodes": "quarantined", "enroot_cache_path": str(tmp_path / "shared/enroot"), **( {"storage_roots": [str(tmp_path / "shared")]} - if pool == "gb200" + if runner == "cluster:gb200-nv" else {"squash_dir": str(tmp_path / "shared/squash")} ), }, @@ -368,13 +371,13 @@ def test_allocation_completion_failure_and_cancellation( ) manifest = tmp_path / "manifest.json" control = ci.plan( - pool, + runner, ["torch"], {"tiny": [{"type": "gemm", "args": {"m": 2}}]}, {"torch": {"image": "image:1"}}, [1], 1, - platforms(pool, gpus, architecture), + platforms(runner, gpus, architecture), ) control.update(source_sha="abc", run_id="12") control["include"][0]["digest"] = "sha256:" + "a" * 64 @@ -451,12 +454,12 @@ def test_allocation_completion_failure_and_cancellation( assert f"--gres=gpu:{gpus}" in allocation assert "--nodes=1" in allocation assert "--account=fixture" in allocation - if pool == "b300": + if runner == "cluster:b300-dsxe": assert not any(arg.startswith("--qos=") for arg in allocation) else: assert "--qos=fixture-qos" in allocation assert "--exclude=quarantined" in allocation - if pool in {"mi300x", "mi355x"}: + if runner in {"cluster:mi300x-amd", "cluster:mi355x-amds"}: assert "--cpus-per-task=16" in allocation if not cancel: imported = next(c for c in calls if "import" in c["argv"]) @@ -465,12 +468,13 @@ def test_allocation_completion_failure_and_cancellation( assert Path(imported["argv"][0]).name == "srun" launched = next(c["argv"] for c in calls if c["argv"][-1] == "rank") assert "--ntasks=1" in launched - if pool in ("gb200", "gb300", "b300", "h100-dgxc", "mi300x", "mi355x"): + if runner in ("cluster:gb200-nv", "cluster:gb300-nv", "cluster:b300-dsxe", "cluster:h100-dsxe", + "cluster:mi300x-amd", "cluster:mi355x-amds"): assert "--container-remap-root" in launched - if pool in ("b300", "h100-dgxc"): + if runner in ("cluster:b300-dsxe", "cluster:h100-dsxe"): assert "--mpi=none" in launched assert "--mem=0" in allocation - if pool == "mi300x": + if runner == "cluster:mi300x-amd": mounts = next( arg for arg in launched if arg.startswith("--container-mounts=") ) @@ -535,7 +539,7 @@ def test_recovery_refuses_unrelated_pool_or_storage(tmp_path): profile, { "platforms": { - "h100-dgxc": { + "cluster:h100-dsxe": { "operator": {"squash_dir": str(tmp_path / "shared/squash")} } } @@ -543,16 +547,16 @@ def test_recovery_refuses_unrelated_pool_or_storage(tmp_path): ) data = { "run_id": "12", - "cell": {"pool": "h200-dgxc"}, + "cell": {"runner": "cluster:h200-dgxc"}, "stage": str(tmp_path / "unrelated"), } ci.write_json(artifacts / "execution.json", data) - with pytest.raises(ValueError, match="run/pool"): - ci.recover(artifacts, "12", "h100-dgxc", profile, 2) - data["cell"]["pool"] = "h100-dgxc" + with pytest.raises(ValueError, match="run/runner"): + ci.recover(artifacts, "12", "cluster:h100-dsxe", profile, 2) + data["cell"]["runner"] = "cluster:h100-dsxe" ci.write_json(artifacts / "execution.json", data) - with pytest.raises(ValueError, match="pool/user"): - ci.recover(artifacts, "12", "h100-dgxc", profile, 2) + with pytest.raises(ValueError, match="runner/user"): + ci.recover(artifacts, "12", "cluster:h100-dsxe", profile, 2) @pytest.mark.parametrize("release", [True, False]) @@ -581,19 +585,3 @@ def slurm(argv, **kwargs): with pytest.raises(RuntimeError, match="retaining staged evidence"): ci.cleanup(tmp_path, 180) assert clock[0] == 180 - - -def test_amd_plan_keeps_requested_moe_shard_factors_with_one_gpu(): - result = ci.plan( - "mi300x", ["vllm"], - {"tiny": [ - {"type": "moe_gemm", "args": {"num_tokens": 16, "expert_parallel_size": 8}}, - {"type": "moe_gemm", "args": {"num_tokens": 32, "world_size": 2}}, - ]}, - {"vllm": {"image": "rocm:fixture"}}, [1], 50, platforms("mi300x"), - ) - cell = result["include"][0] - assert cell["world_size"] == cell["nodes"] == 1 - assert result["excluded_shapes"] == 1 - assert len(result["include"]) == len(cell["cases"]) == 1 - assert cell["cases"][0]["shape"]["args"]["num_tokens"] == 16 diff --git a/operatorx/tests/test_inferencex_generator.py b/operatorx/tests/test_inferencex_generator.py deleted file mode 100644 index 10aa7c0008..0000000000 --- a/operatorx/tests/test_inferencex_generator.py +++ /dev/null @@ -1,69 +0,0 @@ -"""Check InferenceX generator subprocesses against selected checkout layouts.""" - -import os -import shutil -import sys -from pathlib import Path - -import pytest -import yaml - -sys.path.insert(0, str(Path(__file__).resolve().parents[2])) -from operatorx.scripts.inferencex_testlist.matrix import _run_inferencex_generator - - -@pytest.mark.parametrize("layout", ["legacy", "flat", "nested"]) -def test_generator_uses_requested_checkout_with_safe_path(tmp_path, monkeypatch, layout): - root = tmp_path / "selected checkout" - root.mkdir() - foreign = tmp_path / "other checkout" - (foreign / "infx").mkdir(parents=True) - (foreign / "infx/__init__.py").write_text("raise RuntimeError('wrong checkout')\n") - monkeypatch.setenv("PYTHONPATH", str(foreign)) - monkeypatch.setenv("PYTHONSAFEPATH", "1") - monkeypatch.chdir(tmp_path) - - if layout == "legacy": - # The old generator is an external executable. Its sibling dependency - # distinguishes the selected revision from inherited import paths. - script = root / "utils/matrix_logic/generate_sweep_configs.py" - script.parent.mkdir(parents=True) - script.write_text( - "import json, sys\nfrom validation import model\n" - "assert sys.argv[1:] == ['full-sweep', '--config-files', 'configs/master.yaml', '--no-evals']\n" - "print(json.dumps([{'model': model, 'conc': 2}]))\n" - ) - script.with_name("validation.py").write_text("model = 'legacy revision'\n") - expected = "legacy revision" - else: - source = Path(__file__).resolve().parents[2] / "inferencex-e2e" - shutil.copytree(source / "infx", root / "infx", - ignore=shutil.ignore_patterns("__pycache__", "tests")) - (root / "configs").mkdir() - (root / "configs/runners.yaml").write_text( - 'labels: {fixture: [node-a], "cluster:fixture": [node-a]}\n' - "clusters: {fixture: {gpus-per-node: 8, arch: x86_64, scheduler: slurm," - " slurm: {partition: p, exclusive: false}}}\n" - ) - (root / "configs/master.yaml").write_text(yaml.safe_dump({"fixture": { - "image": "example/image:stable", "model": "selected revision", - "model-prefix": "dsr1", "precision": "fp8", "framework": "sglang", - "runner": "fixture", "multinode": False, "srt-recipe-dir": "fixture", - "scenarios": {"fixed-seq-len": [{ - "isl": 1024, "osl": 1024, - "search-space": [{"tp": 1, "conc-list": [2], "srt-recipe": "recipe.yaml"}], - }]}, - }})) - recipe = root / "benchmarks/single_node/srt-slurm-recipes/fixture/recipe.yaml" - recipe.parent.mkdir(parents=True) - recipe.write_text("{}\n") - expected = "selected revision" - - if layout == "nested": - nested = root / "inferencex-e2e" - nested.mkdir() - for name in ("infx", "configs", "benchmarks"): - (root / name).rename(nested / name) - - rows = _run_inferencex_generator(os.path.relpath(root), ["configs/master.yaml"], ["--no-evals"]) - assert [(row["model"], row["conc"]) for row in rows] == [(expected, 2)] diff --git a/operatorx/tests/test_moe.py b/operatorx/tests/test_moe.py deleted file mode 100644 index d920d73aa1..0000000000 --- a/operatorx/tests/test_moe.py +++ /dev/null @@ -1,72 +0,0 @@ -"""Exercise routed MoE shard derivation and tensor preparation.""" - -import sys -from pathlib import Path -from types import SimpleNamespace - -import pytest -import torch - -sys.path.insert(0, str(Path(__file__).resolve().parents[2])) -from operatorx.core import Op, UnsupportedOpError -from operatorx.runners import moe - - -def args(**overrides): - return { - "num_tokens": 2, - "hidden": 4, - "intermediate": 8, - "num_experts": 6, - "top_k": 2, - "dtype_act": "bf16", - "dtype_weight": "bf16", - "expert_parallel_size": 2, - "routed_tensor_parallel_size": 2, - **overrides, - } - - -def test_prepare_builds_local_weights_and_normalized_distinct_routes(monkeypatch): - # Substitute GPU allocation and the external GPU kernel import, not tensor math. - for name in ("empty", "randn"): - original = getattr(torch, name) - - def allocate(*values, _original=original, **kwargs): - kwargs["device"] = "cpu" - return _original(*values, **kwargs) - - monkeypatch.setattr(torch, name, allocate) - monkeypatch.setattr( - moe, "import_module", lambda _: SimpleNamespace(fused_experts=None) - ) - ctx = moe.prepare(Op("moe_gemm", args(), "vllm")) - assert ctx["x"].shape == (2, 4) - assert ctx["w1"].shape == (3, 8, 4) - assert ctx["w2"].shape == (3, 4, 4) - assert ctx["w1"].dtype == ctx["w2"].dtype == torch.bfloat16 - assert ctx["topk_ids"].dtype == torch.int32 - assert ctx["topk_ids"].shape == (2, 2) - assert torch.all((ctx["topk_ids"] >= 0) & (ctx["topk_ids"] < 3)) - assert torch.all(ctx["topk_ids"][:, 0] != ctx["topk_ids"][:, 1]) - torch.testing.assert_close( - ctx["topk_weights"].sum(dim=-1), torch.tensor([1.0, 1.0]) - ) - - -@pytest.mark.parametrize( - "overrides", - [ - {"num_tokens": 0}, - {"expert_parallel_size": 0}, - {"num_experts": 5}, - {"intermediate": 7}, - {"top_k": 4}, - {"dtype_weight": "fp8"}, - {"n_shared_experts": 1}, - {"expert_distribution": "single_hot"}, - ], -) -def test_unsupported_requests_fail_before_gpu_allocation(overrides): - with pytest.raises(UnsupportedOpError): - moe.prepare(Op("moe_gemm", args(**overrides), "vllm")) diff --git a/operatorx/uv.lock b/operatorx/uv.lock index cd10f1c2a0..d1fabdbe38 100644 --- a/operatorx/uv.lock +++ b/operatorx/uv.lock @@ -2,6 +2,10 @@ version = 1 revision = 3 requires-python = ">=3.10" +[options] +exclude-newer = "2026-09-26T15:44:46.373722019Z" +exclude-newer-span = "PT12H" + [[package]] name = "operatorx" version = "0.1.0"